open-classify 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -63
- package/dist/src/aggregator.d.ts +7 -23
- package/dist/src/aggregator.js +108 -186
- package/dist/src/classifiers/{routing → model_tier}/manifest.json +2 -2
- package/dist/src/classifiers/{routing → model_tier}/prompt.md +1 -1
- package/dist/src/classifiers/preflight/manifest.json +9 -8
- package/dist/src/classifiers/preflight/prompt.md +12 -6
- package/dist/src/classifiers/prompt_injection/manifest.json +2 -3
- package/dist/src/classify.d.ts +1 -2
- package/dist/src/classify.js +0 -2
- package/dist/src/config.d.ts +0 -2
- package/dist/src/config.js +1 -23
- package/dist/src/index.js +2 -3
- package/dist/src/manifest.d.ts +25 -70
- package/dist/src/pipeline.d.ts +1 -2
- package/dist/src/pipeline.js +22 -89
- package/dist/src/stock-validation.js +8 -4
- package/docs/adding-a-classifier.md +5 -3
- package/docs/manifests.md +6 -6
- package/docs/resolver.md +20 -44
- package/docs/signals.md +18 -8
- package/open-classify.config.example.json +1 -4
- package/package.json +1 -1
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
|
-
"name": "
|
|
2
|
+
"name": "model_tier",
|
|
3
3
|
"version": "1.0.0",
|
|
4
4
|
"purpose": "Recommend the downstream model tier.",
|
|
5
5
|
"dispatch_order": 20,
|
|
6
6
|
"reserved_fields": ["model_tier"],
|
|
7
7
|
"fallback": {
|
|
8
|
-
"reason": "Classifier failed; no
|
|
8
|
+
"reason": "Classifier failed; no model tier signal.",
|
|
9
9
|
"certainty": "no_signal"
|
|
10
10
|
}
|
|
11
11
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
You are the
|
|
1
|
+
You are the model_tier classifier for an AI assistant routing system.
|
|
2
2
|
|
|
3
3
|
Pick the coarse model tier that best fits the target user message. Emit only `model_tier`; do not infer specialization, tools, or prompt-injection risk — other classifiers own those axes.
|
|
4
4
|
|
|
@@ -1,29 +1,30 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "preflight",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"purpose": "
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"purpose": "Assess whether the latest message can be answered immediately (final_reply) or should route downstream with an acknowledgement (ack_reply). Always emits exactly one.",
|
|
5
5
|
"dispatch_order": 10,
|
|
6
6
|
"reserved_fields": ["final_reply", "ack_reply"],
|
|
7
7
|
"output_schema": {
|
|
8
8
|
"examples": [
|
|
9
9
|
{
|
|
10
|
-
"reason": "
|
|
10
|
+
"reason": "Simple greeting — answerable directly.",
|
|
11
11
|
"certainty": "near_certain",
|
|
12
12
|
"final_reply": { "text": "Hi!" }
|
|
13
13
|
},
|
|
14
14
|
{
|
|
15
|
-
"reason": "Trivial arithmetic.",
|
|
15
|
+
"reason": "Trivial arithmetic — answerable directly.",
|
|
16
16
|
"certainty": "very_strong",
|
|
17
17
|
"final_reply": { "text": "4" }
|
|
18
18
|
},
|
|
19
19
|
{
|
|
20
|
-
"reason": "
|
|
20
|
+
"reason": "Code review task requires substantive downstream work.",
|
|
21
21
|
"certainty": "very_strong",
|
|
22
|
-
"ack_reply": { "text": "On it." }
|
|
22
|
+
"ack_reply": { "text": "On it — reviewing the code now." }
|
|
23
23
|
},
|
|
24
24
|
{
|
|
25
|
-
"reason": "
|
|
26
|
-
"certainty": "strong"
|
|
25
|
+
"reason": "Reminder request requires downstream action.",
|
|
26
|
+
"certainty": "strong",
|
|
27
|
+
"ack_reply": { "text": "Got it, I'll set that reminder for 3pm." }
|
|
27
28
|
}
|
|
28
29
|
]
|
|
29
30
|
},
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
You are the preflight classifier for an AI assistant routing system.
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Your primary task is to assess: **can you fully answer the target message yourself**, given the conversation history? Make this judgment first — the reply text follows from it.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
**Step 1 — assess whether you can fully answer:**
|
|
6
|
+
Ask yourself: Is the intent clear? Is the answer fully derivable from context right now, without real-time data, external tools, code execution, non-trivial generation, analysis, or judgment? Would a one-sentence reply genuinely resolve the request?
|
|
7
|
+
|
|
8
|
+
If yes → emit `final_reply` with the complete answer.
|
|
9
|
+
|
|
10
|
+
If no (the downstream model should handle it) → emit `ack_reply` with a brief, contextually specific acknowledgement that shows you understood the request. The ack must reflect the actual request — not a generic "On it." — so the user knows their message was understood while the model works.
|
|
9
11
|
|
|
10
|
-
|
|
12
|
+
**Rule: always emit exactly one of `final_reply` or `ack_reply`. Never emit both. Never emit neither.**
|
|
13
|
+
|
|
14
|
+
- `final_reply` is for tiny terminal answers only: greetings, thanks, spelling lookups, simple arithmetic, yes/no factual questions answerable from context. If answering requires drafting, rewriting, analysis, coding, research, planning, or any substantive generation — use `ack_reply` instead.
|
|
15
|
+
- `ack_reply` text must not contain the answer. It acknowledges the request and confirms it is being worked on.
|
|
16
|
+
- Do not address the user anywhere except inside `final_reply.text` or `ack_reply.text`.
|
|
@@ -9,8 +9,7 @@
|
|
|
9
9
|
"required": ["risk_level"]
|
|
10
10
|
},
|
|
11
11
|
"fallback": {
|
|
12
|
-
"reason": "Classifier failed; prompt-injection risk
|
|
13
|
-
"certainty": "no_signal"
|
|
14
|
-
"risk_level": "unknown"
|
|
12
|
+
"reason": "Classifier failed; prompt-injection risk could not be assessed.",
|
|
13
|
+
"certainty": "no_signal"
|
|
15
14
|
}
|
|
16
15
|
}
|
package/dist/src/classify.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type RunClassifier } from "./classifiers.js";
|
|
2
2
|
import { type OpenClassifyConfig } from "./config.js";
|
|
3
|
-
import type {
|
|
3
|
+
import type { Catalog, InspectResult, PipelineResult } from "./manifest.js";
|
|
4
4
|
import type { OpenClassifyInput } from "./types.js";
|
|
5
5
|
export type Classifier = (input: OpenClassifyInput, options?: {
|
|
6
6
|
signal?: AbortSignal;
|
|
@@ -25,6 +25,5 @@ export interface CreateClassifierOptions {
|
|
|
25
25
|
classifierTimeoutMs?: number;
|
|
26
26
|
classifierRetryCount?: number;
|
|
27
27
|
maxConcurrency?: number;
|
|
28
|
-
aggregator?: AggregatorConfig;
|
|
29
28
|
}
|
|
30
29
|
export declare function createClassifier(options?: CreateClassifierOptions): OpenClassify;
|
package/dist/src/classify.js
CHANGED
|
@@ -28,7 +28,6 @@ export function createClassifier(options = {}) {
|
|
|
28
28
|
});
|
|
29
29
|
const catalog = options.catalog ??
|
|
30
30
|
loadCatalog(options.catalogPath ?? fileConfig?.catalog ?? OLLAMA_DEFAULT_CATALOG_PATH);
|
|
31
|
-
const aggregator = options.aggregator ?? fileConfig?.aggregator;
|
|
32
31
|
let resourceCheck;
|
|
33
32
|
const ensureResources = async () => {
|
|
34
33
|
if (!needsResourceCheck)
|
|
@@ -47,7 +46,6 @@ export function createClassifier(options = {}) {
|
|
|
47
46
|
classifierTimeoutMs: options.classifierTimeoutMs,
|
|
48
47
|
classifierRetryCount: options.classifierRetryCount,
|
|
49
48
|
maxConcurrency: options.maxConcurrency,
|
|
50
|
-
aggregator,
|
|
51
49
|
signal: callOptions?.signal,
|
|
52
50
|
});
|
|
53
51
|
};
|
package/dist/src/config.d.ts
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import { type ClassifierName } from "./classifiers.js";
|
|
2
|
-
import { type AggregatorConfig } from "./manifest.js";
|
|
3
2
|
export declare const DEFAULT_OPEN_CLASSIFY_CONFIG_PATH = "open-classify.config.json";
|
|
4
3
|
export interface OpenClassifyConfig {
|
|
5
4
|
readonly runner?: OllamaRunnerConfig;
|
|
6
5
|
readonly catalog?: string;
|
|
7
|
-
readonly aggregator?: AggregatorConfig;
|
|
8
6
|
}
|
|
9
7
|
export interface OllamaRunnerConfig {
|
|
10
8
|
readonly provider: "ollama";
|
package/dist/src/config.js
CHANGED
|
@@ -30,25 +30,10 @@ export function validateOpenClassifyConfig(value, path = "open-classify config")
|
|
|
30
30
|
if (!isRecord(value)) {
|
|
31
31
|
throwConfig(path, "config must be a JSON object");
|
|
32
32
|
}
|
|
33
|
-
ensureAllowedKeys(value, ["runner", "catalog"
|
|
33
|
+
ensureAllowedKeys(value, ["runner", "catalog"], path, "<root>");
|
|
34
34
|
return {
|
|
35
35
|
...(value.runner === undefined ? {} : { runner: validateRunner(value.runner, path) }),
|
|
36
36
|
...(value.catalog === undefined ? {} : { catalog: requireString(value.catalog, path, "catalog") }),
|
|
37
|
-
...(value.aggregator === undefined ? {} : { aggregator: validateAggregator(value.aggregator, path) }),
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
|
-
function validateAggregator(value, path) {
|
|
41
|
-
if (!isRecord(value)) {
|
|
42
|
-
throwConfig(path, "aggregator must be an object");
|
|
43
|
-
}
|
|
44
|
-
ensureAllowedKeys(value, ["certaintyThreshold", "confidenceThreshold"], path, "aggregator");
|
|
45
|
-
return {
|
|
46
|
-
...(value.certaintyThreshold === undefined
|
|
47
|
-
? {}
|
|
48
|
-
: { certaintyThreshold: requireUnitFloat(value.certaintyThreshold, path, "aggregator.certaintyThreshold") }),
|
|
49
|
-
...(value.confidenceThreshold === undefined
|
|
50
|
-
? {}
|
|
51
|
-
: { confidenceThreshold: requireUnitFloat(value.confidenceThreshold, path, "aggregator.confidenceThreshold") }),
|
|
52
37
|
};
|
|
53
38
|
}
|
|
54
39
|
function validateRunner(value, path) {
|
|
@@ -118,13 +103,6 @@ function requireNumber(value, path, field) {
|
|
|
118
103
|
}
|
|
119
104
|
return value;
|
|
120
105
|
}
|
|
121
|
-
function requireUnitFloat(value, path, field) {
|
|
122
|
-
const number = requireNumber(value, path, field);
|
|
123
|
-
if (number < 0 || number > 1) {
|
|
124
|
-
throwConfig(path, `${field} must be a finite number between 0 and 1 inclusive`);
|
|
125
|
-
}
|
|
126
|
-
return number;
|
|
127
|
-
}
|
|
128
106
|
function ensureAllowedKeys(value, allowedKeys, path, field) {
|
|
129
107
|
const allowed = new Set(allowedKeys);
|
|
130
108
|
for (const key of Object.keys(value)) {
|
package/dist/src/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
// Public barrel for the Open Classify package. Everything an external caller
|
|
2
2
|
// would need — input types, enums, the registry, the pipeline, the Ollama
|
|
3
|
-
// runner, the catalog loader
|
|
4
|
-
//
|
|
5
|
-
// consumers can import from `open-classify`.
|
|
3
|
+
// runner, the catalog loader — is re-exported here. The build emits a single
|
|
4
|
+
// `index.js` that downstream consumers can import from `open-classify`.
|
|
6
5
|
export * from "./aggregator.js";
|
|
7
6
|
export * from "./catalog.js";
|
|
8
7
|
export * from "./classifiers.js";
|
package/dist/src/manifest.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { ClassifierInput
|
|
3
|
-
import type { DownstreamModelTier, ModelSpecialization } from "./enums.js";
|
|
1
|
+
import type { RuntimeClassifierManifest } from "./stock.js";
|
|
2
|
+
import type { ClassifierInput } from "./types.js";
|
|
3
|
+
import type { DownstreamModelTier, ModelSpecialization, PromptInjectionRiskLevel } from "./enums.js";
|
|
4
4
|
export type ClassifierName = string;
|
|
5
|
-
export type ClassifierResults = Record<ClassifierName, ClassifierOutput>;
|
|
6
|
-
export type RunClassifier = (name: ClassifierName, input: ClassifierInput, signal: AbortSignal) => Promise<ClassifierOutput>;
|
|
5
|
+
export type ClassifierResults = Record<ClassifierName, import("./stock.js").ClassifierOutput>;
|
|
6
|
+
export type RunClassifier = (name: ClassifierName, input: ClassifierInput, signal: AbortSignal) => Promise<import("./stock.js").ClassifierOutput>;
|
|
7
7
|
export interface CatalogEntry {
|
|
8
8
|
readonly id: string;
|
|
9
9
|
readonly specializations: ReadonlyArray<ModelSpecialization>;
|
|
@@ -18,78 +18,33 @@ export interface Catalog {
|
|
|
18
18
|
readonly models: ReadonlyArray<CatalogEntry>;
|
|
19
19
|
readonly default: string;
|
|
20
20
|
}
|
|
21
|
-
export interface ModelRecommendationResolution {
|
|
22
|
-
readonly constraints_used: Partial<{
|
|
23
|
-
model_specialization: ModelSpecialization;
|
|
24
|
-
model_tier: DownstreamModelTier;
|
|
25
|
-
}>;
|
|
26
|
-
readonly constraints_dropped: ReadonlyArray<{
|
|
27
|
-
readonly axis: "model_specialization" | "model_tier";
|
|
28
|
-
readonly reason: "low_confidence" | "no_match_relaxed" | "default_fallback";
|
|
29
|
-
}>;
|
|
30
|
-
readonly confidences: Partial<{
|
|
31
|
-
routing: number;
|
|
32
|
-
}>;
|
|
33
|
-
readonly fell_back_to_default: boolean;
|
|
34
|
-
}
|
|
35
|
-
export interface ModelRecommendation {
|
|
36
|
-
readonly id: string;
|
|
37
|
-
readonly params_in_billions: number | null;
|
|
38
|
-
readonly context_window: number;
|
|
39
|
-
readonly input_tokens_cpm?: number;
|
|
40
|
-
readonly cached_tokens_cpm?: number;
|
|
41
|
-
readonly output_tokens_cpm?: number;
|
|
42
|
-
readonly resolution: ModelRecommendationResolution;
|
|
43
|
-
}
|
|
44
|
-
export interface Envelope {
|
|
45
|
-
readonly final_reply?: FinalReplySignal;
|
|
46
|
-
readonly ack_reply?: AckReplySignal;
|
|
47
|
-
readonly routing?: RoutingSignal;
|
|
48
|
-
readonly tools?: ToolsSignal;
|
|
49
|
-
readonly prompt_injection?: PromptInjectionSignal;
|
|
50
|
-
readonly classifier_outputs: ReadonlyArray<ClassifierAuditOutput>;
|
|
51
|
-
readonly model_recommendation: ModelRecommendation;
|
|
52
|
-
}
|
|
53
21
|
export type ClassifierPublicOutputs = Record<string, Record<string, unknown>>;
|
|
54
|
-
export
|
|
55
|
-
|
|
22
|
+
export type PipelineAction = "route" | "block" | "reply";
|
|
23
|
+
export type BlockReason = "prompt_injection" | "classification_error";
|
|
24
|
+
export interface ReplySignal {
|
|
56
25
|
readonly text: string;
|
|
57
|
-
readonly hash: string;
|
|
58
26
|
}
|
|
59
|
-
export interface
|
|
60
|
-
readonly
|
|
61
|
-
readonly
|
|
62
|
-
readonly tools: ToolsSignal;
|
|
63
|
-
}
|
|
64
|
-
export type ClassifierEntry = ClassifierOutput & {
|
|
65
|
-
readonly status: ClassifierRunStatus;
|
|
66
|
-
readonly version: string;
|
|
67
|
-
};
|
|
68
|
-
export interface CertaintySummary {
|
|
69
|
-
readonly min: number;
|
|
70
|
-
readonly avg: number;
|
|
71
|
-
}
|
|
72
|
-
export interface PipelineMeta {
|
|
73
|
-
readonly classifiers: Record<string, ClassifierEntry>;
|
|
74
|
-
readonly certainty: CertaintySummary;
|
|
75
|
-
}
|
|
76
|
-
export interface PipelineAudit extends Envelope {
|
|
77
|
-
readonly meta: PipelineMeta;
|
|
78
|
-
}
|
|
79
|
-
export interface InspectResult {
|
|
27
|
+
export interface PipelineResult {
|
|
28
|
+
readonly action: PipelineAction;
|
|
29
|
+
readonly block_reason?: BlockReason;
|
|
80
30
|
readonly target_message_hash: string;
|
|
31
|
+
readonly model_id: string | null;
|
|
32
|
+
readonly tools: ReadonlyArray<string>;
|
|
33
|
+
readonly reply: ReplySignal | null;
|
|
34
|
+
readonly prompt_injection: {
|
|
35
|
+
readonly risk_level: PromptInjectionRiskLevel;
|
|
36
|
+
} | null;
|
|
37
|
+
readonly avg_certainty: number;
|
|
38
|
+
readonly min_certainty: number;
|
|
39
|
+
readonly failed_classifiers: ReadonlyArray<string>;
|
|
81
40
|
readonly classifier_outputs: ClassifierPublicOutputs;
|
|
82
41
|
}
|
|
83
|
-
export interface
|
|
84
|
-
readonly action: "route";
|
|
42
|
+
export interface InspectResult {
|
|
85
43
|
readonly target_message_hash: string;
|
|
86
|
-
readonly
|
|
44
|
+
readonly message: {
|
|
45
|
+
readonly role: "assistant";
|
|
46
|
+
readonly text: string;
|
|
47
|
+
};
|
|
87
48
|
readonly classifier_outputs: ClassifierPublicOutputs;
|
|
88
|
-
readonly audit: PipelineAudit;
|
|
89
|
-
}
|
|
90
|
-
export interface AggregatorConfig {
|
|
91
|
-
readonly certaintyThreshold?: number;
|
|
92
|
-
/** @deprecated Use certaintyThreshold. */
|
|
93
|
-
readonly confidenceThreshold?: number;
|
|
94
49
|
}
|
|
95
50
|
export type ClassifierRegistry = ReadonlyArray<RuntimeClassifierManifest>;
|
package/dist/src/pipeline.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type RunClassifier } from "./classifiers.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { Catalog, InspectResult, PipelineResult } from "./manifest.js";
|
|
3
3
|
import type { OpenClassifyInput } from "./types.js";
|
|
4
4
|
export declare const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15000;
|
|
5
5
|
export declare const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
|
|
@@ -13,7 +13,6 @@ export interface ClassifyOptions {
|
|
|
13
13
|
classifierTimeoutMs?: number;
|
|
14
14
|
classifierRetryCount?: number;
|
|
15
15
|
maxConcurrency?: number;
|
|
16
|
-
aggregator?: AggregatorConfig;
|
|
17
16
|
signal?: AbortSignal;
|
|
18
17
|
}
|
|
19
18
|
export interface InspectOptions {
|
package/dist/src/pipeline.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { assembleResult, buildPublicOutputs } from "./aggregator.js";
|
|
2
2
|
import { MODULES_BY_NAME, REGISTRY, } from "./classifiers.js";
|
|
3
3
|
import { normalizeOpenClassifyInput, toClassifierInput } from "./input.js";
|
|
4
|
-
import { certaintyScore } from "./stock.js";
|
|
5
4
|
export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15_000;
|
|
6
5
|
export const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
|
|
7
6
|
export const DEFAULT_MAX_CONCURRENCY = 7;
|
|
@@ -12,22 +11,27 @@ export class OpenClassifyNormalizationError extends Error {
|
|
|
12
11
|
}
|
|
13
12
|
}
|
|
14
13
|
export async function classifyOpenClassifyInput(input, options) {
|
|
15
|
-
const { request, results,
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
registry:
|
|
14
|
+
const { request, results, failedClassifiers } = await runPipeline(input, "user", options);
|
|
15
|
+
const reg = filteredRegistry("user");
|
|
16
|
+
const assembled = assembleResult({
|
|
17
|
+
registry: reg,
|
|
19
18
|
results,
|
|
19
|
+
failedClassifiers,
|
|
20
20
|
catalog: options.catalog,
|
|
21
|
-
input: classifierInput,
|
|
22
|
-
config: options.aggregator,
|
|
23
21
|
});
|
|
24
|
-
return
|
|
22
|
+
return {
|
|
23
|
+
...assembled,
|
|
24
|
+
target_message_hash: request.target_message_hash,
|
|
25
|
+
};
|
|
25
26
|
}
|
|
26
27
|
export async function inspectOpenClassifyInput(input, options) {
|
|
27
28
|
const { request, results } = await runPipeline(input, "assistant", options);
|
|
29
|
+
const reg = filteredRegistry("assistant");
|
|
30
|
+
const lastMsg = request.messages[request.messages.length - 1];
|
|
28
31
|
return {
|
|
29
32
|
target_message_hash: request.target_message_hash,
|
|
30
|
-
|
|
33
|
+
message: { role: "assistant", text: lastMsg.text },
|
|
34
|
+
classifier_outputs: buildPublicOutputs(reg, results),
|
|
31
35
|
};
|
|
32
36
|
}
|
|
33
37
|
async function runPipeline(input, role, options) {
|
|
@@ -52,15 +56,12 @@ async function runPipeline(input, role, options) {
|
|
|
52
56
|
const classifierTimeoutMs = options.classifierTimeoutMs ?? DEFAULT_CLASSIFIER_TIMEOUT_MS;
|
|
53
57
|
const classifierRetryCount = options.classifierRetryCount ?? DEFAULT_CLASSIFIER_RETRY_COUNT;
|
|
54
58
|
const maxConcurrency = resolveMaxConcurrency(options.maxConcurrency);
|
|
55
|
-
// REGISTRY is already sorted by `dispatch_order` ascending. Filter by
|
|
56
|
-
// applies_to so we only dispatch classifiers relevant to this role; the
|
|
57
|
-
// worker pool then runs them in the remaining order.
|
|
58
59
|
const registry = filteredRegistry(role);
|
|
59
60
|
const queue = registry.map((m) => m.name);
|
|
60
61
|
try {
|
|
61
62
|
const settled = await runWithConcurrency(queue, maxConcurrency, controller.signal, (name) => runClassifierWithRetry(name, classifierInput, options.runClassifier, controller.signal, classifierTimeoutMs, classifierRetryCount));
|
|
62
|
-
const { results,
|
|
63
|
-
return { request, results,
|
|
63
|
+
const { results, failedClassifiers } = collectResults(settled);
|
|
64
|
+
return { request, results, failedClassifiers };
|
|
64
65
|
}
|
|
65
66
|
finally {
|
|
66
67
|
options.signal?.removeEventListener("abort", abortFromOptions);
|
|
@@ -91,9 +92,6 @@ async function runWithConcurrency(names, maxConcurrency, signal, start) {
|
|
|
91
92
|
return;
|
|
92
93
|
const name = names[i];
|
|
93
94
|
if (signal.aborted) {
|
|
94
|
-
// Queued classifiers that never started are reported as not-run so
|
|
95
|
-
// the audit shows their fallback in `meta.classifiers`. In-flight
|
|
96
|
-
// classifiers receive the abort signal directly and resolve normally.
|
|
97
95
|
results[i] = {
|
|
98
96
|
ok: false,
|
|
99
97
|
name,
|
|
@@ -109,71 +107,16 @@ async function runWithConcurrency(names, maxConcurrency, signal, start) {
|
|
|
109
107
|
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
110
108
|
return results;
|
|
111
109
|
}
|
|
112
|
-
function
|
|
110
|
+
function collectResults(settled) {
|
|
113
111
|
const results = {};
|
|
114
|
-
const
|
|
112
|
+
const failedClassifiers = [];
|
|
115
113
|
for (const s of settled) {
|
|
116
114
|
const manifest = MODULES_BY_NAME[s.name];
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
...value,
|
|
121
|
-
status: classifierRunStatus(s),
|
|
122
|
-
version: manifest.version,
|
|
123
|
-
};
|
|
115
|
+
results[s.name] = s.ok ? s.value : manifest.fallback;
|
|
116
|
+
if (!s.ok)
|
|
117
|
+
failedClassifiers.push(s.name);
|
|
124
118
|
}
|
|
125
|
-
return { results,
|
|
126
|
-
}
|
|
127
|
-
function certaintySummary(results, registry) {
|
|
128
|
-
const scores = registry.map((m) => scoreCertainty(results[m.name]?.certainty));
|
|
129
|
-
if (scores.length === 0)
|
|
130
|
-
return { min: 0, avg: 0 };
|
|
131
|
-
const min = Math.min(...scores);
|
|
132
|
-
const avg = scores.reduce((sum, v) => sum + v, 0) / scores.length;
|
|
133
|
-
return { min, avg };
|
|
134
|
-
}
|
|
135
|
-
function scoreCertainty(certainty) {
|
|
136
|
-
return certainty === undefined ? 0 : certaintyScore[certainty];
|
|
137
|
-
}
|
|
138
|
-
function buildRouteResult(request, envelope, results, meta) {
|
|
139
|
-
const downstream = {
|
|
140
|
-
model_id: envelope.model_recommendation.id,
|
|
141
|
-
target_message: {
|
|
142
|
-
role: "user",
|
|
143
|
-
text: request.text,
|
|
144
|
-
hash: request.target_message_hash,
|
|
145
|
-
},
|
|
146
|
-
tools: envelope.tools ?? { tools: [] },
|
|
147
|
-
};
|
|
148
|
-
return {
|
|
149
|
-
action: "route",
|
|
150
|
-
target_message_hash: request.target_message_hash,
|
|
151
|
-
downstream,
|
|
152
|
-
classifier_outputs: classifierPublicOutputs(filteredRegistry("user"), results),
|
|
153
|
-
audit: {
|
|
154
|
-
...envelope,
|
|
155
|
-
meta,
|
|
156
|
-
},
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
// Expose each classifier's payload — every output field except `reason` and
|
|
160
|
-
// `certainty`. Iterates the supplied registry so we only surface classifiers
|
|
161
|
-
// that actually ran for this role.
|
|
162
|
-
function classifierPublicOutputs(registry, results) {
|
|
163
|
-
const out = {};
|
|
164
|
-
for (const manifest of registry) {
|
|
165
|
-
const result = results[manifest.name];
|
|
166
|
-
if (result === undefined)
|
|
167
|
-
continue;
|
|
168
|
-
out[manifest.name] = stripMetadata(result);
|
|
169
|
-
}
|
|
170
|
-
return out;
|
|
171
|
-
}
|
|
172
|
-
function stripMetadata(output) {
|
|
173
|
-
const { reason, certainty, ...payload } = output;
|
|
174
|
-
void reason;
|
|
175
|
-
void certainty;
|
|
176
|
-
return payload;
|
|
119
|
+
return { results, failedClassifiers };
|
|
177
120
|
}
|
|
178
121
|
async function runClassifierWithRetry(name, input, runClassifier, rootSignal, timeoutMs, retryCount) {
|
|
179
122
|
let lastError = new Error(`${name} classifier did not run`);
|
|
@@ -219,16 +162,6 @@ async function runClassifierAttempt(name, input, runClassifier, rootSignal, time
|
|
|
219
162
|
rootSignal.removeEventListener("abort", abortAttempt);
|
|
220
163
|
}
|
|
221
164
|
}
|
|
222
|
-
function classifierRunStatus(settled) {
|
|
223
|
-
if (settled.ok)
|
|
224
|
-
return { ok: true, source: "model" };
|
|
225
|
-
return {
|
|
226
|
-
ok: false,
|
|
227
|
-
source: "fallback",
|
|
228
|
-
reason: settled.reason,
|
|
229
|
-
error: errorMessage(settled.error),
|
|
230
|
-
};
|
|
231
|
-
}
|
|
232
165
|
function errorMessage(error) {
|
|
233
166
|
return error instanceof Error ? error.message : String(error);
|
|
234
167
|
}
|
|
@@ -212,10 +212,14 @@ function validateFallback(raw, composedSchema, classifier, model) {
|
|
|
212
212
|
if (!isRecord(raw)) {
|
|
213
213
|
throwInvalid(classifier, model, "fallback must be a JSON object");
|
|
214
214
|
}
|
|
215
|
-
// Fallback
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
|
|
215
|
+
// Fallback is the "no signal" state: only reason and certainty are required.
|
|
216
|
+
// Strip any custom `required` entries beyond those two so that reserved fields
|
|
217
|
+
// and output_schema.required fields don't force the fallback to emit values
|
|
218
|
+
// it cannot meaningfully provide when the classifier has failed.
|
|
219
|
+
const fallbackSchema = isRecord(composedSchema)
|
|
220
|
+
? { ...composedSchema, required: ["reason", "certainty"] }
|
|
221
|
+
: composedSchema;
|
|
222
|
+
const validate = ajv.compile(fallbackSchema);
|
|
219
223
|
if (!validate(raw)) {
|
|
220
224
|
const message = formatSchemaErrors(validate.errors, "fallback");
|
|
221
225
|
throwInvalid(classifier, model, `fallback is invalid: ${message}`);
|
|
@@ -71,7 +71,7 @@ Rules:
|
|
|
71
71
|
- `name` must match the directory name.
|
|
72
72
|
- Reserved field names cannot appear in `output_schema.properties`; declare them in `reserved_fields` instead.
|
|
73
73
|
- `reason` and `certainty` are added to the composed schema by the runtime — don't declare them.
|
|
74
|
-
- `fallback` must validate against the composed schema.
|
|
74
|
+
- `fallback` must validate against the composed schema. Only `reason` and `certainty` are required in fallback; reserved fields and `output_schema.required` fields are exempt (a "no signal" fallback usually omits them).
|
|
75
75
|
- `output_schema.examples` (JSON Schema standard) must validate against the composed schema at load time, so a broken example fails the build, not the model call.
|
|
76
76
|
|
|
77
77
|
See [manifests.md](manifests.md) for the full field list.
|
|
@@ -107,7 +107,7 @@ const result = await classify(input);
|
|
|
107
107
|
const tags = result.classifier_outputs.topic_tags?.tags ?? [];
|
|
108
108
|
```
|
|
109
109
|
|
|
110
|
-
`
|
|
110
|
+
`classifier_outputs[name]` includes all payload fields plus `reason` (string) and `certainty` (float).
|
|
111
111
|
|
|
112
112
|
## Targeting the assistant response
|
|
113
113
|
|
|
@@ -117,7 +117,7 @@ Classifiers run against the user message by default. To run a classifier against
|
|
|
117
117
|
- `"assistant"` — only `inspect()` runs it.
|
|
118
118
|
- `"both"` — both passes run it.
|
|
119
119
|
|
|
120
|
-
Use `inspect()` from `createClassifier()` for the assistant-side pass. It returns a lean shape
|
|
120
|
+
Use `inspect()` from `createClassifier()` for the assistant-side pass. It returns a lean shape: `target_message_hash`, the `message` that was inspected, and `classifier_outputs`. No routing, no action, no block logic.
|
|
121
121
|
|
|
122
122
|
```ts
|
|
123
123
|
const { inspect } = createClassifier({ catalog });
|
|
@@ -130,6 +130,8 @@ const post = await inspect({
|
|
|
130
130
|
const risk = post.classifier_outputs.prompt_injection?.risk_level;
|
|
131
131
|
```
|
|
132
132
|
|
|
133
|
+
The built-in `prompt_injection` ships tagged `"both"` so it runs on both sides.
|
|
134
|
+
|
|
133
135
|
## Choosing the classifier model
|
|
134
136
|
|
|
135
137
|
For apps and OSS installs, prefer `open-classify.config.json`:
|
package/docs/manifests.md
CHANGED
|
@@ -17,7 +17,7 @@ The loader skips any top-level directory whose name starts with `_` (those are s
|
|
|
17
17
|
| Field | Required | Description |
|
|
18
18
|
|---|---|---|
|
|
19
19
|
| `name` | yes | Classifier id. Must match the directory name. |
|
|
20
|
-
| `version` | yes | Contract version
|
|
20
|
+
| `version` | yes | Contract version string for this classifier. |
|
|
21
21
|
| `purpose` | yes | Human-readable description of the classifier's job. Treated as a hard scope boundary in the prompt. |
|
|
22
22
|
| `dispatch_order` | no | Non-negative integer scheduling priority. Lower runs first. Omit to schedule this classifier last (treated as +Infinity). Duplicate names are rejected; duplicate dispatch_orders are allowed and schedule adjacent. |
|
|
23
23
|
| `applies_to` | no | One of `"user"`, `"assistant"`, `"both"`. Controls which pipeline pass the classifier participates in: `classify()` runs `"user"` + `"both"`; `inspect()` runs `"assistant"` + `"both"`. Defaults to `"user"`. |
|
|
@@ -34,12 +34,12 @@ Reserved fields are well-known output keys the aggregator knows how to consume.
|
|
|
34
34
|
|
|
35
35
|
| Reserved field | Shape | What the aggregator does with it |
|
|
36
36
|
|---|---|---|
|
|
37
|
-
| `final_reply` | `{ text: string ≤200 chars }` |
|
|
38
|
-
| `ack_reply` | `{ text: string ≤200 chars }` |
|
|
37
|
+
| `final_reply` | `{ text: string ≤200 chars }` | Sets `result.action = "reply"` and `result.reply`; caller returns it as the terminal reply |
|
|
38
|
+
| `ack_reply` | `{ text: string ≤200 chars }` | Sets `result.reply` (when action is `"route"`); caller shows it as an acknowledgement while downstream works |
|
|
39
39
|
| `model_tier` | one of `DOWNSTREAM_MODEL_TIER_VALUES` | Soft constraint for catalog resolver |
|
|
40
40
|
| `model_specialization` | one of `MODEL_SPECIALIZATION_VALUES` | Soft constraint for catalog resolver |
|
|
41
|
-
| `tools` | array of allowed tool ids | Sets `
|
|
42
|
-
| `risk_level` | one of `PROMPT_INJECTION_RISK_LEVEL_VALUES` | Surfaced in `
|
|
41
|
+
| `tools` | array of allowed tool ids | Sets `result.tools` |
|
|
42
|
+
| `risk_level` | one of `PROMPT_INJECTION_RISK_LEVEL_VALUES` | Surfaced in `result.prompt_injection`; `"high_risk"` or `"unknown"` triggers `action: "block"` |
|
|
43
43
|
|
|
44
44
|
`final_reply` and `ack_reply` are mutually exclusive — a single output may contain at most one.
|
|
45
45
|
|
|
@@ -49,7 +49,7 @@ When multiple classifiers emit the same reserved field, the highest-certainty co
|
|
|
49
49
|
|
|
50
50
|
```json
|
|
51
51
|
{
|
|
52
|
-
"name": "
|
|
52
|
+
"name": "model_tier",
|
|
53
53
|
"version": "1.0.0",
|
|
54
54
|
"purpose": "Recommend the downstream model tier.",
|
|
55
55
|
"dispatch_order": 20,
|