open-classify 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,11 +1,11 @@
1
1
  {
2
- "name": "routing",
2
+ "name": "model_tier",
3
3
  "version": "1.0.0",
4
4
  "purpose": "Recommend the downstream model tier.",
5
5
  "dispatch_order": 20,
6
6
  "reserved_fields": ["model_tier"],
7
7
  "fallback": {
8
- "reason": "Classifier failed; no routing signal.",
8
+ "reason": "Classifier failed; no model tier signal.",
9
9
  "certainty": "no_signal"
10
10
  }
11
11
  }
@@ -1,4 +1,4 @@
1
- You are the routing classifier for an AI assistant routing system.
1
+ You are the model_tier classifier for an AI assistant routing system.
2
2
 
3
3
  Pick the coarse model tier that best fits the target user message. Emit only `model_tier`; do not infer specialization, tools, or prompt-injection risk — other classifiers own those axes.
4
4
 
@@ -1,29 +1,30 @@
1
1
  {
2
2
  "name": "preflight",
3
- "version": "1.0.0",
4
- "purpose": "Determine whether the latest message can be answered immediately or should continue downstream.",
3
+ "version": "1.1.0",
4
+ "purpose": "Assess whether the latest message can be answered immediately (final_reply) or should route downstream with an acknowledgement (ack_reply). Always emits exactly one.",
5
5
  "dispatch_order": 10,
6
6
  "reserved_fields": ["final_reply", "ack_reply"],
7
7
  "output_schema": {
8
8
  "examples": [
9
9
  {
10
- "reason": "Greeting.",
10
+ "reason": "Simple greeting — answerable directly.",
11
11
  "certainty": "near_certain",
12
12
  "final_reply": { "text": "Hi!" }
13
13
  },
14
14
  {
15
- "reason": "Trivial arithmetic.",
15
+ "reason": "Trivial arithmetic — answerable directly.",
16
16
  "certainty": "very_strong",
17
17
  "final_reply": { "text": "4" }
18
18
  },
19
19
  {
20
- "reason": "Generated writing task.",
20
+ "reason": "Code review task requires substantive downstream work.",
21
21
  "certainty": "very_strong",
22
- "ack_reply": { "text": "On it." }
22
+ "ack_reply": { "text": "On it — reviewing the code now." }
23
23
  },
24
24
  {
25
- "reason": "Ambiguous; needs downstream model.",
26
- "certainty": "strong"
25
+ "reason": "Reminder request requires downstream action.",
26
+ "certainty": "strong",
27
+ "ack_reply": { "text": "Got it, I'll set that reminder for 3pm." }
27
28
  }
28
29
  ]
29
30
  },
@@ -1,10 +1,16 @@
1
1
  You are the preflight classifier for an AI assistant routing system.
2
2
 
3
- Decide whether the target user message can be answered immediately with a tiny terminal reply, or whether downstream work should continue (optionally with a brief acknowledgement).
3
+ Your primary task is to assess: **can you fully answer the target message yourself**, given the conversation history? Make this judgment first the reply text follows from it.
4
4
 
5
- - Emit `final_reply` only for tiny terminal answers like greetings, thanks, spelling lookups, and simple arithmetic. The reply text IS the complete answer to the user — nothing else happens after this.
6
- - Emit `ack_reply` when downstream work should continue and a brief acknowledgement would help (drafting, analysis, coding, research). The text must not contain the answer.
7
- - Omit both fields when the request is ambiguous or no acknowledgement is useful.
8
- - Do not address the user anywhere except inside `final_reply.text` or `ack_reply.text`.
5
+ **Step 1 assess whether you can fully answer:**
6
+ Ask yourself: Is the intent clear? Is the answer fully derivable from context right now, without real-time data, external tools, code execution, non-trivial generation, analysis, or judgment? Would a one-sentence reply genuinely resolve the request?
7
+
8
+ If yes emit `final_reply` with the complete answer.
9
+
10
+ If no (the downstream model should handle it) → emit `ack_reply` with a brief, contextually specific acknowledgement that shows you understood the request. The ack must reflect the actual request — not a generic "On it." — so the user knows their message was understood while the model works.
9
11
 
10
- If answering would require non-trivial generation, analysis, or judgment, do not use `final_reply`. Use `ack_reply` (or omit both) and let the downstream model produce the answer.
12
+ **Rule: always emit exactly one of `final_reply` or `ack_reply`. Never emit both. Never emit neither.**
13
+
14
+ - `final_reply` is for tiny terminal answers only: greetings, thanks, spelling lookups, simple arithmetic, yes/no factual questions answerable from context. If answering requires drafting, rewriting, analysis, coding, research, planning, or any substantive generation — use `ack_reply` instead.
15
+ - `ack_reply` text must not contain the answer. It acknowledges the request and confirms it is being worked on.
16
+ - Do not address the user anywhere except inside `final_reply.text` or `ack_reply.text`.
@@ -9,8 +9,7 @@
9
9
  "required": ["risk_level"]
10
10
  },
11
11
  "fallback": {
12
- "reason": "Classifier failed; prompt-injection risk is unknown.",
13
- "certainty": "no_signal",
14
- "risk_level": "unknown"
12
+ "reason": "Classifier failed; prompt-injection risk could not be assessed.",
13
+ "certainty": "no_signal"
15
14
  }
16
15
  }
@@ -1,6 +1,6 @@
1
1
  import { type RunClassifier } from "./classifiers.js";
2
2
  import { type OpenClassifyConfig } from "./config.js";
3
- import type { AggregatorConfig, Catalog, InspectResult, PipelineResult } from "./manifest.js";
3
+ import type { Catalog, InspectResult, PipelineResult } from "./manifest.js";
4
4
  import type { OpenClassifyInput } from "./types.js";
5
5
  export type Classifier = (input: OpenClassifyInput, options?: {
6
6
  signal?: AbortSignal;
@@ -25,6 +25,5 @@ export interface CreateClassifierOptions {
25
25
  classifierTimeoutMs?: number;
26
26
  classifierRetryCount?: number;
27
27
  maxConcurrency?: number;
28
- aggregator?: AggregatorConfig;
29
28
  }
30
29
  export declare function createClassifier(options?: CreateClassifierOptions): OpenClassify;
@@ -28,7 +28,6 @@ export function createClassifier(options = {}) {
28
28
  });
29
29
  const catalog = options.catalog ??
30
30
  loadCatalog(options.catalogPath ?? fileConfig?.catalog ?? OLLAMA_DEFAULT_CATALOG_PATH);
31
- const aggregator = options.aggregator ?? fileConfig?.aggregator;
32
31
  let resourceCheck;
33
32
  const ensureResources = async () => {
34
33
  if (!needsResourceCheck)
@@ -47,7 +46,6 @@ export function createClassifier(options = {}) {
47
46
  classifierTimeoutMs: options.classifierTimeoutMs,
48
47
  classifierRetryCount: options.classifierRetryCount,
49
48
  maxConcurrency: options.maxConcurrency,
50
- aggregator,
51
49
  signal: callOptions?.signal,
52
50
  });
53
51
  };
@@ -1,10 +1,8 @@
1
1
  import { type ClassifierName } from "./classifiers.js";
2
- import { type AggregatorConfig } from "./manifest.js";
3
2
  export declare const DEFAULT_OPEN_CLASSIFY_CONFIG_PATH = "open-classify.config.json";
4
3
  export interface OpenClassifyConfig {
5
4
  readonly runner?: OllamaRunnerConfig;
6
5
  readonly catalog?: string;
7
- readonly aggregator?: AggregatorConfig;
8
6
  }
9
7
  export interface OllamaRunnerConfig {
10
8
  readonly provider: "ollama";
@@ -30,25 +30,10 @@ export function validateOpenClassifyConfig(value, path = "open-classify config")
30
30
  if (!isRecord(value)) {
31
31
  throwConfig(path, "config must be a JSON object");
32
32
  }
33
- ensureAllowedKeys(value, ["runner", "catalog", "aggregator"], path, "<root>");
33
+ ensureAllowedKeys(value, ["runner", "catalog"], path, "<root>");
34
34
  return {
35
35
  ...(value.runner === undefined ? {} : { runner: validateRunner(value.runner, path) }),
36
36
  ...(value.catalog === undefined ? {} : { catalog: requireString(value.catalog, path, "catalog") }),
37
- ...(value.aggregator === undefined ? {} : { aggregator: validateAggregator(value.aggregator, path) }),
38
- };
39
- }
40
- function validateAggregator(value, path) {
41
- if (!isRecord(value)) {
42
- throwConfig(path, "aggregator must be an object");
43
- }
44
- ensureAllowedKeys(value, ["certaintyThreshold", "confidenceThreshold"], path, "aggregator");
45
- return {
46
- ...(value.certaintyThreshold === undefined
47
- ? {}
48
- : { certaintyThreshold: requireUnitFloat(value.certaintyThreshold, path, "aggregator.certaintyThreshold") }),
49
- ...(value.confidenceThreshold === undefined
50
- ? {}
51
- : { confidenceThreshold: requireUnitFloat(value.confidenceThreshold, path, "aggregator.confidenceThreshold") }),
52
37
  };
53
38
  }
54
39
  function validateRunner(value, path) {
@@ -118,13 +103,6 @@ function requireNumber(value, path, field) {
118
103
  }
119
104
  return value;
120
105
  }
121
- function requireUnitFloat(value, path, field) {
122
- const number = requireNumber(value, path, field);
123
- if (number < 0 || number > 1) {
124
- throwConfig(path, `${field} must be a finite number between 0 and 1 inclusive`);
125
- }
126
- return number;
127
- }
128
106
  function ensureAllowedKeys(value, allowedKeys, path, field) {
129
107
  const allowed = new Set(allowedKeys);
130
108
  for (const key of Object.keys(value)) {
package/dist/src/index.js CHANGED
@@ -1,8 +1,7 @@
1
1
  // Public barrel for the Open Classify package. Everything an external caller
2
2
  // would need — input types, enums, the registry, the pipeline, the Ollama
3
- // runner, the catalog loader, the aggregator's certainty threshold is
4
- // re-exported here. The build emits a single `index.js` that downstream
5
- // consumers can import from `open-classify`.
3
+ // runner, the catalog loader is re-exported here. The build emits a single
4
+ // `index.js` that downstream consumers can import from `open-classify`.
6
5
  export * from "./aggregator.js";
7
6
  export * from "./catalog.js";
8
7
  export * from "./classifiers.js";
@@ -1,9 +1,9 @@
1
- import type { AckReplySignal, ClassifierAuditOutput, ClassifierOutput, FinalReplySignal, PromptInjectionSignal, RoutingSignal, RuntimeClassifierManifest, ToolsSignal } from "./stock.js";
2
- import type { ClassifierInput, ClassifierRunStatus } from "./types.js";
3
- import type { DownstreamModelTier, ModelSpecialization } from "./enums.js";
1
+ import type { RuntimeClassifierManifest } from "./stock.js";
2
+ import type { ClassifierInput } from "./types.js";
3
+ import type { DownstreamModelTier, ModelSpecialization, PromptInjectionRiskLevel } from "./enums.js";
4
4
  export type ClassifierName = string;
5
- export type ClassifierResults = Record<ClassifierName, ClassifierOutput>;
6
- export type RunClassifier = (name: ClassifierName, input: ClassifierInput, signal: AbortSignal) => Promise<ClassifierOutput>;
5
+ export type ClassifierResults = Record<ClassifierName, import("./stock.js").ClassifierOutput>;
6
+ export type RunClassifier = (name: ClassifierName, input: ClassifierInput, signal: AbortSignal) => Promise<import("./stock.js").ClassifierOutput>;
7
7
  export interface CatalogEntry {
8
8
  readonly id: string;
9
9
  readonly specializations: ReadonlyArray<ModelSpecialization>;
@@ -18,78 +18,33 @@ export interface Catalog {
18
18
  readonly models: ReadonlyArray<CatalogEntry>;
19
19
  readonly default: string;
20
20
  }
21
- export interface ModelRecommendationResolution {
22
- readonly constraints_used: Partial<{
23
- model_specialization: ModelSpecialization;
24
- model_tier: DownstreamModelTier;
25
- }>;
26
- readonly constraints_dropped: ReadonlyArray<{
27
- readonly axis: "model_specialization" | "model_tier";
28
- readonly reason: "low_confidence" | "no_match_relaxed" | "default_fallback";
29
- }>;
30
- readonly confidences: Partial<{
31
- routing: number;
32
- }>;
33
- readonly fell_back_to_default: boolean;
34
- }
35
- export interface ModelRecommendation {
36
- readonly id: string;
37
- readonly params_in_billions: number | null;
38
- readonly context_window: number;
39
- readonly input_tokens_cpm?: number;
40
- readonly cached_tokens_cpm?: number;
41
- readonly output_tokens_cpm?: number;
42
- readonly resolution: ModelRecommendationResolution;
43
- }
44
- export interface Envelope {
45
- readonly final_reply?: FinalReplySignal;
46
- readonly ack_reply?: AckReplySignal;
47
- readonly routing?: RoutingSignal;
48
- readonly tools?: ToolsSignal;
49
- readonly prompt_injection?: PromptInjectionSignal;
50
- readonly classifier_outputs: ReadonlyArray<ClassifierAuditOutput>;
51
- readonly model_recommendation: ModelRecommendation;
52
- }
53
21
  export type ClassifierPublicOutputs = Record<string, Record<string, unknown>>;
54
- export interface DownstreamTargetMessage {
55
- readonly role: "user";
22
+ export type PipelineAction = "route" | "block" | "reply";
23
+ export type BlockReason = "prompt_injection" | "classification_error";
24
+ export interface ReplySignal {
56
25
  readonly text: string;
57
- readonly hash: string;
58
26
  }
59
- export interface DownstreamPayload {
60
- readonly model_id: string;
61
- readonly target_message: DownstreamTargetMessage;
62
- readonly tools: ToolsSignal;
63
- }
64
- export type ClassifierEntry = ClassifierOutput & {
65
- readonly status: ClassifierRunStatus;
66
- readonly version: string;
67
- };
68
- export interface CertaintySummary {
69
- readonly min: number;
70
- readonly avg: number;
71
- }
72
- export interface PipelineMeta {
73
- readonly classifiers: Record<string, ClassifierEntry>;
74
- readonly certainty: CertaintySummary;
75
- }
76
- export interface PipelineAudit extends Envelope {
77
- readonly meta: PipelineMeta;
78
- }
79
- export interface InspectResult {
27
+ export interface PipelineResult {
28
+ readonly action: PipelineAction;
29
+ readonly block_reason?: BlockReason;
80
30
  readonly target_message_hash: string;
31
+ readonly model_id: string | null;
32
+ readonly tools: ReadonlyArray<string>;
33
+ readonly reply: ReplySignal | null;
34
+ readonly prompt_injection: {
35
+ readonly risk_level: PromptInjectionRiskLevel;
36
+ } | null;
37
+ readonly avg_certainty: number;
38
+ readonly min_certainty: number;
39
+ readonly failed_classifiers: ReadonlyArray<string>;
81
40
  readonly classifier_outputs: ClassifierPublicOutputs;
82
41
  }
83
- export interface PipelineResult {
84
- readonly action: "route";
42
+ export interface InspectResult {
85
43
  readonly target_message_hash: string;
86
- readonly downstream: DownstreamPayload;
44
+ readonly message: {
45
+ readonly role: "assistant";
46
+ readonly text: string;
47
+ };
87
48
  readonly classifier_outputs: ClassifierPublicOutputs;
88
- readonly audit: PipelineAudit;
89
- }
90
- export interface AggregatorConfig {
91
- readonly certaintyThreshold?: number;
92
- /** @deprecated Use certaintyThreshold. */
93
- readonly confidenceThreshold?: number;
94
49
  }
95
50
  export type ClassifierRegistry = ReadonlyArray<RuntimeClassifierManifest>;
@@ -1,5 +1,5 @@
1
1
  import { type RunClassifier } from "./classifiers.js";
2
- import type { AggregatorConfig, Catalog, InspectResult, PipelineResult } from "./manifest.js";
2
+ import type { Catalog, InspectResult, PipelineResult } from "./manifest.js";
3
3
  import type { OpenClassifyInput } from "./types.js";
4
4
  export declare const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15000;
5
5
  export declare const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
@@ -13,7 +13,6 @@ export interface ClassifyOptions {
13
13
  classifierTimeoutMs?: number;
14
14
  classifierRetryCount?: number;
15
15
  maxConcurrency?: number;
16
- aggregator?: AggregatorConfig;
17
16
  signal?: AbortSignal;
18
17
  }
19
18
  export interface InspectOptions {
@@ -1,7 +1,6 @@
1
- import { composeEnvelope } from "./aggregator.js";
1
+ import { assembleResult, buildPublicOutputs } from "./aggregator.js";
2
2
  import { MODULES_BY_NAME, REGISTRY, } from "./classifiers.js";
3
3
  import { normalizeOpenClassifyInput, toClassifierInput } from "./input.js";
4
- import { certaintyScore } from "./stock.js";
5
4
  export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15_000;
6
5
  export const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
7
6
  export const DEFAULT_MAX_CONCURRENCY = 7;
@@ -12,22 +11,27 @@ export class OpenClassifyNormalizationError extends Error {
12
11
  }
13
12
  }
14
13
  export async function classifyOpenClassifyInput(input, options) {
15
- const { request, results, meta } = await runPipeline(input, "user", options);
16
- const classifierInput = toClassifierInput(request);
17
- const envelope = composeEnvelope({
18
- registry: filteredRegistry("user"),
14
+ const { request, results, failedClassifiers } = await runPipeline(input, "user", options);
15
+ const reg = filteredRegistry("user");
16
+ const assembled = assembleResult({
17
+ registry: reg,
19
18
  results,
19
+ failedClassifiers,
20
20
  catalog: options.catalog,
21
- input: classifierInput,
22
- config: options.aggregator,
23
21
  });
24
- return buildRouteResult(request, envelope, results, meta);
22
+ return {
23
+ ...assembled,
24
+ target_message_hash: request.target_message_hash,
25
+ };
25
26
  }
26
27
  export async function inspectOpenClassifyInput(input, options) {
27
28
  const { request, results } = await runPipeline(input, "assistant", options);
29
+ const reg = filteredRegistry("assistant");
30
+ const lastMsg = request.messages[request.messages.length - 1];
28
31
  return {
29
32
  target_message_hash: request.target_message_hash,
30
- classifier_outputs: classifierPublicOutputs(filteredRegistry("assistant"), results),
33
+ message: { role: "assistant", text: lastMsg.text },
34
+ classifier_outputs: buildPublicOutputs(reg, results),
31
35
  };
32
36
  }
33
37
  async function runPipeline(input, role, options) {
@@ -52,15 +56,12 @@ async function runPipeline(input, role, options) {
52
56
  const classifierTimeoutMs = options.classifierTimeoutMs ?? DEFAULT_CLASSIFIER_TIMEOUT_MS;
53
57
  const classifierRetryCount = options.classifierRetryCount ?? DEFAULT_CLASSIFIER_RETRY_COUNT;
54
58
  const maxConcurrency = resolveMaxConcurrency(options.maxConcurrency);
55
- // REGISTRY is already sorted by `dispatch_order` ascending. Filter by
56
- // applies_to so we only dispatch classifiers relevant to this role; the
57
- // worker pool then runs them in the remaining order.
58
59
  const registry = filteredRegistry(role);
59
60
  const queue = registry.map((m) => m.name);
60
61
  try {
61
62
  const settled = await runWithConcurrency(queue, maxConcurrency, controller.signal, (name) => runClassifierWithRetry(name, classifierInput, options.runClassifier, controller.signal, classifierTimeoutMs, classifierRetryCount));
62
- const { results, meta } = collectFullEntries(settled, registry);
63
- return { request, results, meta };
63
+ const { results, failedClassifiers } = collectResults(settled);
64
+ return { request, results, failedClassifiers };
64
65
  }
65
66
  finally {
66
67
  options.signal?.removeEventListener("abort", abortFromOptions);
@@ -91,9 +92,6 @@ async function runWithConcurrency(names, maxConcurrency, signal, start) {
91
92
  return;
92
93
  const name = names[i];
93
94
  if (signal.aborted) {
94
- // Queued classifiers that never started are reported as not-run so
95
- // the audit shows their fallback in `meta.classifiers`. In-flight
96
- // classifiers receive the abort signal directly and resolve normally.
97
95
  results[i] = {
98
96
  ok: false,
99
97
  name,
@@ -109,71 +107,16 @@ async function runWithConcurrency(names, maxConcurrency, signal, start) {
109
107
  await Promise.all(Array.from({ length: workerCount }, () => worker()));
110
108
  return results;
111
109
  }
112
- function collectFullEntries(settled, registry) {
110
+ function collectResults(settled) {
113
111
  const results = {};
114
- const classifiers = {};
112
+ const failedClassifiers = [];
115
113
  for (const s of settled) {
116
114
  const manifest = MODULES_BY_NAME[s.name];
117
- const value = s.ok ? s.value : manifest.fallback;
118
- results[s.name] = value;
119
- classifiers[s.name] = {
120
- ...value,
121
- status: classifierRunStatus(s),
122
- version: manifest.version,
123
- };
115
+ results[s.name] = s.ok ? s.value : manifest.fallback;
116
+ if (!s.ok)
117
+ failedClassifiers.push(s.name);
124
118
  }
125
- return { results, meta: { classifiers, certainty: certaintySummary(results, registry) } };
126
- }
127
- function certaintySummary(results, registry) {
128
- const scores = registry.map((m) => scoreCertainty(results[m.name]?.certainty));
129
- if (scores.length === 0)
130
- return { min: 0, avg: 0 };
131
- const min = Math.min(...scores);
132
- const avg = scores.reduce((sum, v) => sum + v, 0) / scores.length;
133
- return { min, avg };
134
- }
135
- function scoreCertainty(certainty) {
136
- return certainty === undefined ? 0 : certaintyScore[certainty];
137
- }
138
- function buildRouteResult(request, envelope, results, meta) {
139
- const downstream = {
140
- model_id: envelope.model_recommendation.id,
141
- target_message: {
142
- role: "user",
143
- text: request.text,
144
- hash: request.target_message_hash,
145
- },
146
- tools: envelope.tools ?? { tools: [] },
147
- };
148
- return {
149
- action: "route",
150
- target_message_hash: request.target_message_hash,
151
- downstream,
152
- classifier_outputs: classifierPublicOutputs(filteredRegistry("user"), results),
153
- audit: {
154
- ...envelope,
155
- meta,
156
- },
157
- };
158
- }
159
- // Expose each classifier's payload — every output field except `reason` and
160
- // `certainty`. Iterates the supplied registry so we only surface classifiers
161
- // that actually ran for this role.
162
- function classifierPublicOutputs(registry, results) {
163
- const out = {};
164
- for (const manifest of registry) {
165
- const result = results[manifest.name];
166
- if (result === undefined)
167
- continue;
168
- out[manifest.name] = stripMetadata(result);
169
- }
170
- return out;
171
- }
172
- function stripMetadata(output) {
173
- const { reason, certainty, ...payload } = output;
174
- void reason;
175
- void certainty;
176
- return payload;
119
+ return { results, failedClassifiers };
177
120
  }
178
121
  async function runClassifierWithRetry(name, input, runClassifier, rootSignal, timeoutMs, retryCount) {
179
122
  let lastError = new Error(`${name} classifier did not run`);
@@ -219,16 +162,6 @@ async function runClassifierAttempt(name, input, runClassifier, rootSignal, time
219
162
  rootSignal.removeEventListener("abort", abortAttempt);
220
163
  }
221
164
  }
222
- function classifierRunStatus(settled) {
223
- if (settled.ok)
224
- return { ok: true, source: "model" };
225
- return {
226
- ok: false,
227
- source: "fallback",
228
- reason: settled.reason,
229
- error: errorMessage(settled.error),
230
- };
231
- }
232
165
  function errorMessage(error) {
233
166
  return error instanceof Error ? error.message : String(error);
234
167
  }
@@ -212,10 +212,14 @@ function validateFallback(raw, composedSchema, classifier, model) {
212
212
  if (!isRecord(raw)) {
213
213
  throwInvalid(classifier, model, "fallback must be a JSON object");
214
214
  }
215
- // Fallback represents the "I have no signal" state, so reserved fields are
216
- // optional. The composed schema already marks them optional except for
217
- // reason/certainty.
218
- const validate = ajv.compile(composedSchema);
215
+ // Fallback is the "no signal" state: only reason and certainty are required.
216
+ // Strip any custom `required` entries beyond those two so that reserved fields
217
+ // and output_schema.required fields don't force the fallback to emit values
218
+ // it cannot meaningfully provide when the classifier has failed.
219
+ const fallbackSchema = isRecord(composedSchema)
220
+ ? { ...composedSchema, required: ["reason", "certainty"] }
221
+ : composedSchema;
222
+ const validate = ajv.compile(fallbackSchema);
219
223
  if (!validate(raw)) {
220
224
  const message = formatSchemaErrors(validate.errors, "fallback");
221
225
  throwInvalid(classifier, model, `fallback is invalid: ${message}`);
@@ -71,7 +71,7 @@ Rules:
71
71
  - `name` must match the directory name.
72
72
  - Reserved field names cannot appear in `output_schema.properties`; declare them in `reserved_fields` instead.
73
73
  - `reason` and `certainty` are added to the composed schema by the runtime — don't declare them.
74
- - `fallback` must validate against the composed schema. Reserved fields are optional in fallback (a "no signal" fallback usually omits them).
74
+ - `fallback` must validate against the composed schema. Only `reason` and `certainty` are required in fallback; reserved fields and `output_schema.required` fields are exempt (a "no signal" fallback usually omits them).
75
75
  - `output_schema.examples` (JSON Schema standard) must validate against the composed schema at load time, so a broken example fails the build, not the model call.
76
76
 
77
77
  See [manifests.md](manifests.md) for the full field list.
@@ -107,7 +107,7 @@ const result = await classify(input);
107
107
  const tags = result.classifier_outputs.topic_tags?.tags ?? [];
108
108
  ```
109
109
 
110
- `result.audit.classifier_outputs[]` carries the same data with `reason` and `certainty` attached if you need to inspect them.
110
+ `classifier_outputs[name]` includes all payload fields plus `reason` (string) and `certainty` (float).
111
111
 
112
112
  ## Targeting the assistant response
113
113
 
@@ -117,7 +117,7 @@ Classifiers run against the user message by default. To run a classifier against
117
117
  - `"assistant"` — only `inspect()` runs it.
118
118
  - `"both"` — both passes run it.
119
119
 
120
- Use `inspect()` from `createClassifier()` for the assistant-side pass. It returns a lean shape (`target_message_hash` + `classifier_outputs`) no routing, no audit envelope. The built-in `prompt_injection` ships tagged `"both"` so it runs on both sides.
120
+ Use `inspect()` from `createClassifier()` for the assistant-side pass. It returns a lean shape: `target_message_hash`, the `message` that was inspected, and `classifier_outputs`. No routing, no action, no block logic.
121
121
 
122
122
  ```ts
123
123
  const { inspect } = createClassifier({ catalog });
@@ -130,6 +130,8 @@ const post = await inspect({
130
130
  const risk = post.classifier_outputs.prompt_injection?.risk_level;
131
131
  ```
132
132
 
133
+ The built-in `prompt_injection` ships tagged `"both"` so it runs on both sides.
134
+
133
135
  ## Choosing the classifier model
134
136
 
135
137
  For apps and OSS installs, prefer `open-classify.config.json`:
package/docs/manifests.md CHANGED
@@ -17,7 +17,7 @@ The loader skips any top-level directory whose name starts with `_` (those are s
17
17
  | Field | Required | Description |
18
18
  |---|---|---|
19
19
  | `name` | yes | Classifier id. Must match the directory name. |
20
- | `version` | yes | Contract version surfaced in `meta.classifiers[name].version`. |
20
+ | `version` | yes | Contract version string for this classifier. |
21
21
  | `purpose` | yes | Human-readable description of the classifier's job. Treated as a hard scope boundary in the prompt. |
22
22
  | `dispatch_order` | no | Non-negative integer scheduling priority. Lower runs first. Omit to schedule this classifier last (treated as +Infinity). Duplicate names are rejected; duplicate dispatch_orders are allowed and schedule adjacent. |
23
23
  | `applies_to` | no | One of `"user"`, `"assistant"`, `"both"`. Controls which pipeline pass the classifier participates in: `classify()` runs `"user"` + `"both"`; `inspect()` runs `"assistant"` + `"both"`. Defaults to `"user"`. |
@@ -34,12 +34,12 @@ Reserved fields are well-known output keys the aggregator knows how to consume.
34
34
 
35
35
  | Reserved field | Shape | What the aggregator does with it |
36
36
  |---|---|---|
37
- | `final_reply` | `{ text: string ≤200 chars }` | Surfaced in `audit.final_reply`; caller can return as the terminal reply |
38
- | `ack_reply` | `{ text: string ≤200 chars }` | Surfaced in `audit.ack_reply`; caller can show as an acknowledgement |
37
+ | `final_reply` | `{ text: string ≤200 chars }` | Sets `result.action = "reply"` and `result.reply`; caller returns it as the terminal reply |
38
+ | `ack_reply` | `{ text: string ≤200 chars }` | Sets `result.reply` (when action is `"route"`); caller shows it as an acknowledgement while downstream works |
39
39
  | `model_tier` | one of `DOWNSTREAM_MODEL_TIER_VALUES` | Soft constraint for catalog resolver |
40
40
  | `model_specialization` | one of `MODEL_SPECIALIZATION_VALUES` | Soft constraint for catalog resolver |
41
- | `tools` | array of allowed tool ids | Sets `downstream.tools` |
42
- | `risk_level` | one of `PROMPT_INJECTION_RISK_LEVEL_VALUES` | Surfaced in `audit.prompt_injection` |
41
+ | `tools` | array of allowed tool ids | Sets `result.tools` |
42
+ | `risk_level` | one of `PROMPT_INJECTION_RISK_LEVEL_VALUES` | Surfaced in `result.prompt_injection`; `"high_risk"` or `"unknown"` triggers `action: "block"` |
43
43
 
44
44
  `final_reply` and `ack_reply` are mutually exclusive — a single output may contain at most one.
45
45
 
@@ -49,7 +49,7 @@ When multiple classifiers emit the same reserved field, the highest-certainty co
49
49
 
50
50
  ```json
51
51
  {
52
- "name": "routing",
52
+ "name": "model_tier",
53
53
  "version": "1.0.0",
54
54
  "purpose": "Recommend the downstream model tier.",
55
55
  "dispatch_order": 20,