open-classify 0.2.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +134 -97
  2. package/dist/src/aggregator.d.ts +11 -4
  3. package/dist/src/aggregator.js +108 -121
  4. package/dist/src/classifiers/{custom/context_shift → context_shift}/manifest.json +6 -11
  5. package/dist/src/classifiers/{custom/context_shift → context_shift}/prompt.md +1 -1
  6. package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/manifest.json +7 -12
  7. package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/prompt.md +2 -2
  8. package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/manifest.json +6 -11
  9. package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/prompt.md +2 -2
  10. package/dist/src/classifiers/{stock/model_specialization → model_specialization}/manifest.json +2 -2
  11. package/dist/src/classifiers/model_specialization/prompt.md +5 -0
  12. package/dist/src/classifiers/preflight/manifest.json +34 -0
  13. package/dist/src/classifiers/preflight/prompt.md +10 -0
  14. package/dist/src/classifiers/{stock/prompt_injection → prompt_injection}/manifest.json +6 -2
  15. package/dist/src/classifiers/prompt_injection/prompt.md +14 -0
  16. package/dist/src/classifiers/{stock/routing → routing}/manifest.json +2 -2
  17. package/dist/src/classifiers/routing/prompt.md +5 -0
  18. package/dist/src/classifiers/{stock/tools → tools}/manifest.json +3 -3
  19. package/dist/src/classifiers/tools/prompt.md +5 -0
  20. package/dist/src/classifiers.js +31 -32
  21. package/dist/src/classify.d.ts +10 -2
  22. package/dist/src/classify.js +27 -12
  23. package/dist/src/config.d.ts +1 -4
  24. package/dist/src/config.js +7 -45
  25. package/dist/src/index.d.ts +1 -0
  26. package/dist/src/index.js +1 -0
  27. package/dist/src/input.d.ts +4 -1
  28. package/dist/src/input.js +12 -10
  29. package/dist/src/manifest.d.ts +18 -46
  30. package/dist/src/manifest.js +1 -5
  31. package/dist/src/pipeline.d.ts +11 -2
  32. package/dist/src/pipeline.js +98 -168
  33. package/dist/src/reserved-fields.d.ts +18 -0
  34. package/dist/src/reserved-fields.js +175 -0
  35. package/dist/src/stock-prompt.d.ts +9 -2
  36. package/dist/src/stock-prompt.js +165 -45
  37. package/dist/src/stock-validation.d.ts +16 -17
  38. package/dist/src/stock-validation.js +263 -236
  39. package/dist/src/stock.d.ts +26 -62
  40. package/dist/src/stock.js +7 -14
  41. package/docs/adding-a-classifier.md +74 -32
  42. package/docs/manifests.md +112 -71
  43. package/docs/resolver.md +25 -34
  44. package/docs/signals.md +39 -58
  45. package/open-classify.config.example.json +10 -13
  46. package/package.json +1 -3
  47. package/dist/src/classifiers/stock/preflight/manifest.json +0 -11
  48. package/dist/src/classifiers/stock/prompts/classifier-header.md +0 -4
  49. package/dist/src/classifiers/stock/prompts/custom-output.md +0 -7
  50. package/dist/src/classifiers/stock/prompts/model_specialization.md +0 -7
  51. package/dist/src/classifiers/stock/prompts/preflight-output.md +0 -10
  52. package/dist/src/classifiers/stock/prompts/preflight.md +0 -47
  53. package/dist/src/classifiers/stock/prompts/prompt-injection-output.md +0 -5
  54. package/dist/src/classifiers/stock/prompts/prompt_injection.md +0 -24
  55. package/dist/src/classifiers/stock/prompts/routing-output.md +0 -5
  56. package/dist/src/classifiers/stock/prompts/routing.md +0 -9
  57. package/dist/src/classifiers/stock/prompts/specialty.md +0 -12
  58. package/dist/src/classifiers/stock/prompts/tier.md +0 -7
  59. package/dist/src/classifiers/stock/prompts/tools-output.md +0 -11
  60. package/dist/src/classifiers/stock/prompts/tools.md +0 -10
  61. package/dist/src/ui-server.d.ts +0 -1
  62. package/dist/src/ui-server.js +0 -257
  63. /package/dist/src/classifiers/{stock/prompts → _prompts}/base.md +0 -0
  64. /package/dist/src/classifiers/{stock/prompts → _prompts}/confidence.md +0 -0
  65. /package/dist/src/classifiers/{stock/prompts → _prompts}/reason.md +0 -0
@@ -1,251 +1,226 @@
1
- import { DOWNSTREAM_MODEL_TIER_VALUES, MODEL_SPECIALIZATION_VALUES, } from "./enums.js";
1
+ // Manifest and classifier-output validation.
2
+ //
3
+ // The runtime composes every classifier's effective output schema by merging:
4
+ //
5
+ // - required `reason` / `certainty` metadata
6
+ // - canonical sub-schemas for each declared reserved field (optional)
7
+ // - the manifest's own `output_schema.properties` for custom fields
8
+ //
9
+ // That composed schema is what runs against actual classifier outputs, so the
10
+ // LLM cannot emit an invalid enum value for a reserved field even if the
11
+ // manifest author forgot to constrain it themselves.
2
12
  import { Ajv } from "ajv/dist/ajv.js";
3
- import { CERTAINTY_VALUES, STOCK_CLASSIFIER_NAMES } from "./stock.js";
4
- import { ensureNoDuplicates, isRecord, requireEnum, requireNonEmptyStringMaxLength, requireNonNegativeSafeInteger, requireString, requireStringArray, throwInvalid, } from "./validation.js";
5
- export const STOCK_REASON_MAX_CHARS = 120;
6
- export const STOCK_REPLY_MAX_CHARS = 200;
7
- export const STOCK_TOOL_ID_MAX_CHARS = 64;
8
- export const STOCK_TOOL_DESCRIPTION_MAX_CHARS = 240;
9
- export const STOCK_MANIFEST_NAME_MAX_CHARS = 80;
10
- export const STOCK_MANIFEST_VERSION_MAX_CHARS = 40;
11
- export const STOCK_MANIFEST_PURPOSE_MAX_CHARS = 400;
12
- const STOCK_PROMPT_INJECTION_RISK_LEVEL_VALUES = [
13
- "normal",
14
- "suspicious",
15
- "high_risk",
16
- "unknown",
17
- ];
18
- const MANIFEST_KIND_VALUES = ["stock", "custom"];
19
- const ajv = new Ajv({ allErrors: true, strict: false });
20
- const COMMON_MANIFEST_KEYS = [
21
- "kind",
13
+ import { APPLIES_TO_VALUES, CERTAINTY_VALUES } from "./stock.js";
14
+ import { RESERVED_FIELD_EXCLUSIONS, RESERVED_FIELD_NAMES, RESERVED_FIELDS, isReservedFieldName, normalizeToolId, } from "./reserved-fields.js";
15
+ import { isRecord, requireNonEmptyStringMaxLength, requireNonNegativeSafeInteger, requireString, throwInvalid, } from "./validation.js";
16
+ export const REASON_MAX_CHARS = 120;
17
+ export const TOOL_ID_MAX_CHARS = 64;
18
+ export const TOOL_DESCRIPTION_MAX_CHARS = 240;
19
+ export const MANIFEST_NAME_MAX_CHARS = 80;
20
+ export const MANIFEST_VERSION_MAX_CHARS = 40;
21
+ export const MANIFEST_PURPOSE_MAX_CHARS = 400;
22
+ const MANIFEST_KEYS = [
22
23
  "name",
23
24
  "version",
24
25
  "purpose",
25
- "order",
26
+ "dispatch_order",
27
+ "applies_to",
28
+ "reserved_fields",
29
+ "allowed_tools",
30
+ "output_schema",
26
31
  "fallback",
27
32
  "backend",
28
33
  ];
29
- const STOCK_MANIFEST_KEYS = [
30
- ...COMMON_MANIFEST_KEYS,
31
- "tools",
32
- ];
33
- const CUSTOM_MANIFEST_KEYS = [
34
- ...COMMON_MANIFEST_KEYS,
35
- "output_schema",
36
- ];
34
+ const ajv = new Ajv({ allErrors: true, strict: false });
37
35
  export function validateJsonClassifierManifest(value, model = "manifest") {
38
36
  if (!isRecord(value)) {
39
37
  throwInvalid("manifest", model, "manifest must be a JSON object");
40
38
  }
41
- const kind = requireEnum(value.kind, MANIFEST_KIND_VALUES, "manifest", model, "kind");
42
- return kind === "stock"
43
- ? validateStockManifest(value, model)
44
- : validateCustomManifest(value, model);
45
- }
46
- function validateStockManifest(value, model) {
47
- ensureAllowedObjectKeys(value, STOCK_MANIFEST_KEYS, "manifest", model, "manifest");
48
- const name = requireEnum(value.name, STOCK_CLASSIFIER_NAMES, "manifest", model, "name");
49
- const base = validateManifestCommon(value, model);
50
- const tools = value.tools === undefined
39
+ ensureAllowedObjectKeys(value, MANIFEST_KEYS, "manifest", model, "manifest");
40
+ const name = requireNonEmptyStringMaxLength(value.name, "manifest", model, "name", MANIFEST_NAME_MAX_CHARS);
41
+ const version = requireNonEmptyStringMaxLength(value.version, "manifest", model, "version", MANIFEST_VERSION_MAX_CHARS);
42
+ const purpose = requireNonEmptyStringMaxLength(value.purpose, "manifest", model, "purpose", MANIFEST_PURPOSE_MAX_CHARS);
43
+ const dispatchOrder = value.dispatch_order === undefined
51
44
  ? undefined
52
- : validateTools(value.tools, model);
53
- if (name !== "tools" && tools !== undefined) {
54
- throwInvalid("manifest", model, "tools is only supported on the tools classifier");
55
- }
56
- const fallback = validateStockOutputForName(name, value.fallback, model, tools);
57
- return {
58
- kind: "stock",
59
- name,
60
- ...base,
61
- fallback,
62
- ...(tools === undefined ? {} : { tools }),
63
- };
64
- }
65
- function validateCustomManifest(value, model) {
66
- ensureAllowedObjectKeys(value, CUSTOM_MANIFEST_KEYS, "manifest", model, "manifest");
67
- const name = requireNonEmptyStringMaxLength(value.name, "manifest", model, "name", STOCK_MANIFEST_NAME_MAX_CHARS);
68
- if (STOCK_CLASSIFIER_NAMES.includes(name)) {
69
- throwInvalid("manifest", model, `custom classifier name "${name}" collides with a stock classifier`);
45
+ : requireNonNegativeSafeInteger(value.dispatch_order, "manifest", model, "dispatch_order");
46
+ const appliesTo = validateAppliesTo(value.applies_to, model);
47
+ const reservedFields = validateReservedFields(value.reserved_fields, model);
48
+ const allowedTools = value.allowed_tools === undefined
49
+ ? undefined
50
+ : validateAllowedTools(value.allowed_tools, model);
51
+ if (allowedTools !== undefined && !reservedFields.includes("tools")) {
52
+ throwInvalid("manifest", model, "allowed_tools is only supported when reserved_fields includes \"tools\"");
70
53
  }
71
- const base = validateManifestCommon(value, model);
72
- if (value.output_schema === undefined) {
73
- throwInvalid("manifest", model, "output_schema is required for custom classifiers");
54
+ if (reservedFields.includes("tools") && allowedTools === undefined) {
55
+ throwInvalid("manifest", model, "allowed_tools is required when reserved_fields includes \"tools\"");
74
56
  }
75
- const outputSchema = value.output_schema;
76
- compileOutputSchema(outputSchema, "manifest", model);
77
- const fallback = validateCustomOutput(value.fallback, name, model, outputSchema);
78
- return {
79
- kind: "custom",
57
+ const outputSchema = validateOutputSchemaShape(value.output_schema, reservedFields, model);
58
+ const composedOutputSchema = composeOutputSchema(reservedFields, allowedTools, outputSchema);
59
+ compileSchema(composedOutputSchema, "manifest", model, "composed output_schema");
60
+ validateExamples(outputSchema, composedOutputSchema, model);
61
+ const fallback = validateFallback(value.fallback, composedOutputSchema, name, model);
62
+ const backend = value.backend === undefined ? undefined : validateBackend(value.backend, model);
63
+ const manifest = {
80
64
  name,
81
- ...base,
65
+ version,
66
+ purpose,
67
+ ...(dispatchOrder === undefined ? {} : { dispatch_order: dispatchOrder }),
68
+ ...(value.applies_to === undefined ? {} : { applies_to: appliesTo }),
69
+ ...(reservedFields.length === 0 ? {} : { reserved_fields: reservedFields }),
70
+ ...(allowedTools === undefined ? {} : { allowed_tools: allowedTools }),
71
+ ...(outputSchema === undefined ? {} : { output_schema: outputSchema }),
82
72
  fallback,
83
- output_schema: outputSchema,
73
+ ...(backend === undefined ? {} : { backend }),
84
74
  };
85
- }
86
- function validateManifestCommon(value, model) {
87
- const version = requireNonEmptyStringMaxLength(value.version, "manifest", model, "version", STOCK_MANIFEST_VERSION_MAX_CHARS);
88
- const purpose = requireNonEmptyStringMaxLength(value.purpose, "manifest", model, "purpose", STOCK_MANIFEST_PURPOSE_MAX_CHARS);
89
- const order = requireNonNegativeSafeInteger(value.order, "manifest", model, "order");
90
75
  return {
91
- version,
92
- purpose,
93
- order,
94
- ...(value.backend === undefined ? {} : { backend: validateBackend(value.backend, model) }),
76
+ manifest,
77
+ reservedFields,
78
+ composedOutputSchema,
79
+ appliesTo,
95
80
  };
96
81
  }
97
- export function validateOutputForManifest(manifest, value, context) {
98
- if (manifest.kind === "stock") {
99
- return validateStockOutputForName(manifest.name, value, context.model, manifest.tools);
82
+ function validateAppliesTo(raw, model) {
83
+ if (raw === undefined)
84
+ return "user";
85
+ if (typeof raw !== "string" || !APPLIES_TO_VALUES.includes(raw)) {
86
+ throwInvalid("manifest", model, `applies_to must be one of ${APPLIES_TO_VALUES.join(", ")}`);
100
87
  }
101
- return validateCustomOutput(value, context.classifier, context.model, manifest.output_schema);
88
+ return raw;
102
89
  }
103
- function validateStockOutputForName(name, value, model, tools) {
104
- if (!isRecord(value)) {
105
- throwInvalid(name, model, "output must be a JSON object");
90
+ function validateReservedFields(raw, model) {
91
+ if (raw === undefined)
92
+ return [];
93
+ if (!Array.isArray(raw)) {
94
+ throwInvalid("manifest", model, "reserved_fields must be an array of strings");
106
95
  }
107
- switch (name) {
108
- case "preflight":
109
- return validatePreflightOutput(value, model);
110
- case "routing":
111
- return validateTierRoutingOutput(value, model);
112
- case "model_specialization":
113
- return validateModelSpecializationOutput(value, model);
114
- case "tools":
115
- return validateToolsOutput(value, model, tools?.map((tool) => tool.id));
116
- case "prompt_injection":
117
- return validatePromptInjectionOutput(value, model);
118
- default: {
119
- const _exhaustive = name;
120
- void _exhaustive;
121
- throwInvalid("manifest", model, `unknown stock classifier name`);
96
+ const seen = new Set();
97
+ const result = [];
98
+ for (let i = 0; i < raw.length; i++) {
99
+ const item = raw[i];
100
+ if (typeof item !== "string") {
101
+ throwInvalid("manifest", model, `reserved_fields[${i}] must be a string`);
122
102
  }
103
+ if (!isReservedFieldName(item)) {
104
+ throwInvalid("manifest", model, `reserved_fields[${i}] "${item}" is not a known reserved field. Allowed: ${RESERVED_FIELD_NAMES.join(", ")}`);
105
+ }
106
+ if (seen.has(item)) {
107
+ throwInvalid("manifest", model, `reserved_fields[${i}] duplicates "${item}"`);
108
+ }
109
+ seen.add(item);
110
+ result.push(item);
123
111
  }
112
+ return result;
124
113
  }
125
- function validateMetadata(value, classifier, model) {
126
- if (value.reason === undefined) {
127
- throwInvalid(classifier, model, "reason is required");
128
- }
129
- if (value.certainty === undefined) {
130
- throwInvalid(classifier, model, "certainty is required");
114
+ function validateAllowedTools(value, model) {
115
+ if (!Array.isArray(value)) {
116
+ throwInvalid("manifest", model, "allowed_tools must be an array");
131
117
  }
132
- return {
133
- reason: truncateText(requireString(value.reason, classifier, model, "reason"), STOCK_REASON_MAX_CHARS),
134
- certainty: requireEnum(value.certainty, CERTAINTY_VALUES, classifier, model, "certainty"),
135
- };
136
- }
137
- function validatePreflightOutput(value, model) {
138
- ensureAllowedObjectKeys(value, ["reason", "certainty", "final_reply", "ack_reply"], "preflight", model, "output");
139
- if (value.final_reply !== undefined && value.ack_reply !== undefined) {
140
- throwInvalid("preflight", model, "final_reply and ack_reply are mutually exclusive");
118
+ const ids = new Set();
119
+ const out = [];
120
+ for (let i = 0; i < value.length; i++) {
121
+ const item = value[i];
122
+ if (!isRecord(item)) {
123
+ throwInvalid("manifest", model, `allowed_tools[${i}] must be an object`);
124
+ }
125
+ ensureAllowedObjectKeys(item, ["id", "description"], "manifest", model, `allowed_tools[${i}]`);
126
+ const id = requireNonEmptyStringMaxLength(item.id, "manifest", model, `allowed_tools[${i}].id`, TOOL_ID_MAX_CHARS);
127
+ const description = requireNonEmptyStringMaxLength(item.description, "manifest", model, `allowed_tools[${i}].description`, TOOL_DESCRIPTION_MAX_CHARS);
128
+ if (ids.has(id)) {
129
+ throwInvalid("manifest", model, `allowed_tools[${i}].id "${id}" is duplicated`);
130
+ }
131
+ ids.add(id);
132
+ out.push({ id, description });
141
133
  }
142
- const meta = validateMetadata(value, "preflight", model);
143
- return {
144
- ...meta,
145
- ...(value.final_reply === undefined
146
- ? {}
147
- : { final_reply: validateReplySignal(value.final_reply, "preflight", model, "final_reply") }),
148
- ...(value.ack_reply === undefined
149
- ? {}
150
- : { ack_reply: validateReplySignal(value.ack_reply, "preflight", model, "ack_reply") }),
151
- };
134
+ return out;
152
135
  }
153
- function validateReplySignal(value, classifier, model, field) {
154
- if (!isRecord(value)) {
155
- throwInvalid(classifier, model, `${field} must be an object`);
136
+ function validateOutputSchemaShape(raw, reservedFields, model) {
137
+ if (raw === undefined)
138
+ return undefined;
139
+ if (!isRecord(raw)) {
140
+ throwInvalid("manifest", model, "output_schema must be a JSON object");
156
141
  }
157
- ensureAllowedObjectKeys(value, ["reply"], classifier, model, field);
158
- const reply = requireString(value.reply, classifier, model, `${field}.reply`);
159
- if (reply.trim().length === 0) {
160
- throwInvalid(classifier, model, `${field}.reply must not be empty`);
142
+ // We don't fully validate JSON Schema shape — Ajv does that when we compile
143
+ // the composed schema. But we do enforce the no-reserved-keys rule, since
144
+ // mixing reserved fields into properties is always a manifest error.
145
+ const properties = raw.properties;
146
+ if (properties !== undefined) {
147
+ if (!isRecord(properties)) {
148
+ throwInvalid("manifest", model, "output_schema.properties must be a JSON object");
149
+ }
150
+ for (const key of Object.keys(properties)) {
151
+ if (isReservedFieldName(key)) {
152
+ throwInvalid("manifest", model, `output_schema.properties.${key} collides with a reserved field. Declare it in reserved_fields instead.`);
153
+ }
154
+ if (key === "reason" || key === "certainty") {
155
+ throwInvalid("manifest", model, `output_schema.properties.${key} is reserved. Do not declare it.`);
156
+ }
157
+ }
161
158
  }
162
- if (reply.length > STOCK_REPLY_MAX_CHARS) {
163
- throwInvalid(classifier, model, `${field}.reply must be ${STOCK_REPLY_MAX_CHARS} characters or fewer`);
159
+ // Examples must be an array if present. Per-example validation happens
160
+ // after the composed schema is built so we can validate against it.
161
+ if (raw.examples !== undefined && !Array.isArray(raw.examples)) {
162
+ throwInvalid("manifest", model, "output_schema.examples must be an array");
164
163
  }
165
- return { reply };
164
+ void reservedFields;
165
+ return raw;
166
166
  }
167
- function validateTierRoutingOutput(value, model) {
168
- ensureAllowedObjectKeys(value, ["reason", "certainty", "model_tier"], "routing", model, "output");
169
- const meta = validateMetadata(value, "routing", model);
170
- const modelTier = normalizeOptionalEnumValue(value.model_tier);
171
- return {
172
- ...meta,
173
- ...(modelTier === undefined
174
- ? {}
175
- : { model_tier: requireEnum(modelTier, DOWNSTREAM_MODEL_TIER_VALUES, "routing", model, "model_tier") }),
176
- };
177
- }
178
- function validateModelSpecializationOutput(value, model) {
179
- ensureAllowedObjectKeys(value, ["reason", "certainty", "specialization"], "model_specialization", model, "output");
180
- const meta = validateMetadata(value, "model_specialization", model);
181
- const specialization = normalizeOptionalEnumValue(value.specialization);
182
- return {
183
- ...meta,
184
- ...(specialization === undefined
185
- ? {}
186
- : { specialization: requireEnum(specialization, MODEL_SPECIALIZATION_VALUES, "model_specialization", model, "specialization") }),
167
+ function composeOutputSchema(reservedFields, allowedTools, outputSchema) {
168
+ const context = { allowed_tools: allowedTools };
169
+ const properties = {
170
+ reason: { type: "string", minLength: 1, maxLength: REASON_MAX_CHARS },
171
+ certainty: { type: "string", enum: [...CERTAINTY_VALUES] },
187
172
  };
188
- }
189
- function normalizeOptionalEnumValue(value) {
190
- if (value === undefined || value === null) {
191
- return undefined;
192
- }
193
- if (typeof value === "string" && value.trim().length === 0) {
194
- return undefined;
173
+ for (const field of reservedFields) {
174
+ properties[field] = RESERVED_FIELDS[field].subSchema(context);
195
175
  }
196
- return value;
197
- }
198
- function validateToolsOutput(value, model, configuredTools) {
199
- ensureAllowedObjectKeys(value, ["reason", "certainty", "tools"], "tools", model, "output");
200
- const meta = validateMetadata(value, "tools", model);
201
- const tools = requireStringArray(value.tools, "tools", model, "tools").map(normalizeTool);
202
- ensureNoDuplicates(tools, "tools", model, "tools");
203
- if (configuredTools) {
204
- const allowed = new Set(configuredTools);
205
- for (const tool of tools) {
206
- if (!allowed.has(tool)) {
207
- throwInvalid("tools", model, `tools includes unsupported tool ${tool}`);
208
- }
209
- }
176
+ const customProperties = isRecord(outputSchema?.properties)
177
+ ? outputSchema.properties
178
+ : {};
179
+ for (const [key, schema] of Object.entries(customProperties)) {
180
+ properties[key] = schema;
210
181
  }
211
- return { ...meta, tools };
212
- }
213
- function validatePromptInjectionOutput(value, model) {
214
- ensureAllowedObjectKeys(value, ["reason", "certainty", "risk_level"], "prompt_injection", model, "output");
215
- const meta = validateMetadata(value, "prompt_injection", model);
216
- const riskLevel = requireEnum(value.risk_level, STOCK_PROMPT_INJECTION_RISK_LEVEL_VALUES, "prompt_injection", model, "risk_level");
182
+ const customRequired = Array.isArray(outputSchema?.required)
183
+ ? outputSchema.required.filter((key) => typeof key === "string")
184
+ : [];
185
+ const required = Array.from(new Set(["reason", "certainty", ...customRequired]));
217
186
  return {
218
- ...meta,
219
- risk_level: riskLevel,
187
+ type: "object",
188
+ additionalProperties: false,
189
+ required,
190
+ properties,
220
191
  };
221
192
  }
222
- function validateCustomOutput(value, classifier, model, schema) {
223
- if (!isRecord(value)) {
224
- throwInvalid(classifier, model, "output must be a JSON object");
225
- }
226
- ensureAllowedObjectKeys(value, ["reason", "certainty", "output"], classifier, model, "output");
227
- if (value.output === undefined) {
228
- throwInvalid(classifier, model, "output is required for custom classifiers");
193
+ function validateExamples(outputSchema, composedSchema, model) {
194
+ if (outputSchema === undefined)
195
+ return;
196
+ const examples = outputSchema.examples;
197
+ if (!Array.isArray(examples))
198
+ return;
199
+ const validate = ajv.compile(composedSchema);
200
+ for (let i = 0; i < examples.length; i++) {
201
+ const example = examples[i];
202
+ if (!validate(example)) {
203
+ const message = formatSchemaErrors(validate.errors, `examples[${i}]`);
204
+ throwInvalid("manifest", model, `output_schema.examples[${i}] is invalid: ${message}`);
205
+ }
206
+ if (isRecord(example)) {
207
+ enforceMutualExclusions(example, "manifest", model, `output_schema.examples[${i}]`);
208
+ }
229
209
  }
230
- const meta = validateMetadata(value, classifier, model);
231
- validateWithSchema(value.output, schema, classifier, model, "output");
232
- return { ...meta, output: value.output };
233
210
  }
234
- function validateTools(value, model) {
235
- if (!Array.isArray(value)) {
236
- throwInvalid("manifest", model, "tools must be an array");
211
+ function validateFallback(raw, composedSchema, classifier, model) {
212
+ if (!isRecord(raw)) {
213
+ throwInvalid(classifier, model, "fallback must be a JSON object");
237
214
  }
238
- const out = value.map((item, index) => {
239
- if (!isRecord(item)) {
240
- throwInvalid("manifest", model, `tools[${index}] must be an object`);
241
- }
242
- return {
243
- id: requireNonEmptyStringMaxLength(item.id, "manifest", model, `tools[${index}].id`, STOCK_TOOL_ID_MAX_CHARS),
244
- description: requireNonEmptyStringMaxLength(item.description, "manifest", model, `tools[${index}].description`, STOCK_TOOL_DESCRIPTION_MAX_CHARS),
245
- };
246
- });
247
- ensureNoDuplicates(out.map((item) => item.id), "manifest", model, "tools[].id");
248
- return out;
215
+ // Fallback represents the "I have no signal" state, so reserved fields are
216
+ // optional. The composed schema already marks them optional except for
217
+ // reason/certainty.
218
+ const validate = ajv.compile(composedSchema);
219
+ if (!validate(raw)) {
220
+ const message = formatSchemaErrors(validate.errors, "fallback");
221
+ throwInvalid(classifier, model, `fallback is invalid: ${message}`);
222
+ }
223
+ return raw;
249
224
  }
250
225
  function validateBackend(value, model) {
251
226
  if (!isRecord(value))
@@ -265,6 +240,56 @@ function validateBackend(value, model) {
265
240
  },
266
241
  };
267
242
  }
243
+ export function validateOutputForManifest(manifest, value, context) {
244
+ if (!isRecord(value)) {
245
+ throwInvalid(context.classifier, context.model, "output must be a JSON object");
246
+ }
247
+ const normalized = normalizeOutput(value, manifest);
248
+ const validate = ajv.compile(manifest.composedOutputSchema);
249
+ if (!validate(normalized)) {
250
+ const message = formatSchemaErrors(validate.errors, "output");
251
+ throwInvalid(context.classifier, context.model, message);
252
+ }
253
+ enforceMutualExclusions(normalized, context.classifier, context.model, "output");
254
+ return truncateReason(normalized);
255
+ }
256
+ // Apply small, well-known cleanups that consistently improve LLM compliance:
257
+ // - blank / null routing-style enums → omitted entirely
258
+ // - tools aliases (browser → web, etc.)
259
+ function normalizeOutput(value, manifest) {
260
+ const out = { ...value };
261
+ for (const field of manifest.reservedFields) {
262
+ if (field === "model_tier" || field === "model_specialization") {
263
+ const raw = out[field];
264
+ if (raw === null || (typeof raw === "string" && raw.trim().length === 0)) {
265
+ delete out[field];
266
+ }
267
+ }
268
+ if (field === "tools") {
269
+ const raw = out[field];
270
+ if (Array.isArray(raw)) {
271
+ out[field] = raw.map((item) => typeof item === "string" ? normalizeToolId(item) : item);
272
+ }
273
+ }
274
+ }
275
+ return out;
276
+ }
277
+ function truncateReason(value) {
278
+ if (typeof value.reason !== "string")
279
+ return value;
280
+ if (value.reason.length <= REASON_MAX_CHARS)
281
+ return value;
282
+ return { ...value, reason: value.reason.slice(0, REASON_MAX_CHARS).trimEnd() };
283
+ }
284
+ function enforceMutualExclusions(output, classifier, model, path) {
285
+ for (const group of RESERVED_FIELD_EXCLUSIONS) {
286
+ const present = group.filter((field) => output[field] !== undefined);
287
+ if (present.length > 1) {
288
+ throwInvalid(classifier, model, `${path}: reserved fields are mutually exclusive: ${present.join(", ")}`);
289
+ }
290
+ }
291
+ }
292
+ // ─── Helpers ────────────────────────────────────────────────────────────────
268
293
  function ensureAllowedObjectKeys(value, allowedKeys, classifier, model, path) {
269
294
  const allowed = new Set(allowedKeys);
270
295
  for (const key of Object.keys(value)) {
@@ -273,40 +298,42 @@ function ensureAllowedObjectKeys(value, allowedKeys, classifier, model, path) {
273
298
  }
274
299
  }
275
300
  }
276
- function compileOutputSchema(schema, classifier, model) {
301
+ function compileSchema(schema, classifier, model, label) {
277
302
  try {
278
303
  ajv.compile(schema);
279
304
  }
280
305
  catch (error) {
281
- throwInvalid(classifier, model, `output_schema is invalid JSON Schema: ${errorMessage(error)}`);
282
- }
283
- }
284
- export function validateWithSchema(value, schema, classifier, model, path) {
285
- const validate = ajv.compile(schema);
286
- if (!validate(value)) {
287
- const message = ajv.errorsText(validate.errors, { dataVar: path });
288
- throwInvalid(classifier, model, message);
306
+ throwInvalid(classifier, model, `${label} is invalid JSON Schema: ${error instanceof Error ? error.message : String(error)}`);
289
307
  }
290
308
  }
291
- function normalizeTool(tool) {
292
- const aliases = {
293
- browser: "web",
294
- browsing: "web",
295
- internet: "web",
296
- web_browsing: "web",
297
- web_search: "web",
298
- };
299
- return aliases[tool] ?? tool;
300
- }
301
- function truncateText(text, maxChars) {
302
- return text.length <= maxChars ? text : text.slice(0, maxChars).trimEnd();
303
- }
304
- function errorMessage(error) {
305
- return error instanceof Error ? error.message : String(error);
306
- }
307
- export function validateClassifierOutputWithManifest(value, options) {
308
- return validateOutputForManifest(options.manifest, value, {
309
- classifier: options.classifier,
310
- model: options.model,
311
- });
309
+ // Format Ajv errors with the offending property name surfaced inline. Ajv's
310
+ // default errorsText says "must NOT have additional properties" without
311
+ // naming the property, which is unhelpful to debug.
312
+ function formatSchemaErrors(errors, dataVar) {
313
+ if (!errors || errors.length === 0)
314
+ return `${dataVar} is invalid`;
315
+ return errors
316
+ .map((err) => {
317
+ const path = `${dataVar}${err.instancePath ?? ""}`;
318
+ if (err.keyword === "additionalProperties") {
319
+ const extra = err.params?.additionalProperty;
320
+ if (extra) {
321
+ return `${path}.${extra} is not a supported field`;
322
+ }
323
+ }
324
+ if (err.keyword === "required") {
325
+ const missing = err.params?.missingProperty;
326
+ if (missing) {
327
+ return `${path}.${missing} is required`;
328
+ }
329
+ }
330
+ if (err.keyword === "enum") {
331
+ const allowed = err.params?.allowedValues;
332
+ if (allowed) {
333
+ return `${path} has an unsupported value (allowed: ${allowed.join(", ")})`;
334
+ }
335
+ }
336
+ return `${path} ${err.message ?? "is invalid"}`;
337
+ })
338
+ .join("; ");
312
339
  }