open-classify 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -97
- package/dist/src/aggregator.d.ts +11 -4
- package/dist/src/aggregator.js +108 -121
- package/dist/src/classifiers/{custom/context_shift → context_shift}/manifest.json +6 -11
- package/dist/src/classifiers/{custom/context_shift → context_shift}/prompt.md +1 -1
- package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/manifest.json +7 -12
- package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/prompt.md +2 -2
- package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/manifest.json +6 -11
- package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/prompt.md +2 -2
- package/dist/src/classifiers/{stock/model_specialization → model_specialization}/manifest.json +2 -2
- package/dist/src/classifiers/model_specialization/prompt.md +5 -0
- package/dist/src/classifiers/preflight/manifest.json +34 -0
- package/dist/src/classifiers/preflight/prompt.md +10 -0
- package/dist/src/classifiers/{stock/prompt_injection → prompt_injection}/manifest.json +6 -2
- package/dist/src/classifiers/prompt_injection/prompt.md +14 -0
- package/dist/src/classifiers/{stock/routing → routing}/manifest.json +2 -2
- package/dist/src/classifiers/routing/prompt.md +5 -0
- package/dist/src/classifiers/{stock/tools → tools}/manifest.json +3 -3
- package/dist/src/classifiers/tools/prompt.md +5 -0
- package/dist/src/classifiers.js +31 -32
- package/dist/src/classify.d.ts +10 -2
- package/dist/src/classify.js +27 -12
- package/dist/src/config.d.ts +1 -4
- package/dist/src/config.js +7 -45
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/input.d.ts +4 -1
- package/dist/src/input.js +12 -10
- package/dist/src/manifest.d.ts +18 -46
- package/dist/src/manifest.js +1 -5
- package/dist/src/pipeline.d.ts +11 -2
- package/dist/src/pipeline.js +98 -168
- package/dist/src/reserved-fields.d.ts +18 -0
- package/dist/src/reserved-fields.js +175 -0
- package/dist/src/stock-prompt.d.ts +9 -2
- package/dist/src/stock-prompt.js +165 -45
- package/dist/src/stock-validation.d.ts +16 -17
- package/dist/src/stock-validation.js +263 -236
- package/dist/src/stock.d.ts +26 -62
- package/dist/src/stock.js +7 -14
- package/docs/adding-a-classifier.md +74 -32
- package/docs/manifests.md +112 -71
- package/docs/resolver.md +25 -34
- package/docs/signals.md +39 -58
- package/open-classify.config.example.json +10 -13
- package/package.json +1 -3
- package/dist/src/classifiers/stock/preflight/manifest.json +0 -11
- package/dist/src/classifiers/stock/prompts/classifier-header.md +0 -4
- package/dist/src/classifiers/stock/prompts/custom-output.md +0 -7
- package/dist/src/classifiers/stock/prompts/model_specialization.md +0 -7
- package/dist/src/classifiers/stock/prompts/preflight-output.md +0 -10
- package/dist/src/classifiers/stock/prompts/preflight.md +0 -47
- package/dist/src/classifiers/stock/prompts/prompt-injection-output.md +0 -5
- package/dist/src/classifiers/stock/prompts/prompt_injection.md +0 -24
- package/dist/src/classifiers/stock/prompts/routing-output.md +0 -5
- package/dist/src/classifiers/stock/prompts/routing.md +0 -9
- package/dist/src/classifiers/stock/prompts/specialty.md +0 -12
- package/dist/src/classifiers/stock/prompts/tier.md +0 -7
- package/dist/src/classifiers/stock/prompts/tools-output.md +0 -11
- package/dist/src/classifiers/stock/prompts/tools.md +0 -10
- package/dist/src/ui-server.d.ts +0 -1
- package/dist/src/ui-server.js +0 -257
- /package/dist/src/classifiers/{stock/prompts → _prompts}/base.md +0 -0
- /package/dist/src/classifiers/{stock/prompts → _prompts}/confidence.md +0 -0
- /package/dist/src/classifiers/{stock/prompts → _prompts}/reason.md +0 -0
|
@@ -1,251 +1,226 @@
|
|
|
1
|
-
|
|
1
|
+
// Manifest and classifier-output validation.
|
|
2
|
+
//
|
|
3
|
+
// The runtime composes every classifier's effective output schema by merging:
|
|
4
|
+
//
|
|
5
|
+
// - required `reason` / `certainty` metadata
|
|
6
|
+
// - canonical sub-schemas for each declared reserved field (optional)
|
|
7
|
+
// - the manifest's own `output_schema.properties` for custom fields
|
|
8
|
+
//
|
|
9
|
+
// That composed schema is what runs against actual classifier outputs, so the
|
|
10
|
+
// LLM cannot emit an invalid enum value for a reserved field even if the
|
|
11
|
+
// manifest author forgot to constrain it themselves.
|
|
2
12
|
import { Ajv } from "ajv/dist/ajv.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
export const
|
|
7
|
-
export const
|
|
8
|
-
export const
|
|
9
|
-
export const
|
|
10
|
-
export const
|
|
11
|
-
export const
|
|
12
|
-
const
|
|
13
|
-
"normal",
|
|
14
|
-
"suspicious",
|
|
15
|
-
"high_risk",
|
|
16
|
-
"unknown",
|
|
17
|
-
];
|
|
18
|
-
const MANIFEST_KIND_VALUES = ["stock", "custom"];
|
|
19
|
-
const ajv = new Ajv({ allErrors: true, strict: false });
|
|
20
|
-
const COMMON_MANIFEST_KEYS = [
|
|
21
|
-
"kind",
|
|
13
|
+
import { APPLIES_TO_VALUES, CERTAINTY_VALUES } from "./stock.js";
|
|
14
|
+
import { RESERVED_FIELD_EXCLUSIONS, RESERVED_FIELD_NAMES, RESERVED_FIELDS, isReservedFieldName, normalizeToolId, } from "./reserved-fields.js";
|
|
15
|
+
import { isRecord, requireNonEmptyStringMaxLength, requireNonNegativeSafeInteger, requireString, throwInvalid, } from "./validation.js";
|
|
16
|
+
export const REASON_MAX_CHARS = 120;
|
|
17
|
+
export const TOOL_ID_MAX_CHARS = 64;
|
|
18
|
+
export const TOOL_DESCRIPTION_MAX_CHARS = 240;
|
|
19
|
+
export const MANIFEST_NAME_MAX_CHARS = 80;
|
|
20
|
+
export const MANIFEST_VERSION_MAX_CHARS = 40;
|
|
21
|
+
export const MANIFEST_PURPOSE_MAX_CHARS = 400;
|
|
22
|
+
const MANIFEST_KEYS = [
|
|
22
23
|
"name",
|
|
23
24
|
"version",
|
|
24
25
|
"purpose",
|
|
25
|
-
"
|
|
26
|
+
"dispatch_order",
|
|
27
|
+
"applies_to",
|
|
28
|
+
"reserved_fields",
|
|
29
|
+
"allowed_tools",
|
|
30
|
+
"output_schema",
|
|
26
31
|
"fallback",
|
|
27
32
|
"backend",
|
|
28
33
|
];
|
|
29
|
-
const
|
|
30
|
-
...COMMON_MANIFEST_KEYS,
|
|
31
|
-
"tools",
|
|
32
|
-
];
|
|
33
|
-
const CUSTOM_MANIFEST_KEYS = [
|
|
34
|
-
...COMMON_MANIFEST_KEYS,
|
|
35
|
-
"output_schema",
|
|
36
|
-
];
|
|
34
|
+
const ajv = new Ajv({ allErrors: true, strict: false });
|
|
37
35
|
export function validateJsonClassifierManifest(value, model = "manifest") {
|
|
38
36
|
if (!isRecord(value)) {
|
|
39
37
|
throwInvalid("manifest", model, "manifest must be a JSON object");
|
|
40
38
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
function validateStockManifest(value, model) {
|
|
47
|
-
ensureAllowedObjectKeys(value, STOCK_MANIFEST_KEYS, "manifest", model, "manifest");
|
|
48
|
-
const name = requireEnum(value.name, STOCK_CLASSIFIER_NAMES, "manifest", model, "name");
|
|
49
|
-
const base = validateManifestCommon(value, model);
|
|
50
|
-
const tools = value.tools === undefined
|
|
39
|
+
ensureAllowedObjectKeys(value, MANIFEST_KEYS, "manifest", model, "manifest");
|
|
40
|
+
const name = requireNonEmptyStringMaxLength(value.name, "manifest", model, "name", MANIFEST_NAME_MAX_CHARS);
|
|
41
|
+
const version = requireNonEmptyStringMaxLength(value.version, "manifest", model, "version", MANIFEST_VERSION_MAX_CHARS);
|
|
42
|
+
const purpose = requireNonEmptyStringMaxLength(value.purpose, "manifest", model, "purpose", MANIFEST_PURPOSE_MAX_CHARS);
|
|
43
|
+
const dispatchOrder = value.dispatch_order === undefined
|
|
51
44
|
? undefined
|
|
52
|
-
:
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
...base,
|
|
61
|
-
fallback,
|
|
62
|
-
...(tools === undefined ? {} : { tools }),
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
function validateCustomManifest(value, model) {
|
|
66
|
-
ensureAllowedObjectKeys(value, CUSTOM_MANIFEST_KEYS, "manifest", model, "manifest");
|
|
67
|
-
const name = requireNonEmptyStringMaxLength(value.name, "manifest", model, "name", STOCK_MANIFEST_NAME_MAX_CHARS);
|
|
68
|
-
if (STOCK_CLASSIFIER_NAMES.includes(name)) {
|
|
69
|
-
throwInvalid("manifest", model, `custom classifier name "${name}" collides with a stock classifier`);
|
|
45
|
+
: requireNonNegativeSafeInteger(value.dispatch_order, "manifest", model, "dispatch_order");
|
|
46
|
+
const appliesTo = validateAppliesTo(value.applies_to, model);
|
|
47
|
+
const reservedFields = validateReservedFields(value.reserved_fields, model);
|
|
48
|
+
const allowedTools = value.allowed_tools === undefined
|
|
49
|
+
? undefined
|
|
50
|
+
: validateAllowedTools(value.allowed_tools, model);
|
|
51
|
+
if (allowedTools !== undefined && !reservedFields.includes("tools")) {
|
|
52
|
+
throwInvalid("manifest", model, "allowed_tools is only supported when reserved_fields includes \"tools\"");
|
|
70
53
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
throwInvalid("manifest", model, "output_schema is required for custom classifiers");
|
|
54
|
+
if (reservedFields.includes("tools") && allowedTools === undefined) {
|
|
55
|
+
throwInvalid("manifest", model, "allowed_tools is required when reserved_fields includes \"tools\"");
|
|
74
56
|
}
|
|
75
|
-
const outputSchema = value.output_schema;
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
57
|
+
const outputSchema = validateOutputSchemaShape(value.output_schema, reservedFields, model);
|
|
58
|
+
const composedOutputSchema = composeOutputSchema(reservedFields, allowedTools, outputSchema);
|
|
59
|
+
compileSchema(composedOutputSchema, "manifest", model, "composed output_schema");
|
|
60
|
+
validateExamples(outputSchema, composedOutputSchema, model);
|
|
61
|
+
const fallback = validateFallback(value.fallback, composedOutputSchema, name, model);
|
|
62
|
+
const backend = value.backend === undefined ? undefined : validateBackend(value.backend, model);
|
|
63
|
+
const manifest = {
|
|
80
64
|
name,
|
|
81
|
-
|
|
65
|
+
version,
|
|
66
|
+
purpose,
|
|
67
|
+
...(dispatchOrder === undefined ? {} : { dispatch_order: dispatchOrder }),
|
|
68
|
+
...(value.applies_to === undefined ? {} : { applies_to: appliesTo }),
|
|
69
|
+
...(reservedFields.length === 0 ? {} : { reserved_fields: reservedFields }),
|
|
70
|
+
...(allowedTools === undefined ? {} : { allowed_tools: allowedTools }),
|
|
71
|
+
...(outputSchema === undefined ? {} : { output_schema: outputSchema }),
|
|
82
72
|
fallback,
|
|
83
|
-
|
|
73
|
+
...(backend === undefined ? {} : { backend }),
|
|
84
74
|
};
|
|
85
|
-
}
|
|
86
|
-
function validateManifestCommon(value, model) {
|
|
87
|
-
const version = requireNonEmptyStringMaxLength(value.version, "manifest", model, "version", STOCK_MANIFEST_VERSION_MAX_CHARS);
|
|
88
|
-
const purpose = requireNonEmptyStringMaxLength(value.purpose, "manifest", model, "purpose", STOCK_MANIFEST_PURPOSE_MAX_CHARS);
|
|
89
|
-
const order = requireNonNegativeSafeInteger(value.order, "manifest", model, "order");
|
|
90
75
|
return {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
76
|
+
manifest,
|
|
77
|
+
reservedFields,
|
|
78
|
+
composedOutputSchema,
|
|
79
|
+
appliesTo,
|
|
95
80
|
};
|
|
96
81
|
}
|
|
97
|
-
|
|
98
|
-
if (
|
|
99
|
-
return
|
|
82
|
+
function validateAppliesTo(raw, model) {
|
|
83
|
+
if (raw === undefined)
|
|
84
|
+
return "user";
|
|
85
|
+
if (typeof raw !== "string" || !APPLIES_TO_VALUES.includes(raw)) {
|
|
86
|
+
throwInvalid("manifest", model, `applies_to must be one of ${APPLIES_TO_VALUES.join(", ")}`);
|
|
100
87
|
}
|
|
101
|
-
return
|
|
88
|
+
return raw;
|
|
102
89
|
}
|
|
103
|
-
function
|
|
104
|
-
if (
|
|
105
|
-
|
|
90
|
+
function validateReservedFields(raw, model) {
|
|
91
|
+
if (raw === undefined)
|
|
92
|
+
return [];
|
|
93
|
+
if (!Array.isArray(raw)) {
|
|
94
|
+
throwInvalid("manifest", model, "reserved_fields must be an array of strings");
|
|
106
95
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
return validateModelSpecializationOutput(value, model);
|
|
114
|
-
case "tools":
|
|
115
|
-
return validateToolsOutput(value, model, tools?.map((tool) => tool.id));
|
|
116
|
-
case "prompt_injection":
|
|
117
|
-
return validatePromptInjectionOutput(value, model);
|
|
118
|
-
default: {
|
|
119
|
-
const _exhaustive = name;
|
|
120
|
-
void _exhaustive;
|
|
121
|
-
throwInvalid("manifest", model, `unknown stock classifier name`);
|
|
96
|
+
const seen = new Set();
|
|
97
|
+
const result = [];
|
|
98
|
+
for (let i = 0; i < raw.length; i++) {
|
|
99
|
+
const item = raw[i];
|
|
100
|
+
if (typeof item !== "string") {
|
|
101
|
+
throwInvalid("manifest", model, `reserved_fields[${i}] must be a string`);
|
|
122
102
|
}
|
|
103
|
+
if (!isReservedFieldName(item)) {
|
|
104
|
+
throwInvalid("manifest", model, `reserved_fields[${i}] "${item}" is not a known reserved field. Allowed: ${RESERVED_FIELD_NAMES.join(", ")}`);
|
|
105
|
+
}
|
|
106
|
+
if (seen.has(item)) {
|
|
107
|
+
throwInvalid("manifest", model, `reserved_fields[${i}] duplicates "${item}"`);
|
|
108
|
+
}
|
|
109
|
+
seen.add(item);
|
|
110
|
+
result.push(item);
|
|
123
111
|
}
|
|
112
|
+
return result;
|
|
124
113
|
}
|
|
125
|
-
function
|
|
126
|
-
if (value
|
|
127
|
-
throwInvalid(
|
|
128
|
-
}
|
|
129
|
-
if (value.certainty === undefined) {
|
|
130
|
-
throwInvalid(classifier, model, "certainty is required");
|
|
114
|
+
function validateAllowedTools(value, model) {
|
|
115
|
+
if (!Array.isArray(value)) {
|
|
116
|
+
throwInvalid("manifest", model, "allowed_tools must be an array");
|
|
131
117
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
118
|
+
const ids = new Set();
|
|
119
|
+
const out = [];
|
|
120
|
+
for (let i = 0; i < value.length; i++) {
|
|
121
|
+
const item = value[i];
|
|
122
|
+
if (!isRecord(item)) {
|
|
123
|
+
throwInvalid("manifest", model, `allowed_tools[${i}] must be an object`);
|
|
124
|
+
}
|
|
125
|
+
ensureAllowedObjectKeys(item, ["id", "description"], "manifest", model, `allowed_tools[${i}]`);
|
|
126
|
+
const id = requireNonEmptyStringMaxLength(item.id, "manifest", model, `allowed_tools[${i}].id`, TOOL_ID_MAX_CHARS);
|
|
127
|
+
const description = requireNonEmptyStringMaxLength(item.description, "manifest", model, `allowed_tools[${i}].description`, TOOL_DESCRIPTION_MAX_CHARS);
|
|
128
|
+
if (ids.has(id)) {
|
|
129
|
+
throwInvalid("manifest", model, `allowed_tools[${i}].id "${id}" is duplicated`);
|
|
130
|
+
}
|
|
131
|
+
ids.add(id);
|
|
132
|
+
out.push({ id, description });
|
|
141
133
|
}
|
|
142
|
-
|
|
143
|
-
return {
|
|
144
|
-
...meta,
|
|
145
|
-
...(value.final_reply === undefined
|
|
146
|
-
? {}
|
|
147
|
-
: { final_reply: validateReplySignal(value.final_reply, "preflight", model, "final_reply") }),
|
|
148
|
-
...(value.ack_reply === undefined
|
|
149
|
-
? {}
|
|
150
|
-
: { ack_reply: validateReplySignal(value.ack_reply, "preflight", model, "ack_reply") }),
|
|
151
|
-
};
|
|
134
|
+
return out;
|
|
152
135
|
}
|
|
153
|
-
function
|
|
154
|
-
if (
|
|
155
|
-
|
|
136
|
+
function validateOutputSchemaShape(raw, reservedFields, model) {
|
|
137
|
+
if (raw === undefined)
|
|
138
|
+
return undefined;
|
|
139
|
+
if (!isRecord(raw)) {
|
|
140
|
+
throwInvalid("manifest", model, "output_schema must be a JSON object");
|
|
156
141
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
142
|
+
// We don't fully validate JSON Schema shape — Ajv does that when we compile
|
|
143
|
+
// the composed schema. But we do enforce the no-reserved-keys rule, since
|
|
144
|
+
// mixing reserved fields into properties is always a manifest error.
|
|
145
|
+
const properties = raw.properties;
|
|
146
|
+
if (properties !== undefined) {
|
|
147
|
+
if (!isRecord(properties)) {
|
|
148
|
+
throwInvalid("manifest", model, "output_schema.properties must be a JSON object");
|
|
149
|
+
}
|
|
150
|
+
for (const key of Object.keys(properties)) {
|
|
151
|
+
if (isReservedFieldName(key)) {
|
|
152
|
+
throwInvalid("manifest", model, `output_schema.properties.${key} collides with a reserved field. Declare it in reserved_fields instead.`);
|
|
153
|
+
}
|
|
154
|
+
if (key === "reason" || key === "certainty") {
|
|
155
|
+
throwInvalid("manifest", model, `output_schema.properties.${key} is reserved. Do not declare it.`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
161
158
|
}
|
|
162
|
-
if
|
|
163
|
-
|
|
159
|
+
// Examples must be an array if present. Per-example validation happens
|
|
160
|
+
// after the composed schema is built so we can validate against it.
|
|
161
|
+
if (raw.examples !== undefined && !Array.isArray(raw.examples)) {
|
|
162
|
+
throwInvalid("manifest", model, "output_schema.examples must be an array");
|
|
164
163
|
}
|
|
165
|
-
|
|
164
|
+
void reservedFields;
|
|
165
|
+
return raw;
|
|
166
166
|
}
|
|
167
|
-
function
|
|
168
|
-
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
...meta,
|
|
173
|
-
...(modelTier === undefined
|
|
174
|
-
? {}
|
|
175
|
-
: { model_tier: requireEnum(modelTier, DOWNSTREAM_MODEL_TIER_VALUES, "routing", model, "model_tier") }),
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
function validateModelSpecializationOutput(value, model) {
|
|
179
|
-
ensureAllowedObjectKeys(value, ["reason", "certainty", "specialization"], "model_specialization", model, "output");
|
|
180
|
-
const meta = validateMetadata(value, "model_specialization", model);
|
|
181
|
-
const specialization = normalizeOptionalEnumValue(value.specialization);
|
|
182
|
-
return {
|
|
183
|
-
...meta,
|
|
184
|
-
...(specialization === undefined
|
|
185
|
-
? {}
|
|
186
|
-
: { specialization: requireEnum(specialization, MODEL_SPECIALIZATION_VALUES, "model_specialization", model, "specialization") }),
|
|
167
|
+
function composeOutputSchema(reservedFields, allowedTools, outputSchema) {
|
|
168
|
+
const context = { allowed_tools: allowedTools };
|
|
169
|
+
const properties = {
|
|
170
|
+
reason: { type: "string", minLength: 1, maxLength: REASON_MAX_CHARS },
|
|
171
|
+
certainty: { type: "string", enum: [...CERTAINTY_VALUES] },
|
|
187
172
|
};
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if (value === undefined || value === null) {
|
|
191
|
-
return undefined;
|
|
192
|
-
}
|
|
193
|
-
if (typeof value === "string" && value.trim().length === 0) {
|
|
194
|
-
return undefined;
|
|
173
|
+
for (const field of reservedFields) {
|
|
174
|
+
properties[field] = RESERVED_FIELDS[field].subSchema(context);
|
|
195
175
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const tools = requireStringArray(value.tools, "tools", model, "tools").map(normalizeTool);
|
|
202
|
-
ensureNoDuplicates(tools, "tools", model, "tools");
|
|
203
|
-
if (configuredTools) {
|
|
204
|
-
const allowed = new Set(configuredTools);
|
|
205
|
-
for (const tool of tools) {
|
|
206
|
-
if (!allowed.has(tool)) {
|
|
207
|
-
throwInvalid("tools", model, `tools includes unsupported tool ${tool}`);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
176
|
+
const customProperties = isRecord(outputSchema?.properties)
|
|
177
|
+
? outputSchema.properties
|
|
178
|
+
: {};
|
|
179
|
+
for (const [key, schema] of Object.entries(customProperties)) {
|
|
180
|
+
properties[key] = schema;
|
|
210
181
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
const meta = validateMetadata(value, "prompt_injection", model);
|
|
216
|
-
const riskLevel = requireEnum(value.risk_level, STOCK_PROMPT_INJECTION_RISK_LEVEL_VALUES, "prompt_injection", model, "risk_level");
|
|
182
|
+
const customRequired = Array.isArray(outputSchema?.required)
|
|
183
|
+
? outputSchema.required.filter((key) => typeof key === "string")
|
|
184
|
+
: [];
|
|
185
|
+
const required = Array.from(new Set(["reason", "certainty", ...customRequired]));
|
|
217
186
|
return {
|
|
218
|
-
|
|
219
|
-
|
|
187
|
+
type: "object",
|
|
188
|
+
additionalProperties: false,
|
|
189
|
+
required,
|
|
190
|
+
properties,
|
|
220
191
|
};
|
|
221
192
|
}
|
|
222
|
-
function
|
|
223
|
-
if (
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
193
|
+
function validateExamples(outputSchema, composedSchema, model) {
|
|
194
|
+
if (outputSchema === undefined)
|
|
195
|
+
return;
|
|
196
|
+
const examples = outputSchema.examples;
|
|
197
|
+
if (!Array.isArray(examples))
|
|
198
|
+
return;
|
|
199
|
+
const validate = ajv.compile(composedSchema);
|
|
200
|
+
for (let i = 0; i < examples.length; i++) {
|
|
201
|
+
const example = examples[i];
|
|
202
|
+
if (!validate(example)) {
|
|
203
|
+
const message = formatSchemaErrors(validate.errors, `examples[${i}]`);
|
|
204
|
+
throwInvalid("manifest", model, `output_schema.examples[${i}] is invalid: ${message}`);
|
|
205
|
+
}
|
|
206
|
+
if (isRecord(example)) {
|
|
207
|
+
enforceMutualExclusions(example, "manifest", model, `output_schema.examples[${i}]`);
|
|
208
|
+
}
|
|
229
209
|
}
|
|
230
|
-
const meta = validateMetadata(value, classifier, model);
|
|
231
|
-
validateWithSchema(value.output, schema, classifier, model, "output");
|
|
232
|
-
return { ...meta, output: value.output };
|
|
233
210
|
}
|
|
234
|
-
function
|
|
235
|
-
if (!
|
|
236
|
-
throwInvalid(
|
|
211
|
+
function validateFallback(raw, composedSchema, classifier, model) {
|
|
212
|
+
if (!isRecord(raw)) {
|
|
213
|
+
throwInvalid(classifier, model, "fallback must be a JSON object");
|
|
237
214
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
ensureNoDuplicates(out.map((item) => item.id), "manifest", model, "tools[].id");
|
|
248
|
-
return out;
|
|
215
|
+
// Fallback represents the "I have no signal" state, so reserved fields are
|
|
216
|
+
// optional. The composed schema already marks them optional except for
|
|
217
|
+
// reason/certainty.
|
|
218
|
+
const validate = ajv.compile(composedSchema);
|
|
219
|
+
if (!validate(raw)) {
|
|
220
|
+
const message = formatSchemaErrors(validate.errors, "fallback");
|
|
221
|
+
throwInvalid(classifier, model, `fallback is invalid: ${message}`);
|
|
222
|
+
}
|
|
223
|
+
return raw;
|
|
249
224
|
}
|
|
250
225
|
function validateBackend(value, model) {
|
|
251
226
|
if (!isRecord(value))
|
|
@@ -265,6 +240,56 @@ function validateBackend(value, model) {
|
|
|
265
240
|
},
|
|
266
241
|
};
|
|
267
242
|
}
|
|
243
|
+
export function validateOutputForManifest(manifest, value, context) {
|
|
244
|
+
if (!isRecord(value)) {
|
|
245
|
+
throwInvalid(context.classifier, context.model, "output must be a JSON object");
|
|
246
|
+
}
|
|
247
|
+
const normalized = normalizeOutput(value, manifest);
|
|
248
|
+
const validate = ajv.compile(manifest.composedOutputSchema);
|
|
249
|
+
if (!validate(normalized)) {
|
|
250
|
+
const message = formatSchemaErrors(validate.errors, "output");
|
|
251
|
+
throwInvalid(context.classifier, context.model, message);
|
|
252
|
+
}
|
|
253
|
+
enforceMutualExclusions(normalized, context.classifier, context.model, "output");
|
|
254
|
+
return truncateReason(normalized);
|
|
255
|
+
}
|
|
256
|
+
// Apply small, well-known cleanups that consistently improve LLM compliance:
|
|
257
|
+
// - blank / null routing-style enums → omitted entirely
|
|
258
|
+
// - tools aliases (browser → web, etc.)
|
|
259
|
+
function normalizeOutput(value, manifest) {
|
|
260
|
+
const out = { ...value };
|
|
261
|
+
for (const field of manifest.reservedFields) {
|
|
262
|
+
if (field === "model_tier" || field === "model_specialization") {
|
|
263
|
+
const raw = out[field];
|
|
264
|
+
if (raw === null || (typeof raw === "string" && raw.trim().length === 0)) {
|
|
265
|
+
delete out[field];
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
if (field === "tools") {
|
|
269
|
+
const raw = out[field];
|
|
270
|
+
if (Array.isArray(raw)) {
|
|
271
|
+
out[field] = raw.map((item) => typeof item === "string" ? normalizeToolId(item) : item);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return out;
|
|
276
|
+
}
|
|
277
|
+
function truncateReason(value) {
|
|
278
|
+
if (typeof value.reason !== "string")
|
|
279
|
+
return value;
|
|
280
|
+
if (value.reason.length <= REASON_MAX_CHARS)
|
|
281
|
+
return value;
|
|
282
|
+
return { ...value, reason: value.reason.slice(0, REASON_MAX_CHARS).trimEnd() };
|
|
283
|
+
}
|
|
284
|
+
function enforceMutualExclusions(output, classifier, model, path) {
|
|
285
|
+
for (const group of RESERVED_FIELD_EXCLUSIONS) {
|
|
286
|
+
const present = group.filter((field) => output[field] !== undefined);
|
|
287
|
+
if (present.length > 1) {
|
|
288
|
+
throwInvalid(classifier, model, `${path}: reserved fields are mutually exclusive: ${present.join(", ")}`);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
268
293
|
function ensureAllowedObjectKeys(value, allowedKeys, classifier, model, path) {
|
|
269
294
|
const allowed = new Set(allowedKeys);
|
|
270
295
|
for (const key of Object.keys(value)) {
|
|
@@ -273,40 +298,42 @@ function ensureAllowedObjectKeys(value, allowedKeys, classifier, model, path) {
|
|
|
273
298
|
}
|
|
274
299
|
}
|
|
275
300
|
}
|
|
276
|
-
function
|
|
301
|
+
function compileSchema(schema, classifier, model, label) {
|
|
277
302
|
try {
|
|
278
303
|
ajv.compile(schema);
|
|
279
304
|
}
|
|
280
305
|
catch (error) {
|
|
281
|
-
throwInvalid(classifier, model,
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
export function validateWithSchema(value, schema, classifier, model, path) {
|
|
285
|
-
const validate = ajv.compile(schema);
|
|
286
|
-
if (!validate(value)) {
|
|
287
|
-
const message = ajv.errorsText(validate.errors, { dataVar: path });
|
|
288
|
-
throwInvalid(classifier, model, message);
|
|
306
|
+
throwInvalid(classifier, model, `${label} is invalid JSON Schema: ${error instanceof Error ? error.message : String(error)}`);
|
|
289
307
|
}
|
|
290
308
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
309
|
+
// Format Ajv errors with the offending property name surfaced inline. Ajv's
|
|
310
|
+
// default errorsText says "must NOT have additional properties" without
|
|
311
|
+
// naming the property, which is unhelpful to debug.
|
|
312
|
+
function formatSchemaErrors(errors, dataVar) {
|
|
313
|
+
if (!errors || errors.length === 0)
|
|
314
|
+
return `${dataVar} is invalid`;
|
|
315
|
+
return errors
|
|
316
|
+
.map((err) => {
|
|
317
|
+
const path = `${dataVar}${err.instancePath ?? ""}`;
|
|
318
|
+
if (err.keyword === "additionalProperties") {
|
|
319
|
+
const extra = err.params?.additionalProperty;
|
|
320
|
+
if (extra) {
|
|
321
|
+
return `${path}.${extra} is not a supported field`;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
if (err.keyword === "required") {
|
|
325
|
+
const missing = err.params?.missingProperty;
|
|
326
|
+
if (missing) {
|
|
327
|
+
return `${path}.${missing} is required`;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (err.keyword === "enum") {
|
|
331
|
+
const allowed = err.params?.allowedValues;
|
|
332
|
+
if (allowed) {
|
|
333
|
+
return `${path} has an unsupported value (allowed: ${allowed.join(", ")})`;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
return `${path} ${err.message ?? "is invalid"}`;
|
|
337
|
+
})
|
|
338
|
+
.join("; ");
|
|
312
339
|
}
|