open-classify 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -86
- package/dist/src/aggregator.d.ts +11 -4
- package/dist/src/aggregator.js +108 -121
- package/dist/src/classifiers/{custom/context_shift → context_shift}/manifest.json +6 -11
- package/dist/src/classifiers/{custom/context_shift → context_shift}/prompt.md +1 -1
- package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/manifest.json +7 -12
- package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/prompt.md +2 -2
- package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/manifest.json +6 -11
- package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/prompt.md +2 -2
- package/dist/src/classifiers/{stock/model_specialization → model_specialization}/manifest.json +2 -2
- package/dist/src/classifiers/model_specialization/prompt.md +5 -0
- package/dist/src/classifiers/preflight/manifest.json +34 -0
- package/dist/src/classifiers/preflight/prompt.md +10 -0
- package/dist/src/classifiers/{stock/prompt_injection → prompt_injection}/manifest.json +6 -2
- package/dist/src/classifiers/prompt_injection/prompt.md +14 -0
- package/dist/src/classifiers/{stock/routing → routing}/manifest.json +2 -2
- package/dist/src/classifiers/routing/prompt.md +5 -0
- package/dist/src/classifiers/{stock/tools → tools}/manifest.json +3 -3
- package/dist/src/classifiers/tools/prompt.md +5 -0
- package/dist/src/classifiers.js +31 -29
- package/dist/src/classify.d.ts +9 -2
- package/dist/src/classify.js +26 -12
- package/dist/src/config.d.ts +1 -4
- package/dist/src/config.js +6 -34
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/input.d.ts +4 -1
- package/dist/src/input.js +12 -10
- package/dist/src/manifest.d.ts +11 -7
- package/dist/src/pipeline.d.ts +9 -1
- package/dist/src/pipeline.js +51 -25
- package/dist/src/reserved-fields.d.ts +18 -0
- package/dist/src/reserved-fields.js +175 -0
- package/dist/src/stock-prompt.d.ts +9 -2
- package/dist/src/stock-prompt.js +165 -45
- package/dist/src/stock-validation.d.ts +16 -17
- package/dist/src/stock-validation.js +263 -236
- package/dist/src/stock.d.ts +24 -60
- package/dist/src/stock.js +7 -14
- package/docs/adding-a-classifier.md +74 -32
- package/docs/manifests.md +112 -71
- package/docs/resolver.md +25 -34
- package/docs/signals.md +39 -58
- package/open-classify.config.example.json +9 -11
- package/package.json +1 -1
- package/dist/src/classifiers/stock/preflight/manifest.json +0 -11
- package/dist/src/classifiers/stock/prompts/classifier-header.md +0 -4
- package/dist/src/classifiers/stock/prompts/custom-output.md +0 -7
- package/dist/src/classifiers/stock/prompts/model_specialization.md +0 -7
- package/dist/src/classifiers/stock/prompts/preflight-output.md +0 -10
- package/dist/src/classifiers/stock/prompts/preflight.md +0 -47
- package/dist/src/classifiers/stock/prompts/prompt-injection-output.md +0 -5
- package/dist/src/classifiers/stock/prompts/prompt_injection.md +0 -24
- package/dist/src/classifiers/stock/prompts/routing-output.md +0 -5
- package/dist/src/classifiers/stock/prompts/routing.md +0 -9
- package/dist/src/classifiers/stock/prompts/specialty.md +0 -12
- package/dist/src/classifiers/stock/prompts/tier.md +0 -7
- package/dist/src/classifiers/stock/prompts/tools-output.md +0 -11
- package/dist/src/classifiers/stock/prompts/tools.md +0 -10
- /package/dist/src/classifiers/{stock/prompts → _prompts}/base.md +0 -0
- /package/dist/src/classifiers/{stock/prompts → _prompts}/confidence.md +0 -0
- /package/dist/src/classifiers/{stock/prompts → _prompts}/reason.md +0 -0
package/dist/src/stock.d.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import type { DownstreamModelTier, ModelSpecialization } from "./enums.js";
|
|
2
|
-
|
|
1
|
+
import type { DownstreamModelTier, ModelSpecialization, PromptInjectionRiskLevel } from "./enums.js";
|
|
2
|
+
import type { ReservedFieldName } from "./reserved-fields.js";
|
|
3
|
+
export interface ClassifierMessageInput {
|
|
3
4
|
readonly role: "user" | "assistant";
|
|
4
5
|
readonly text: string;
|
|
5
6
|
}
|
|
6
|
-
export interface
|
|
7
|
-
readonly messages: ReadonlyArray<
|
|
7
|
+
export interface ClassifierMessageWindowInput {
|
|
8
|
+
readonly messages: ReadonlyArray<ClassifierMessageInput>;
|
|
8
9
|
}
|
|
9
10
|
export interface FinalReplySignal {
|
|
10
11
|
readonly text: string;
|
|
@@ -14,19 +15,13 @@ export interface AckReplySignal {
|
|
|
14
15
|
}
|
|
15
16
|
export interface RoutingSignal {
|
|
16
17
|
readonly model_tier?: DownstreamModelTier;
|
|
17
|
-
readonly
|
|
18
|
-
}
|
|
19
|
-
export interface TierSignal {
|
|
20
|
-
readonly model_tier?: DownstreamModelTier;
|
|
21
|
-
}
|
|
22
|
-
export interface SpecializationSignal {
|
|
23
|
-
readonly specialization?: ModelSpecialization;
|
|
18
|
+
readonly model_specialization?: ModelSpecialization;
|
|
24
19
|
}
|
|
25
20
|
export interface ToolsSignal {
|
|
26
21
|
readonly tools: ReadonlyArray<string>;
|
|
27
22
|
}
|
|
28
23
|
export interface PromptInjectionSignal {
|
|
29
|
-
readonly risk_level:
|
|
24
|
+
readonly risk_level: PromptInjectionRiskLevel;
|
|
30
25
|
}
|
|
31
26
|
export type Certainty = "no_signal" | "very_weak" | "weak" | "tentative" | "reasonable" | "strong" | "very_strong" | "near_certain";
|
|
32
27
|
export declare const CERTAINTY_VALUES: readonly ["no_signal", "very_weak", "weak", "tentative", "reasonable", "strong", "very_strong", "near_certain"];
|
|
@@ -35,68 +30,37 @@ export interface ClassifierOutputMetadata {
|
|
|
35
30
|
readonly reason: string;
|
|
36
31
|
readonly certainty: Certainty;
|
|
37
32
|
}
|
|
38
|
-
export interface
|
|
39
|
-
readonly
|
|
40
|
-
readonly ack_reply?: AckReplySignal;
|
|
41
|
-
}
|
|
42
|
-
export type RoutingClassifierOutput = TierSignal & ClassifierOutputMetadata;
|
|
43
|
-
export type ModelSpecializationClassifierOutput = SpecializationSignal & ClassifierOutputMetadata;
|
|
44
|
-
export type ToolsClassifierOutput = ToolsSignal & ClassifierOutputMetadata;
|
|
45
|
-
export type PromptInjectionClassifierOutput = PromptInjectionSignal & ClassifierOutputMetadata;
|
|
46
|
-
export interface CustomClassifierOutputValue extends ClassifierOutputMetadata {
|
|
47
|
-
readonly output: unknown;
|
|
33
|
+
export interface ClassifierOutput extends ClassifierOutputMetadata {
|
|
34
|
+
readonly [key: string]: unknown;
|
|
48
35
|
}
|
|
49
|
-
export interface StockClassifierOutputs {
|
|
50
|
-
readonly preflight: PreflightClassifierOutput;
|
|
51
|
-
readonly routing: RoutingClassifierOutput;
|
|
52
|
-
readonly model_specialization: ModelSpecializationClassifierOutput;
|
|
53
|
-
readonly tools: ToolsClassifierOutput;
|
|
54
|
-
readonly prompt_injection: PromptInjectionClassifierOutput;
|
|
55
|
-
}
|
|
56
|
-
export declare const STOCK_CLASSIFIER_NAMES: readonly ["preflight", "routing", "model_specialization", "tools", "prompt_injection"];
|
|
57
|
-
export type StockClassifierName = (typeof STOCK_CLASSIFIER_NAMES)[number];
|
|
58
|
-
export type StockClassifierOutput = StockClassifierOutputs[StockClassifierName];
|
|
59
|
-
export type ClassifierOutput = StockClassifierOutput | CustomClassifierOutputValue;
|
|
60
36
|
export interface ToolDefinition {
|
|
61
37
|
readonly id: string;
|
|
62
38
|
readonly description: string;
|
|
63
39
|
}
|
|
64
|
-
|
|
40
|
+
export type AppliesTo = "user" | "assistant" | "both";
|
|
41
|
+
export declare const APPLIES_TO_VALUES: readonly ["user", "assistant", "both"];
|
|
42
|
+
export interface JsonClassifierManifest {
|
|
43
|
+
readonly name: string;
|
|
65
44
|
readonly version: string;
|
|
66
45
|
readonly purpose: string;
|
|
67
|
-
readonly
|
|
46
|
+
readonly dispatch_order?: number;
|
|
47
|
+
readonly applies_to?: AppliesTo;
|
|
48
|
+
readonly reserved_fields?: ReadonlyArray<ReservedFieldName>;
|
|
49
|
+
readonly allowed_tools?: ReadonlyArray<ToolDefinition>;
|
|
50
|
+
readonly output_schema?: unknown;
|
|
51
|
+
readonly fallback: ClassifierOutput;
|
|
68
52
|
readonly backend?: {
|
|
69
53
|
readonly ollama?: {
|
|
70
54
|
readonly base_model?: string;
|
|
71
55
|
};
|
|
72
56
|
};
|
|
73
57
|
}
|
|
74
|
-
export interface
|
|
75
|
-
readonly kind: "stock";
|
|
76
|
-
readonly name: Name;
|
|
77
|
-
readonly fallback: StockClassifierOutputs[Name];
|
|
78
|
-
readonly tools?: ReadonlyArray<ToolDefinition>;
|
|
79
|
-
}
|
|
80
|
-
export interface CustomJsonManifest extends ManifestCommon {
|
|
81
|
-
readonly kind: "custom";
|
|
82
|
-
readonly name: string;
|
|
83
|
-
readonly fallback: CustomClassifierOutputValue;
|
|
84
|
-
readonly output_schema: unknown;
|
|
85
|
-
}
|
|
86
|
-
export type JsonClassifierManifest = StockJsonManifest | CustomJsonManifest;
|
|
87
|
-
export interface RuntimeStockManifest<Name extends StockClassifierName = StockClassifierName> extends StockJsonManifest<Name> {
|
|
88
|
-
readonly systemPrompt: string;
|
|
89
|
-
}
|
|
90
|
-
export interface RuntimeCustomManifest extends CustomJsonManifest {
|
|
58
|
+
export interface RuntimeClassifierManifest extends JsonClassifierManifest {
|
|
91
59
|
readonly systemPrompt: string;
|
|
60
|
+
readonly composedOutputSchema: unknown;
|
|
61
|
+
readonly reservedFields: ReadonlyArray<ReservedFieldName>;
|
|
62
|
+
readonly appliesTo: AppliesTo;
|
|
92
63
|
}
|
|
93
|
-
export
|
|
94
|
-
export declare function isStockManifest(manifest: RuntimeClassifierManifest): manifest is RuntimeStockManifest;
|
|
95
|
-
export declare function isCustomManifest(manifest: RuntimeClassifierManifest): manifest is RuntimeCustomManifest;
|
|
96
|
-
export interface CustomClassifierOutput {
|
|
64
|
+
export interface ClassifierAuditOutput extends ClassifierOutput {
|
|
97
65
|
readonly classifier: string;
|
|
98
|
-
readonly reason: string;
|
|
99
|
-
readonly certainty: Certainty;
|
|
100
|
-
readonly output: unknown;
|
|
101
66
|
}
|
|
102
|
-
export {};
|
package/dist/src/stock.js
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
// Classifier type contracts.
|
|
2
|
+
//
|
|
3
|
+
// Every classifier — reserved-field-bearing or not — uses the same manifest
|
|
4
|
+
// shape and emits the same output envelope: `{ reason, certainty, ...payload }`
|
|
5
|
+
// where `payload` may include any subset of the classifier's declared
|
|
6
|
+
// reserved fields plus its custom (schema-validated) properties.
|
|
1
7
|
export const CERTAINTY_VALUES = [
|
|
2
8
|
"no_signal",
|
|
3
9
|
"very_weak",
|
|
@@ -18,17 +24,4 @@ export const certaintyScore = {
|
|
|
18
24
|
very_strong: 0.88,
|
|
19
25
|
near_certain: 0.97,
|
|
20
26
|
};
|
|
21
|
-
export const
|
|
22
|
-
"preflight",
|
|
23
|
-
"routing",
|
|
24
|
-
"model_specialization",
|
|
25
|
-
"tools",
|
|
26
|
-
"prompt_injection",
|
|
27
|
-
];
|
|
28
|
-
// Helper: narrow a manifest to its stock kind for callers that know the name.
|
|
29
|
-
export function isStockManifest(manifest) {
|
|
30
|
-
return manifest.kind === "stock";
|
|
31
|
-
}
|
|
32
|
-
export function isCustomManifest(manifest) {
|
|
33
|
-
return manifest.kind === "custom";
|
|
34
|
-
}
|
|
27
|
+
export const APPLIES_TO_VALUES = ["user", "assistant", "both"];
|
|
@@ -1,36 +1,28 @@
|
|
|
1
1
|
# Adding a classifier
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Every classifier — reserved-field-bearing or pure custom — uses the same two-file layout. There is no separate "stock" vs "custom" distinction; the runtime only cares about which reserved fields a classifier opts into.
|
|
4
4
|
|
|
5
|
-
## 1.
|
|
6
|
-
|
|
7
|
-
Custom classifier:
|
|
5
|
+
## 1. Create the directory
|
|
8
6
|
|
|
9
7
|
```
|
|
10
|
-
src/classifiers
|
|
8
|
+
src/classifiers/<name>/
|
|
11
9
|
├── manifest.json
|
|
12
10
|
└── prompt.md
|
|
13
11
|
```
|
|
14
12
|
|
|
15
|
-
|
|
13
|
+
The directory name must match `manifest.json`'s `name` field. Top-level directories starting with `_` (like `_prompts/`) are reserved for shared assets and skipped by the loader.
|
|
16
14
|
|
|
17
15
|
## 2. Write the manifest
|
|
18
16
|
|
|
17
|
+
Minimal example — a pure-custom classifier that emits tags. You don't need to provide JSON examples; the runtime synthesizes one from your schema and shows it to the model.
|
|
18
|
+
|
|
19
19
|
```json
|
|
20
20
|
{
|
|
21
|
-
"kind": "custom",
|
|
22
21
|
"name": "topic_tags",
|
|
23
22
|
"version": "1.0.0",
|
|
24
23
|
"purpose": "Tag the message with a small set of topic labels for analytics.",
|
|
25
|
-
"
|
|
26
|
-
"fallback": {
|
|
27
|
-
"reason": "Classifier failed; no tags generated.",
|
|
28
|
-
"certainty": "no_signal",
|
|
29
|
-
"output": { "tags": [] }
|
|
30
|
-
},
|
|
24
|
+
"dispatch_order": 70,
|
|
31
25
|
"output_schema": {
|
|
32
|
-
"type": "object",
|
|
33
|
-
"additionalProperties": false,
|
|
34
26
|
"required": ["tags"],
|
|
35
27
|
"properties": {
|
|
36
28
|
"tags": {
|
|
@@ -38,37 +30,70 @@ Stock classifier names are closed (`preflight`, `routing`, `model_specialization
|
|
|
38
30
|
"items": { "type": "string", "minLength": 1, "maxLength": 40 }
|
|
39
31
|
}
|
|
40
32
|
}
|
|
33
|
+
},
|
|
34
|
+
"fallback": {
|
|
35
|
+
"reason": "Classifier failed; no tags generated.",
|
|
36
|
+
"certainty": "no_signal",
|
|
37
|
+
"tags": []
|
|
41
38
|
}
|
|
42
39
|
}
|
|
43
40
|
```
|
|
44
41
|
|
|
42
|
+
If your classifier's behavior is nuanced enough that hand-picked examples would help the model (preflight is one), add an `output_schema.examples` array. The runtime validates each example against the composed schema at load time, so a broken example fails the build.
|
|
43
|
+
|
|
44
|
+
To also influence routing, opt into a reserved field:
|
|
45
|
+
|
|
46
|
+
```json
|
|
47
|
+
{
|
|
48
|
+
"name": "topic_tags",
|
|
49
|
+
"version": "1.0.0",
|
|
50
|
+
"purpose": "Tag the message and pick a specialization for the downstream model.",
|
|
51
|
+
"dispatch_order": 70,
|
|
52
|
+
"reserved_fields": ["model_specialization"],
|
|
53
|
+
"output_schema": {
|
|
54
|
+
"required": ["tags"],
|
|
55
|
+
"properties": {
|
|
56
|
+
"tags": { "type": "array", "items": { "type": "string" } }
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"fallback": {
|
|
60
|
+
"reason": "Classifier failed.",
|
|
61
|
+
"certainty": "no_signal",
|
|
62
|
+
"tags": []
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The runtime knows `model_specialization` is a reserved field and injects its canonical enum values into the prompt automatically. You don't paste enum values in your `prompt.md`.
|
|
68
|
+
|
|
45
69
|
Rules:
|
|
46
70
|
|
|
47
71
|
- `name` must match the directory name.
|
|
48
|
-
-
|
|
49
|
-
- `
|
|
50
|
-
- `fallback` must validate against
|
|
72
|
+
- Reserved field names cannot appear in `output_schema.properties`; declare them in `reserved_fields` instead.
|
|
73
|
+
- `reason` and `certainty` are added to the composed schema by the runtime — don't declare them.
|
|
74
|
+
- `fallback` must validate against the composed schema. Reserved fields are optional in fallback (a "no signal" fallback usually omits them).
|
|
75
|
+
- `output_schema.examples` (JSON Schema standard) must validate against the composed schema at load time, so a broken example fails the build, not the model call.
|
|
51
76
|
|
|
52
77
|
See [manifests.md](manifests.md) for the full field list.
|
|
53
78
|
|
|
54
79
|
## 3. Write the prompt
|
|
55
80
|
|
|
56
|
-
`prompt.md` is the classifier-specific instruction text. The runtime composes it with
|
|
81
|
+
`prompt.md` is the classifier-specific instruction text. The runtime composes it with auto-generated sections describing the JSON contract and the reserved fields you opted into, so your prompt can focus on the classification rule:
|
|
57
82
|
|
|
58
83
|
```markdown
|
|
59
84
|
You are the topic_tags classifier.
|
|
60
85
|
|
|
61
|
-
|
|
86
|
+
`tags` are short single-word topic labels (lowercase, no spaces). Use at most five.
|
|
62
87
|
Return an empty array when no clear topic applies.
|
|
63
88
|
Do not invent tags for vague or ambiguous messages.
|
|
64
89
|
```
|
|
65
90
|
|
|
66
|
-
|
|
91
|
+
Don't paste enum values for reserved fields — the runtime injects them with canonical wording so they never drift from `src/enums.ts`.
|
|
67
92
|
|
|
68
93
|
## 4. Build and test
|
|
69
94
|
|
|
70
95
|
```sh
|
|
71
|
-
npm run build # validates the manifest,
|
|
96
|
+
npm run build # validates the manifest, composes the schema, copies assets
|
|
72
97
|
npm test
|
|
73
98
|
```
|
|
74
99
|
|
|
@@ -77,14 +102,33 @@ If the manifest is malformed, the loader throws `ClassifierManifestError` with t
|
|
|
77
102
|
## 5. Consume the output
|
|
78
103
|
|
|
79
104
|
```ts
|
|
80
|
-
const classify = createClassifier({ catalog });
|
|
105
|
+
const { classify } = createClassifier({ catalog });
|
|
81
106
|
const result = await classify(input);
|
|
82
|
-
|
|
83
|
-
const tags = result.classifier_outputs.topic_tags?.tags ?? [];
|
|
84
|
-
}
|
|
107
|
+
const tags = result.classifier_outputs.topic_tags?.tags ?? [];
|
|
85
108
|
```
|
|
86
109
|
|
|
87
|
-
`result.audit.
|
|
110
|
+
`result.audit.classifier_outputs[]` carries the same data with `reason` and `certainty` attached if you need to inspect them.
|
|
111
|
+
|
|
112
|
+
## Targeting the assistant response
|
|
113
|
+
|
|
114
|
+
Classifiers run against the user message by default. To run a classifier against the assistant's reply instead (or in addition), set `applies_to` in the manifest:
|
|
115
|
+
|
|
116
|
+
- `"user"` (default) — only `classify()` runs it.
|
|
117
|
+
- `"assistant"` — only `inspect()` runs it.
|
|
118
|
+
- `"both"` — both passes run it.
|
|
119
|
+
|
|
120
|
+
Use `inspect()` from `createClassifier()` for the assistant-side pass. It returns a lean shape (`target_message_hash` + `classifier_outputs`) — no routing, no audit envelope. The built-in `prompt_injection` ships tagged `"both"` so it runs on both sides.
|
|
121
|
+
|
|
122
|
+
```ts
|
|
123
|
+
const { inspect } = createClassifier({ catalog });
|
|
124
|
+
const post = await inspect({
|
|
125
|
+
messages: [
|
|
126
|
+
{ role: "user", text: "Summarize the contract." },
|
|
127
|
+
{ role: "assistant", text: "The contract has three notable risks…" },
|
|
128
|
+
],
|
|
129
|
+
});
|
|
130
|
+
const risk = post.classifier_outputs.prompt_injection?.risk_level;
|
|
131
|
+
```
|
|
88
132
|
|
|
89
133
|
## Choosing the classifier model
|
|
90
134
|
|
|
@@ -96,15 +140,13 @@ For apps and OSS installs, prefer `open-classify.config.json`:
|
|
|
96
140
|
"provider": "ollama",
|
|
97
141
|
"defaultModel": "gemma4:e4b-it-q4_K_M",
|
|
98
142
|
"models": {
|
|
99
|
-
"
|
|
100
|
-
"topic_tags": "qwen2.5:7b-instruct-q4_K_M"
|
|
101
|
-
}
|
|
143
|
+
"topic_tags": "qwen2.5:7b-instruct-q4_K_M"
|
|
102
144
|
}
|
|
103
145
|
}
|
|
104
146
|
}
|
|
105
147
|
```
|
|
106
148
|
|
|
107
|
-
`runner.defaultModel` applies to every classifier without an override. `runner.models
|
|
149
|
+
`runner.defaultModel` applies to every classifier without an override. `runner.models` is a flat map keyed by classifier name — there is no separate stock/custom split.
|
|
108
150
|
|
|
109
151
|
Classifier manifests may also carry an Ollama hint for packaged classifiers:
|
|
110
152
|
|
|
@@ -125,7 +167,7 @@ import { classifyOpenClassifyInput, loadCatalog } from "open-classify";
|
|
|
125
167
|
|
|
126
168
|
const runClassifier: RunClassifier = async (name, input, signal) => {
|
|
127
169
|
// call OpenAI, Anthropic, a remote service, etc.
|
|
128
|
-
// return a ClassifierOutput matching the classifier's
|
|
170
|
+
// return a ClassifierOutput matching the classifier's composed schema.
|
|
129
171
|
};
|
|
130
172
|
|
|
131
173
|
await classifyOpenClassifyInput(input, { runClassifier, catalog: loadCatalog(...) });
|
package/docs/manifests.md
CHANGED
|
@@ -1,127 +1,168 @@
|
|
|
1
1
|
# Manifest reference
|
|
2
2
|
|
|
3
|
-
Every classifier
|
|
4
|
-
|
|
5
|
-
## Layout
|
|
3
|
+
Every classifier lives in `src/classifiers/<name>/` and contains exactly two files:
|
|
6
4
|
|
|
7
5
|
```
|
|
8
6
|
src/classifiers/
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
confidence.md
|
|
12
|
-
reason.md
|
|
13
|
-
tier.md
|
|
14
|
-
specialty.md
|
|
15
|
-
tools-output.md
|
|
16
|
-
tools.md
|
|
17
|
-
stock/<name>/ # built-in classifier
|
|
18
|
-
manifest.json
|
|
19
|
-
custom/<name>/ # caller-defined classifier
|
|
7
|
+
_prompts/ # shared base markdown (base.md, reason.md, confidence.md)
|
|
8
|
+
<classifier_name>/
|
|
20
9
|
manifest.json
|
|
21
10
|
prompt.md
|
|
22
11
|
```
|
|
23
12
|
|
|
24
|
-
The
|
|
13
|
+
The loader skips any top-level directory whose name starts with `_` (those are shared assets, not classifiers).
|
|
25
14
|
|
|
26
|
-
##
|
|
15
|
+
## Fields
|
|
27
16
|
|
|
28
17
|
| Field | Required | Description |
|
|
29
18
|
|---|---|---|
|
|
30
|
-
| `kind` | yes | `"stock"` or `"custom"` |
|
|
31
19
|
| `name` | yes | Classifier id. Must match the directory name. |
|
|
32
20
|
| `version` | yes | Contract version surfaced in `meta.classifiers[name].version`. |
|
|
33
|
-
| `purpose` | yes | Human-readable description. |
|
|
34
|
-
| `
|
|
35
|
-
| `
|
|
21
|
+
| `purpose` | yes | Human-readable description of the classifier's job. Treated as a hard scope boundary in the prompt. |
|
|
22
|
+
| `dispatch_order` | no | Non-negative integer scheduling priority. Lower runs first. Omit to schedule this classifier last (treated as +Infinity). Duplicate names are rejected; duplicate dispatch_orders are allowed and schedule adjacent. |
|
|
23
|
+
| `applies_to` | no | One of `"user"`, `"assistant"`, `"both"`. Controls which pipeline pass the classifier participates in: `classify()` runs `"user"` + `"both"`; `inspect()` runs `"assistant"` + `"both"`. Defaults to `"user"`. |
|
|
24
|
+
| `reserved_fields` | no | Array of reserved field names this classifier may emit at the top level of its output. |
|
|
25
|
+
| `allowed_tools` | conditional | Required if `reserved_fields` includes `"tools"`; rejected otherwise. Array of `{ id, description }` listing the tool ids the classifier may pick from. |
|
|
26
|
+
| `output_schema` | no | JSON Schema (Ajv-validated). Describes only the custom (non-reserved) properties. The runtime composes this with canonical sub-schemas for any declared reserved fields plus `reason` and `certainty`. |
|
|
27
|
+
| `output_schema.examples` | no | Array of full example outputs (reserved + custom + `reason` + `certainty`). Validated against the composed schema at load time. Omit it and the runtime synthesizes a JSON skeleton example from the schema. |
|
|
28
|
+
| `fallback` | yes | Output emitted when the classifier errors or times out. Must validate against the composed schema; reserved fields are optional in fallback. |
|
|
36
29
|
| `backend.ollama.base_model` | no | Packaged Ollama model hint for this classifier. User config and function options take precedence. |
|
|
37
30
|
|
|
38
|
-
##
|
|
39
|
-
|
|
40
|
-
Stock manifests use a closed set of names (`preflight`, `routing`, `model_specialization`, `tools`, `prompt_injection`). The runtime knows each name's signal type, so there's no `emits` field. Fallbacks must satisfy the signal contract for that name (see [signals.md](signals.md)).
|
|
31
|
+
## Reserved fields
|
|
41
32
|
|
|
42
|
-
The
|
|
33
|
+
Reserved fields are well-known output keys the aggregator knows how to consume. The runtime owns their JSON Schema sub-schemas and prompt fragments — your manifest just opts in.
|
|
43
34
|
|
|
44
|
-
|
|
|
35
|
+
| Reserved field | Shape | What the aggregator does with it |
|
|
45
36
|
|---|---|---|
|
|
46
|
-
| `
|
|
37
|
+
| `final_reply` | `{ text: string ≤200 chars }` | Surfaced in `audit.final_reply`; caller can return as the terminal reply |
|
|
38
|
+
| `ack_reply` | `{ text: string ≤200 chars }` | Surfaced in `audit.ack_reply`; caller can show as an acknowledgement |
|
|
39
|
+
| `model_tier` | one of `DOWNSTREAM_MODEL_TIER_VALUES` | Soft constraint for catalog resolver |
|
|
40
|
+
| `model_specialization` | one of `MODEL_SPECIALIZATION_VALUES` | Soft constraint for catalog resolver |
|
|
41
|
+
| `tools` | array of allowed tool ids | Sets `downstream.tools` |
|
|
42
|
+
| `risk_level` | one of `PROMPT_INJECTION_RISK_LEVEL_VALUES` | Surfaced in `audit.prompt_injection` |
|
|
43
|
+
|
|
44
|
+
`final_reply` and `ack_reply` are mutually exclusive — a single output may contain at most one.
|
|
47
45
|
|
|
48
|
-
|
|
46
|
+
When multiple classifiers emit the same reserved field, the highest-certainty contributor wins. Ties are broken by manifest `dispatch_order` ascending (the first encountered in registry order keeps the slot). Classifiers without `dispatch_order` sort last for tie-break purposes too.
|
|
47
|
+
|
|
48
|
+
## Example: reserved-only manifest
|
|
49
49
|
|
|
50
50
|
```json
|
|
51
51
|
{
|
|
52
|
-
"
|
|
53
|
-
"name": "prompt_injection",
|
|
52
|
+
"name": "routing",
|
|
54
53
|
"version": "1.0.0",
|
|
55
|
-
"purpose": "
|
|
56
|
-
"
|
|
54
|
+
"purpose": "Recommend the downstream model tier.",
|
|
55
|
+
"dispatch_order": 20,
|
|
56
|
+
"reserved_fields": ["model_tier"],
|
|
57
|
+
"output_schema": {
|
|
58
|
+
"examples": [
|
|
59
|
+
{ "reason": "Simple factual question.", "certainty": "near_certain", "model_tier": "local_fast" },
|
|
60
|
+
{ "reason": "Multi-step refactor.", "certainty": "very_strong", "model_tier": "frontier_coding" }
|
|
61
|
+
]
|
|
62
|
+
},
|
|
57
63
|
"fallback": {
|
|
58
|
-
"reason": "Classifier failed;
|
|
59
|
-
"certainty": "no_signal"
|
|
60
|
-
"risk_level": "unknown"
|
|
64
|
+
"reason": "Classifier failed; no routing signal.",
|
|
65
|
+
"certainty": "no_signal"
|
|
61
66
|
}
|
|
62
67
|
}
|
|
63
68
|
```
|
|
64
69
|
|
|
65
|
-
|
|
70
|
+
The runtime injects the `model_tier` enum and the canonical prompt fragment automatically. Your `prompt.md` only needs to explain the classification rule.
|
|
66
71
|
|
|
67
|
-
|
|
68
|
-
|---|---|---|
|
|
69
|
-
| `output_schema` | yes | JSON Schema (Ajv-validated) for the `output` payload. |
|
|
70
|
-
|
|
71
|
-
Custom classifier names must not collide with any stock classifier name.
|
|
72
|
-
|
|
73
|
-
Example:
|
|
72
|
+
## Example: custom-only manifest
|
|
74
73
|
|
|
75
74
|
```json
|
|
76
75
|
{
|
|
77
|
-
"kind": "custom",
|
|
78
76
|
"name": "memory_retrieval_queries",
|
|
79
77
|
"version": "1.0.0",
|
|
80
|
-
"purpose": "Generate
|
|
81
|
-
"
|
|
82
|
-
"fallback": {
|
|
83
|
-
"reason": "Classifier failed; no memory queries generated.",
|
|
84
|
-
"certainty": "no_signal",
|
|
85
|
-
"output": { "queries": [] }
|
|
86
|
-
},
|
|
78
|
+
"purpose": "Generate retrieval queries likely to surface helpful user-specific context for the downstream model.",
|
|
79
|
+
"dispatch_order": 60,
|
|
87
80
|
"output_schema": {
|
|
88
|
-
"type": "object",
|
|
89
|
-
"additionalProperties": false,
|
|
90
81
|
"required": ["queries"],
|
|
91
82
|
"properties": {
|
|
92
83
|
"queries": {
|
|
93
84
|
"type": "array", "maxItems": 5,
|
|
94
|
-
"items": { "type": "string", "minLength": 1, "maxLength": 120 }
|
|
85
|
+
"items": { "type": "string", "minLength": 1, "maxLength": 120 },
|
|
86
|
+
"uniqueItems": true
|
|
95
87
|
}
|
|
96
|
-
}
|
|
88
|
+
},
|
|
89
|
+
"examples": [
|
|
90
|
+
{
|
|
91
|
+
"reason": "Saved code-review preferences could improve the response.",
|
|
92
|
+
"certainty": "strong",
|
|
93
|
+
"queries": ["user code review preferences"]
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"reason": "No saved memories likely to help.",
|
|
97
|
+
"certainty": "very_strong",
|
|
98
|
+
"queries": []
|
|
99
|
+
}
|
|
100
|
+
]
|
|
101
|
+
},
|
|
102
|
+
"fallback": {
|
|
103
|
+
"reason": "Classifier failed; no memory queries generated.",
|
|
104
|
+
"certainty": "no_signal",
|
|
105
|
+
"queries": []
|
|
97
106
|
}
|
|
98
107
|
}
|
|
99
108
|
```
|
|
100
109
|
|
|
101
|
-
##
|
|
110
|
+
## Example: hybrid manifest
|
|
111
|
+
|
|
112
|
+
A manifest may declare both reserved fields and custom properties; they sit alongside each other at the top level of every output.
|
|
102
113
|
|
|
103
|
-
|
|
114
|
+
```json
|
|
115
|
+
{
|
|
116
|
+
"name": "task_router",
|
|
117
|
+
"version": "1.0.0",
|
|
118
|
+
"purpose": "Pick the downstream tier and estimate token usage.",
|
|
119
|
+
"dispatch_order": 25,
|
|
120
|
+
"reserved_fields": ["model_tier", "model_specialization"],
|
|
121
|
+
"output_schema": {
|
|
122
|
+
"required": ["estimated_tokens"],
|
|
123
|
+
"properties": {
|
|
124
|
+
"estimated_tokens": { "type": "integer", "minimum": 0 }
|
|
125
|
+
},
|
|
126
|
+
"examples": [
|
|
127
|
+
{
|
|
128
|
+
"reason": "Code refactor needs reasoning.",
|
|
129
|
+
"certainty": "very_strong",
|
|
130
|
+
"model_tier": "frontier_strong",
|
|
131
|
+
"model_specialization": "coding",
|
|
132
|
+
"estimated_tokens": 12000
|
|
133
|
+
}
|
|
134
|
+
]
|
|
135
|
+
},
|
|
136
|
+
"fallback": {
|
|
137
|
+
"reason": "Classifier failed.",
|
|
138
|
+
"certainty": "no_signal",
|
|
139
|
+
"estimated_tokens": 0
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
104
143
|
|
|
105
|
-
|
|
144
|
+
## Prompt files
|
|
106
145
|
|
|
107
|
-
|
|
146
|
+
`prompt.md` is the classifier-specific instruction text. The runtime composes the system prompt at load time from:
|
|
108
147
|
|
|
109
|
-
-
|
|
110
|
-
|
|
111
|
-
-
|
|
112
|
-
|
|
148
|
+
1. Shared base sections (JSON-only contract, `reason` + `certainty` rules) from `src/classifiers/_prompts/`
|
|
149
|
+
2. The classifier header (name and purpose, with the purpose stated as a hard scope boundary)
|
|
150
|
+
3. Auto-injected fragments for each declared reserved field (canonical enum values included, so you can't drift)
|
|
151
|
+
4. Your `prompt.md`
|
|
152
|
+
5. A JSON example of a complete output: the `output_schema.examples` if you provided any, otherwise a synthesized skeleton derived from the schema
|
|
113
153
|
|
|
114
|
-
|
|
154
|
+
Keep `prompt.md` focused on classification behavior — when to emit each field, when to omit, when to abstain. Don't paste enum values for reserved fields; the runtime does that for you.
|
|
115
155
|
|
|
116
156
|
## Validation rejections
|
|
117
157
|
|
|
118
158
|
The loader rejects manifests that:
|
|
119
159
|
|
|
120
|
-
- declare unsupported fields
|
|
121
|
-
- collide on `name`
|
|
122
|
-
-
|
|
123
|
-
-
|
|
124
|
-
-
|
|
125
|
-
- have a `fallback` that doesn't
|
|
126
|
-
-
|
|
127
|
-
-
|
|
160
|
+
- declare unsupported fields at the manifest root
|
|
161
|
+
- collide with another classifier on `name`
|
|
162
|
+
- include a reserved field name in `output_schema.properties`
|
|
163
|
+
- include `reason` or `certainty` in `output_schema.properties`
|
|
164
|
+
- list `allowed_tools` without `tools` in `reserved_fields` (or vice versa)
|
|
165
|
+
- have a `fallback` that doesn't validate against the composed schema
|
|
166
|
+
- have an `output_schema.examples[]` entry that doesn't validate against the composed schema
|
|
167
|
+
- have an empty `prompt.md`
|
|
168
|
+
- have a `name` that doesn't match the parent directory
|