open-classify 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +145 -105
- package/dist/src/aggregator.d.ts +8 -17
- package/dist/src/aggregator.js +127 -218
- package/dist/src/classifiers/{custom/context_shift → context_shift}/manifest.json +6 -11
- package/dist/src/classifiers/{custom/context_shift → context_shift}/prompt.md +1 -1
- package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/manifest.json +7 -12
- package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/prompt.md +2 -2
- package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/manifest.json +6 -11
- package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/prompt.md +2 -2
- package/dist/src/classifiers/{stock/model_specialization → model_specialization}/manifest.json +2 -2
- package/dist/src/classifiers/model_specialization/prompt.md +5 -0
- package/dist/src/classifiers/model_tier/manifest.json +11 -0
- package/dist/src/classifiers/model_tier/prompt.md +5 -0
- package/dist/src/classifiers/preflight/manifest.json +35 -0
- package/dist/src/classifiers/preflight/prompt.md +16 -0
- package/dist/src/classifiers/prompt_injection/manifest.json +15 -0
- package/dist/src/classifiers/prompt_injection/prompt.md +14 -0
- package/dist/src/classifiers/{stock/tools → tools}/manifest.json +3 -3
- package/dist/src/classifiers/tools/prompt.md +5 -0
- package/dist/src/classifiers.js +31 -29
- package/dist/src/classify.d.ts +9 -3
- package/dist/src/classify.js +26 -14
- package/dist/src/config.d.ts +1 -6
- package/dist/src/config.js +7 -57
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +3 -3
- package/dist/src/input.d.ts +4 -1
- package/dist/src/input.js +12 -10
- package/dist/src/manifest.d.ts +29 -70
- package/dist/src/pipeline.d.ts +9 -2
- package/dist/src/pipeline.js +42 -83
- package/dist/src/reserved-fields.d.ts +18 -0
- package/dist/src/reserved-fields.js +175 -0
- package/dist/src/stock-prompt.d.ts +9 -2
- package/dist/src/stock-prompt.js +165 -45
- package/dist/src/stock-validation.d.ts +16 -17
- package/dist/src/stock-validation.js +267 -236
- package/dist/src/stock.d.ts +24 -60
- package/dist/src/stock.js +7 -14
- package/docs/adding-a-classifier.md +76 -32
- package/docs/manifests.md +113 -72
- package/docs/resolver.md +23 -56
- package/docs/signals.md +48 -57
- package/open-classify.config.example.json +9 -14
- package/package.json +1 -1
- package/dist/src/classifiers/stock/preflight/manifest.json +0 -11
- package/dist/src/classifiers/stock/prompt_injection/manifest.json +0 -12
- package/dist/src/classifiers/stock/prompts/classifier-header.md +0 -4
- package/dist/src/classifiers/stock/prompts/custom-output.md +0 -7
- package/dist/src/classifiers/stock/prompts/model_specialization.md +0 -7
- package/dist/src/classifiers/stock/prompts/preflight-output.md +0 -10
- package/dist/src/classifiers/stock/prompts/preflight.md +0 -47
- package/dist/src/classifiers/stock/prompts/prompt-injection-output.md +0 -5
- package/dist/src/classifiers/stock/prompts/prompt_injection.md +0 -24
- package/dist/src/classifiers/stock/prompts/routing-output.md +0 -5
- package/dist/src/classifiers/stock/prompts/routing.md +0 -9
- package/dist/src/classifiers/stock/prompts/specialty.md +0 -12
- package/dist/src/classifiers/stock/prompts/tier.md +0 -7
- package/dist/src/classifiers/stock/prompts/tools-output.md +0 -11
- package/dist/src/classifiers/stock/prompts/tools.md +0 -10
- package/dist/src/classifiers/stock/routing/manifest.json +0 -11
- /package/dist/src/classifiers/{stock/prompts → _prompts}/base.md +0 -0
- /package/dist/src/classifiers/{stock/prompts → _prompts}/confidence.md +0 -0
- /package/dist/src/classifiers/{stock/prompts → _prompts}/reason.md +0 -0
package/dist/src/aggregator.js
CHANGED
|
@@ -1,245 +1,166 @@
|
|
|
1
|
-
import { certaintyScore
|
|
2
|
-
export
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
const
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
:
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
map[manifest.name] = result;
|
|
48
|
-
}
|
|
1
|
+
import { certaintyScore } from "./stock.js";
|
|
2
|
+
export function assembleResult(args) {
|
|
3
|
+
const { registry, results, failedClassifiers, catalog } = args;
|
|
4
|
+
// Pick reserved fields — highest certainty wins, no threshold gate.
|
|
5
|
+
const finalReply = pickField(registry, results, "final_reply");
|
|
6
|
+
const ackReply = pickField(registry, results, "ack_reply");
|
|
7
|
+
const modelTier = pickField(registry, results, "model_tier");
|
|
8
|
+
const modelSpec = pickField(registry, results, "model_specialization");
|
|
9
|
+
const toolsPick = pickField(registry, results, "tools");
|
|
10
|
+
const riskLevel = pickField(registry, results, "risk_level");
|
|
11
|
+
// Resolve concrete model id.
|
|
12
|
+
let model_id = null;
|
|
13
|
+
try {
|
|
14
|
+
const routing = mergeRouting(modelTier?.value, modelSpec?.value);
|
|
15
|
+
model_id = resolveModelFromRouting(routing, catalog).id;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
// Catalog error — model_id stays null.
|
|
19
|
+
}
|
|
20
|
+
const tools = toolsPick?.value ?? [];
|
|
21
|
+
const reply = finalReply?.value
|
|
22
|
+
? { text: finalReply.value.text }
|
|
23
|
+
: ackReply?.value
|
|
24
|
+
? { text: ackReply.value.text }
|
|
25
|
+
: null;
|
|
26
|
+
const prompt_injection = riskLevel?.value !== undefined ? { risk_level: riskLevel.value } : null;
|
|
27
|
+
const { avg_certainty, min_certainty } = certaintySummary(registry, results);
|
|
28
|
+
const classifier_outputs = buildPublicOutputs(registry, results);
|
|
29
|
+
// Determine action. Priority: prompt_injection > classification_error > reply > route.
|
|
30
|
+
const isInjectionBlock = riskLevel?.value === "high_risk" || riskLevel?.value === "unknown";
|
|
31
|
+
const isClassificationError = failedClassifiers.length > 0 || reply === null || model_id === null;
|
|
32
|
+
let action;
|
|
33
|
+
let block_reason;
|
|
34
|
+
if (isInjectionBlock) {
|
|
35
|
+
action = "block";
|
|
36
|
+
block_reason = "prompt_injection";
|
|
37
|
+
}
|
|
38
|
+
else if (isClassificationError) {
|
|
39
|
+
action = "block";
|
|
40
|
+
block_reason = "classification_error";
|
|
41
|
+
}
|
|
42
|
+
else if (finalReply?.value !== undefined) {
|
|
43
|
+
action = "reply";
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
action = "route";
|
|
49
47
|
}
|
|
50
|
-
return map;
|
|
51
|
-
}
|
|
52
|
-
function isConfident(result, threshold) {
|
|
53
|
-
if (!result)
|
|
54
|
-
return false;
|
|
55
|
-
return scoreCertainty(result.certainty) >= threshold;
|
|
56
|
-
}
|
|
57
|
-
function mergeRouting(routing, modelSpec, threshold) {
|
|
58
|
-
const tier = pickConfidentAxis([
|
|
59
|
-
["routing", routing, routing?.model_tier],
|
|
60
|
-
], threshold);
|
|
61
|
-
const specialization = pickConfidentAxis([
|
|
62
|
-
["model_specialization", modelSpec, modelSpec?.specialization],
|
|
63
|
-
], threshold);
|
|
64
|
-
if (tier === undefined && specialization === undefined)
|
|
65
|
-
return undefined;
|
|
66
48
|
return {
|
|
67
|
-
|
|
68
|
-
...(
|
|
49
|
+
action,
|
|
50
|
+
...(block_reason !== undefined ? { block_reason } : {}),
|
|
51
|
+
model_id,
|
|
52
|
+
tools,
|
|
53
|
+
reply,
|
|
54
|
+
prompt_injection,
|
|
55
|
+
avg_certainty,
|
|
56
|
+
min_certainty,
|
|
57
|
+
failed_classifiers: failedClassifiers,
|
|
58
|
+
classifier_outputs,
|
|
69
59
|
};
|
|
70
60
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
continue;
|
|
76
|
-
if (!isConfident(source, threshold))
|
|
77
|
-
continue;
|
|
78
|
-
const confidence = scoreCertainty(source.certainty);
|
|
79
|
-
if (best === undefined || confidence > best.confidence) {
|
|
80
|
-
best = { value, confidence };
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
return best?.value;
|
|
84
|
-
}
|
|
85
|
-
function routingMaxConfidence(routing, modelSpec) {
|
|
86
|
-
const values = [routing?.certainty, modelSpec?.certainty]
|
|
87
|
-
.filter((v) => v !== undefined)
|
|
88
|
-
.map(scoreCertainty);
|
|
89
|
-
if (values.length === 0)
|
|
90
|
-
return undefined;
|
|
91
|
-
return Math.max(...values);
|
|
92
|
-
}
|
|
93
|
-
function extractToolsSignal(result) {
|
|
94
|
-
return { tools: result.tools };
|
|
95
|
-
}
|
|
96
|
-
function extractPromptInjectionSignal(result) {
|
|
97
|
-
return {
|
|
98
|
-
risk_level: result.risk_level,
|
|
99
|
-
};
|
|
100
|
-
}
|
|
101
|
-
function customOutputs(registry, results) {
|
|
102
|
-
const out = [];
|
|
61
|
+
// Build the public classifier_outputs map. Keeps reason + payload fields;
|
|
62
|
+
// converts certainty label to float score.
|
|
63
|
+
export function buildPublicOutputs(registry, results) {
|
|
64
|
+
const out = {};
|
|
103
65
|
for (const manifest of registry) {
|
|
104
|
-
if (!isCustomManifest(manifest))
|
|
105
|
-
continue;
|
|
106
66
|
const result = results[manifest.name];
|
|
107
67
|
if (result === undefined)
|
|
108
68
|
continue;
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
certainty:
|
|
113
|
-
|
|
114
|
-
});
|
|
69
|
+
const { certainty, ...rest } = result;
|
|
70
|
+
out[manifest.name] = {
|
|
71
|
+
...rest,
|
|
72
|
+
certainty: scoreCertainty(certainty),
|
|
73
|
+
};
|
|
115
74
|
}
|
|
116
75
|
return out;
|
|
117
76
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
77
|
+
function certaintySummary(registry, results) {
|
|
78
|
+
const scores = registry.map((m) => scoreCertainty(results[m.name]?.certainty));
|
|
79
|
+
if (scores.length === 0)
|
|
80
|
+
return { avg_certainty: 0, min_certainty: 0 };
|
|
81
|
+
const min_certainty = Math.min(...scores);
|
|
82
|
+
const avg_certainty = scores.reduce((sum, v) => sum + v, 0) / scores.length;
|
|
83
|
+
return { min_certainty, avg_certainty };
|
|
84
|
+
}
|
|
85
|
+
// Highest certainty wins; ties broken by registry order (already sorted by
|
|
86
|
+
// dispatch_order ascending).
|
|
87
|
+
function pickField(registry, results, field) {
|
|
88
|
+
let best;
|
|
89
|
+
for (const manifest of registry) {
|
|
90
|
+
if (!manifest.reservedFields.includes(field))
|
|
91
|
+
continue;
|
|
92
|
+
const output = results[manifest.name];
|
|
93
|
+
if (output === undefined)
|
|
94
|
+
continue;
|
|
95
|
+
const raw = output[field];
|
|
96
|
+
if (raw === undefined)
|
|
97
|
+
continue;
|
|
98
|
+
const score = scoreCertainty(output.certainty);
|
|
99
|
+
if (best === undefined || score > best.score) {
|
|
100
|
+
best = { value: raw, source: manifest.name, score };
|
|
131
101
|
}
|
|
132
102
|
}
|
|
133
|
-
return
|
|
134
|
-
}
|
|
135
|
-
function hasLowConfidenceAxis(result, field, threshold) {
|
|
136
|
-
if (!result)
|
|
137
|
-
return false;
|
|
138
|
-
if (result[field] === undefined)
|
|
139
|
-
return false;
|
|
140
|
-
return scoreCertainty(result.certainty) < threshold;
|
|
103
|
+
return best;
|
|
141
104
|
}
|
|
142
105
|
function scoreCertainty(certainty) {
|
|
143
106
|
return certainty === undefined ? 0 : certaintyScore[certainty];
|
|
144
107
|
}
|
|
145
|
-
|
|
108
|
+
// ─── Model resolution ────────────────────────────────────────────────────────
|
|
109
|
+
function mergeRouting(tier, specialization) {
|
|
110
|
+
if (tier === undefined && specialization === undefined)
|
|
111
|
+
return undefined;
|
|
112
|
+
return {
|
|
113
|
+
...(tier === undefined ? {} : { model_tier: tier }),
|
|
114
|
+
...(specialization === undefined ? {} : { model_specialization: specialization }),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
function resolveModelFromRouting(routing, catalog) {
|
|
146
118
|
const requested = {};
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
119
|
+
if (routing?.model_specialization !== undefined) {
|
|
120
|
+
requested.model_specialization = routing.model_specialization;
|
|
121
|
+
}
|
|
122
|
+
if (routing?.model_tier !== undefined) {
|
|
123
|
+
requested.model_tier = routing.model_tier;
|
|
150
124
|
}
|
|
151
|
-
if (routing?.specialization !== undefined)
|
|
152
|
-
requested.specialization = routing.specialization;
|
|
153
|
-
if (routing?.model_tier !== undefined)
|
|
154
|
-
requested.tier = routing.model_tier;
|
|
155
125
|
const passes = [
|
|
156
|
-
{
|
|
157
|
-
{
|
|
158
|
-
{
|
|
159
|
-
{
|
|
126
|
+
{ useSpec: true, useTier: true },
|
|
127
|
+
{ useSpec: true, useTier: false },
|
|
128
|
+
{ useSpec: false, useTier: true },
|
|
129
|
+
{ useSpec: false, useTier: false },
|
|
160
130
|
];
|
|
161
131
|
for (const pass of passes) {
|
|
162
|
-
const
|
|
163
|
-
const matching = catalog.models.filter((
|
|
132
|
+
const constraints = constraintsForPass(requested, pass);
|
|
133
|
+
const matching = catalog.models.filter((m) => matchesConstraints(m, constraints));
|
|
164
134
|
if (matching.length === 0)
|
|
165
135
|
continue;
|
|
166
|
-
|
|
167
|
-
return {
|
|
168
|
-
...modelRecommendationFields(winner),
|
|
169
|
-
resolution: {
|
|
170
|
-
constraints_used,
|
|
171
|
-
constraints_dropped: [
|
|
172
|
-
...ignoredConstraints,
|
|
173
|
-
...relaxedConstraints(requested, constraints_used),
|
|
174
|
-
],
|
|
175
|
-
confidences,
|
|
176
|
-
fell_back_to_default: false,
|
|
177
|
-
},
|
|
178
|
-
};
|
|
136
|
+
return { id: pickBestModel(matching, catalog.models).id };
|
|
179
137
|
}
|
|
180
|
-
const fallback = catalog.models.find((
|
|
138
|
+
const fallback = catalog.models.find((m) => m.id === catalog.default);
|
|
181
139
|
if (!fallback) {
|
|
182
|
-
throw new Error(`catalog default "${catalog.default}" not found in models
|
|
140
|
+
throw new Error(`catalog default "${catalog.default}" not found in models`);
|
|
183
141
|
}
|
|
184
|
-
return {
|
|
185
|
-
...modelRecommendationFields(fallback),
|
|
186
|
-
resolution: {
|
|
187
|
-
constraints_used: {},
|
|
188
|
-
constraints_dropped: [
|
|
189
|
-
...ignoredConstraints,
|
|
190
|
-
...defaultFallbackConstraints(requested),
|
|
191
|
-
],
|
|
192
|
-
confidences,
|
|
193
|
-
fell_back_to_default: true,
|
|
194
|
-
},
|
|
195
|
-
};
|
|
196
|
-
}
|
|
197
|
-
// Test-friendly convenience wrapper: builds a routing signal from a typed
|
|
198
|
-
// results map and resolves a model. Mirrors `composeEnvelope` for callers
|
|
199
|
-
// that want just the model recommendation without the rest of the envelope.
|
|
200
|
-
export function resolveModel(results, catalog, threshold) {
|
|
201
|
-
const routing = mergeRouting(results.routing, results.model_specialization, threshold);
|
|
202
|
-
return resolveModelFromRouting(routing, catalog, routingMaxConfidence(results.routing, results.model_specialization), lowConfidenceRoutingDrops(results.routing, results.model_specialization, routing, threshold));
|
|
142
|
+
return { id: fallback.id };
|
|
203
143
|
}
|
|
204
144
|
function constraintsForPass(requested, pass) {
|
|
205
145
|
return {
|
|
206
|
-
...(pass.
|
|
207
|
-
? {
|
|
146
|
+
...(pass.useSpec && requested.model_specialization !== undefined
|
|
147
|
+
? { model_specialization: requested.model_specialization }
|
|
148
|
+
: {}),
|
|
149
|
+
...(pass.useTier && requested.model_tier !== undefined
|
|
150
|
+
? { model_tier: requested.model_tier }
|
|
208
151
|
: {}),
|
|
209
|
-
...(pass.useTier && requested.tier !== undefined ? { tier: requested.tier } : {}),
|
|
210
152
|
};
|
|
211
153
|
}
|
|
212
154
|
function matchesConstraints(model, constraints) {
|
|
213
|
-
return ((constraints.
|
|
214
|
-
model.specializations.includes(constraints.
|
|
215
|
-
(constraints.
|
|
216
|
-
}
|
|
217
|
-
function relaxedConstraints(requested, used) {
|
|
218
|
-
const dropped = [];
|
|
219
|
-
if (requested.specialization !== undefined && used.specialization === undefined) {
|
|
220
|
-
dropped.push({ axis: "specialization", reason: "no_match_relaxed" });
|
|
221
|
-
}
|
|
222
|
-
if (requested.tier !== undefined && used.tier === undefined) {
|
|
223
|
-
dropped.push({ axis: "tier", reason: "no_match_relaxed" });
|
|
224
|
-
}
|
|
225
|
-
return dropped;
|
|
226
|
-
}
|
|
227
|
-
function defaultFallbackConstraints(requested) {
|
|
228
|
-
const dropped = [];
|
|
229
|
-
if (requested.specialization !== undefined) {
|
|
230
|
-
dropped.push({ axis: "specialization", reason: "default_fallback" });
|
|
231
|
-
}
|
|
232
|
-
if (requested.tier !== undefined) {
|
|
233
|
-
dropped.push({ axis: "tier", reason: "default_fallback" });
|
|
234
|
-
}
|
|
235
|
-
return dropped;
|
|
155
|
+
return ((constraints.model_specialization === undefined ||
|
|
156
|
+
model.specializations.includes(constraints.model_specialization)) &&
|
|
157
|
+
(constraints.model_tier === undefined || model.tier === constraints.model_tier));
|
|
236
158
|
}
|
|
237
159
|
function pickBestModel(candidates, catalogOrder) {
|
|
238
160
|
let winner = candidates[0];
|
|
239
161
|
for (let i = 1; i < candidates.length; i++) {
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
winner = candidate;
|
|
162
|
+
if (compareModels(candidates[i], winner, catalogOrder) < 0) {
|
|
163
|
+
winner = candidates[i];
|
|
243
164
|
}
|
|
244
165
|
}
|
|
245
166
|
return winner;
|
|
@@ -251,27 +172,15 @@ function compareModels(a, b, catalogOrder) {
|
|
|
251
172
|
if (a.params_in_billions !== b.params_in_billions) {
|
|
252
173
|
return comparableParams(b) - comparableParams(a);
|
|
253
174
|
}
|
|
254
|
-
if (a.context_window !== b.context_window)
|
|
175
|
+
if (a.context_window !== b.context_window)
|
|
255
176
|
return b.context_window - a.context_window;
|
|
256
|
-
}
|
|
257
177
|
return catalogOrder.indexOf(a) - catalogOrder.indexOf(b);
|
|
258
178
|
}
|
|
259
179
|
function priceIndex(model) {
|
|
260
|
-
if (model.input_tokens_cpm === undefined || model.output_tokens_cpm === undefined)
|
|
180
|
+
if (model.input_tokens_cpm === undefined || model.output_tokens_cpm === undefined)
|
|
261
181
|
return 0;
|
|
262
|
-
}
|
|
263
182
|
return model.input_tokens_cpm + model.output_tokens_cpm;
|
|
264
183
|
}
|
|
265
184
|
function comparableParams(model) {
|
|
266
185
|
return model.params_in_billions ?? 0;
|
|
267
186
|
}
|
|
268
|
-
function modelRecommendationFields(winner) {
|
|
269
|
-
return {
|
|
270
|
-
id: winner.id,
|
|
271
|
-
params_in_billions: winner.params_in_billions,
|
|
272
|
-
context_window: winner.context_window,
|
|
273
|
-
...(winner.input_tokens_cpm === undefined ? {} : { input_tokens_cpm: winner.input_tokens_cpm }),
|
|
274
|
-
...(winner.cached_tokens_cpm === undefined ? {} : { cached_tokens_cpm: winner.cached_tokens_cpm }),
|
|
275
|
-
...(winner.output_tokens_cpm === undefined ? {} : { output_tokens_cpm: winner.output_tokens_cpm }),
|
|
276
|
-
};
|
|
277
|
-
}
|
|
@@ -1,19 +1,9 @@
|
|
|
1
1
|
{
|
|
2
|
-
"kind": "custom",
|
|
3
2
|
"name": "context_shift",
|
|
4
3
|
"version": "1.0.0",
|
|
5
4
|
"purpose": "Classify whether the latest message continues, branches from, returns to, or starts a conversation thread.",
|
|
6
|
-
"
|
|
7
|
-
"fallback": {
|
|
8
|
-
"reason": "Classifier failed; context relationship is ambiguous.",
|
|
9
|
-
"certainty": "no_signal",
|
|
10
|
-
"output": {
|
|
11
|
-
"decision": "ambiguous"
|
|
12
|
-
}
|
|
13
|
-
},
|
|
5
|
+
"dispatch_order": 80,
|
|
14
6
|
"output_schema": {
|
|
15
|
-
"type": "object",
|
|
16
|
-
"additionalProperties": false,
|
|
17
7
|
"required": ["decision"],
|
|
18
8
|
"properties": {
|
|
19
9
|
"decision": {
|
|
@@ -27,5 +17,10 @@
|
|
|
27
17
|
]
|
|
28
18
|
}
|
|
29
19
|
}
|
|
20
|
+
},
|
|
21
|
+
"fallback": {
|
|
22
|
+
"reason": "Classifier failed; context relationship is ambiguous.",
|
|
23
|
+
"certainty": "no_signal",
|
|
24
|
+
"decision": "ambiguous"
|
|
30
25
|
}
|
|
31
26
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
You are the context_shift classifier for an AI assistant routing system.
|
|
2
2
|
|
|
3
|
-
`
|
|
3
|
+
`decision` describes how the final user message relates to the visible conversation history.
|
|
4
4
|
|
|
5
5
|
Use `same_active_thread` when the final message directly continues, clarifies, corrects, or asks for the next step on the active topic.
|
|
6
6
|
Use `related_branch` when it starts a distinct subtask or angle that still depends on the active topic.
|
package/dist/src/classifiers/{custom/conversation_digest → conversation_digest}/manifest.json
RENAMED
|
@@ -1,20 +1,9 @@
|
|
|
1
1
|
{
|
|
2
|
-
"kind": "custom",
|
|
3
2
|
"name": "conversation_digest",
|
|
4
3
|
"version": "1.0.0",
|
|
5
4
|
"purpose": "Compress prior conversation history and the latest user message into separate summaries.",
|
|
6
|
-
"
|
|
7
|
-
"fallback": {
|
|
8
|
-
"reason": "Classifier failed; no conversation summary generated.",
|
|
9
|
-
"certainty": "no_signal",
|
|
10
|
-
"output": {
|
|
11
|
-
"history_summary": "",
|
|
12
|
-
"latest_user_message_summary": ""
|
|
13
|
-
}
|
|
14
|
-
},
|
|
5
|
+
"dispatch_order": 70,
|
|
15
6
|
"output_schema": {
|
|
16
|
-
"type": "object",
|
|
17
|
-
"additionalProperties": false,
|
|
18
7
|
"required": ["history_summary", "latest_user_message_summary"],
|
|
19
8
|
"properties": {
|
|
20
9
|
"history_summary": {
|
|
@@ -26,5 +15,11 @@
|
|
|
26
15
|
"maxLength": 1000
|
|
27
16
|
}
|
|
28
17
|
}
|
|
18
|
+
},
|
|
19
|
+
"fallback": {
|
|
20
|
+
"reason": "Classifier failed; no conversation summary generated.",
|
|
21
|
+
"certainty": "no_signal",
|
|
22
|
+
"history_summary": "",
|
|
23
|
+
"latest_user_message_summary": ""
|
|
29
24
|
}
|
|
30
25
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
You are the conversation_digest classifier for an AI assistant routing system.
|
|
2
2
|
|
|
3
|
-
`
|
|
4
|
-
`
|
|
3
|
+
`history_summary` is a maximally compressed summary of every message before the final user message.
|
|
4
|
+
`latest_user_message_summary` is a maximally compressed summary of only the final user message.
|
|
5
5
|
|
|
6
6
|
Use terse, information-dense wording. Preserve concrete goals, constraints, decisions, file paths, identifiers, and unresolved asks. Omit pleasantries and low-value filler.
|
|
7
7
|
If there is no prior conversation history, return an empty string for `history_summary`.
|
|
@@ -1,19 +1,9 @@
|
|
|
1
1
|
{
|
|
2
|
-
"kind": "custom",
|
|
3
2
|
"name": "memory_retrieval_queries",
|
|
4
3
|
"version": "1.0.0",
|
|
5
4
|
"purpose": "Generate retrieval queries likely to surface helpful user-specific context for the downstream model.",
|
|
6
|
-
"
|
|
7
|
-
"fallback": {
|
|
8
|
-
"reason": "Classifier failed; no memory queries generated.",
|
|
9
|
-
"certainty": "no_signal",
|
|
10
|
-
"output": {
|
|
11
|
-
"queries": []
|
|
12
|
-
}
|
|
13
|
-
},
|
|
5
|
+
"dispatch_order": 60,
|
|
14
6
|
"output_schema": {
|
|
15
|
-
"type": "object",
|
|
16
|
-
"additionalProperties": false,
|
|
17
7
|
"required": ["queries"],
|
|
18
8
|
"properties": {
|
|
19
9
|
"queries": {
|
|
@@ -27,5 +17,10 @@
|
|
|
27
17
|
"uniqueItems": true
|
|
28
18
|
}
|
|
29
19
|
}
|
|
20
|
+
},
|
|
21
|
+
"fallback": {
|
|
22
|
+
"reason": "Classifier failed; no memory queries generated.",
|
|
23
|
+
"certainty": "no_signal",
|
|
24
|
+
"queries": []
|
|
30
25
|
}
|
|
31
26
|
}
|
package/dist/src/classifiers/{custom/memory_retrieval_queries → memory_retrieval_queries}/prompt.md
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
You are the memory_retrieval_queries classifier for an AI assistant routing system.
|
|
2
2
|
|
|
3
|
-
`
|
|
4
|
-
Return an empty queries array when saved memories are unlikely to improve the downstream answer.
|
|
3
|
+
`queries` is an array of short search strings the caller may use against its own memory store.
|
|
4
|
+
Return an empty `queries` array when saved memories are unlikely to improve the downstream answer.
|
|
5
5
|
Do not invent known facts about the user; only produce retrieval queries grounded in likely missing user context.
|
package/dist/src/classifiers/{stock/model_specialization → model_specialization}/manifest.json
RENAMED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
|
-
"kind": "stock",
|
|
3
2
|
"name": "model_specialization",
|
|
4
3
|
"version": "1.0.0",
|
|
5
4
|
"purpose": "Choose the most accurate model specialty for serving the target message well.",
|
|
6
|
-
"
|
|
5
|
+
"dispatch_order": 30,
|
|
6
|
+
"reserved_fields": ["model_specialization"],
|
|
7
7
|
"fallback": {
|
|
8
8
|
"reason": "Classifier failed; no specialization signal.",
|
|
9
9
|
"certainty": "no_signal"
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
You are the model specialization classifier for an AI assistant routing system.
|
|
2
|
+
|
|
3
|
+
Pick the prompt/model specialization that best fits the target user message. Emit only `model_specialization`; do not infer tier, tools, or prompt-injection risk — other classifiers own those axes.
|
|
4
|
+
|
|
5
|
+
Omit `model_specialization` when you cannot pick with reasonable certainty.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "model_tier",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"purpose": "Recommend the downstream model tier.",
|
|
5
|
+
"dispatch_order": 20,
|
|
6
|
+
"reserved_fields": ["model_tier"],
|
|
7
|
+
"fallback": {
|
|
8
|
+
"reason": "Classifier failed; no model tier signal.",
|
|
9
|
+
"certainty": "no_signal"
|
|
10
|
+
}
|
|
11
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
You are the model_tier classifier for an AI assistant routing system.
|
|
2
|
+
|
|
3
|
+
Pick the coarse model tier that best fits the target user message. Emit only `model_tier`; do not infer specialization, tools, or prompt-injection risk — other classifiers own those axes.
|
|
4
|
+
|
|
5
|
+
Prefer the weakest tier that should still succeed. Omit `model_tier` rather than guessing when the right tier is not clear.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "preflight",
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"purpose": "Assess whether the latest message can be answered immediately (final_reply) or should route downstream with an acknowledgement (ack_reply). Always emits exactly one.",
|
|
5
|
+
"dispatch_order": 10,
|
|
6
|
+
"reserved_fields": ["final_reply", "ack_reply"],
|
|
7
|
+
"output_schema": {
|
|
8
|
+
"examples": [
|
|
9
|
+
{
|
|
10
|
+
"reason": "Simple greeting — answerable directly.",
|
|
11
|
+
"certainty": "near_certain",
|
|
12
|
+
"final_reply": { "text": "Hi!" }
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"reason": "Trivial arithmetic — answerable directly.",
|
|
16
|
+
"certainty": "very_strong",
|
|
17
|
+
"final_reply": { "text": "4" }
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"reason": "Code review task requires substantive downstream work.",
|
|
21
|
+
"certainty": "very_strong",
|
|
22
|
+
"ack_reply": { "text": "On it — reviewing the code now." }
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"reason": "Reminder request requires downstream action.",
|
|
26
|
+
"certainty": "strong",
|
|
27
|
+
"ack_reply": { "text": "Got it, I'll set that reminder for 3pm." }
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
},
|
|
31
|
+
"fallback": {
|
|
32
|
+
"reason": "Classifier failed; no preflight signal.",
|
|
33
|
+
"certainty": "no_signal"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
You are the preflight classifier for an AI assistant routing system.
|
|
2
|
+
|
|
3
|
+
Your primary task is to assess: **can you fully answer the target message yourself**, given the conversation history? Make this judgment first — the reply text follows from it.
|
|
4
|
+
|
|
5
|
+
**Step 1 — assess whether you can fully answer:**
|
|
6
|
+
Ask yourself: Is the intent clear? Is the answer fully derivable from context right now, without real-time data, external tools, code execution, non-trivial generation, analysis, or judgment? Would a one-sentence reply genuinely resolve the request?
|
|
7
|
+
|
|
8
|
+
If yes → emit `final_reply` with the complete answer.
|
|
9
|
+
|
|
10
|
+
If no (the downstream model should handle it) → emit `ack_reply` with a brief, contextually specific acknowledgement that shows you understood the request. The ack must reflect the actual request — not a generic "On it." — so the user knows their message was understood while the model works.
|
|
11
|
+
|
|
12
|
+
**Rule: always emit exactly one of `final_reply` or `ack_reply`. Never emit both. Never emit neither.**
|
|
13
|
+
|
|
14
|
+
- `final_reply` is for tiny terminal answers only: greetings, thanks, spelling lookups, simple arithmetic, yes/no factual questions answerable from context. If answering requires drafting, rewriting, analysis, coding, research, planning, or any substantive generation — use `ack_reply` instead.
|
|
15
|
+
- `ack_reply` text must not contain the answer. It acknowledges the request and confirms it is being worked on.
|
|
16
|
+
- Do not address the user anywhere except inside `final_reply.text` or `ack_reply.text`.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "prompt_injection",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"purpose": "Assess whether the target message contains prompt-injection attempts.",
|
|
5
|
+
"dispatch_order": 50,
|
|
6
|
+
"applies_to": "both",
|
|
7
|
+
"reserved_fields": ["risk_level"],
|
|
8
|
+
"output_schema": {
|
|
9
|
+
"required": ["risk_level"]
|
|
10
|
+
},
|
|
11
|
+
"fallback": {
|
|
12
|
+
"reason": "Classifier failed; prompt-injection risk could not be assessed.",
|
|
13
|
+
"certainty": "no_signal"
|
|
14
|
+
}
|
|
15
|
+
}
|