open-classify 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +54 -35
  2. package/dist/src/aggregator.d.ts +4 -1
  3. package/dist/src/aggregator.js +25 -15
  4. package/dist/src/classifiers/custom/context_shift/manifest.json +31 -0
  5. package/dist/src/classifiers/custom/context_shift/prompt.md +12 -0
  6. package/dist/src/classifiers/custom/{conversation_diegest → conversation_digest}/manifest.json +3 -1
  7. package/dist/src/classifiers/custom/{conversation_diegest → conversation_digest}/prompt.md +1 -1
  8. package/dist/src/classifiers/custom/memory_retrieval_queries/manifest.json +2 -0
  9. package/dist/src/classifiers/stock/model_specialization/manifest.json +4 -1
  10. package/dist/src/classifiers/stock/preflight/manifest.json +4 -1
  11. package/dist/src/classifiers/stock/prompt_injection/manifest.json +12 -0
  12. package/dist/src/classifiers/stock/prompts/confidence.md +3 -3
  13. package/dist/src/classifiers/stock/prompts/custom-output.md +7 -1
  14. package/dist/src/classifiers/stock/prompts/preflight.md +7 -7
  15. package/dist/src/classifiers/stock/prompts/prompt-injection-output.md +5 -0
  16. package/dist/src/classifiers/stock/prompts/prompt_injection.md +24 -0
  17. package/dist/src/classifiers/stock/prompts/reason.md +1 -1
  18. package/dist/src/classifiers/stock/prompts/specialty.md +8 -6
  19. package/dist/src/classifiers/stock/prompts/tier.md +1 -1
  20. package/dist/src/classifiers/stock/prompts/tools-output.md +4 -0
  21. package/dist/src/classifiers/stock/routing/manifest.json +4 -1
  22. package/dist/src/classifiers/stock/tools/manifest.json +2 -0
  23. package/dist/src/classify.d.ts +22 -0
  24. package/dist/src/classify.js +50 -0
  25. package/dist/src/config.d.ts +2 -0
  26. package/dist/src/config.js +33 -1
  27. package/dist/src/enums.d.ts +3 -7
  28. package/dist/src/enums.js +7 -30
  29. package/dist/src/index.d.ts +1 -0
  30. package/dist/src/index.js +2 -1
  31. package/dist/src/input.js +1 -1
  32. package/dist/src/manifest.d.ts +31 -23
  33. package/dist/src/manifest.js +5 -1
  34. package/dist/src/ollama.d.ts +0 -11
  35. package/dist/src/ollama.js +0 -36
  36. package/dist/src/pipeline.d.ts +1 -0
  37. package/dist/src/pipeline.js +78 -48
  38. package/dist/src/stock-prompt.js +1 -1
  39. package/dist/src/stock-validation.d.ts +1 -2
  40. package/dist/src/stock-validation.js +23 -40
  41. package/dist/src/stock.d.ts +12 -11
  42. package/dist/src/stock.js +21 -1
  43. package/dist/src/ui-server.js +12 -5
  44. package/dist/src/validation.d.ts +0 -1
  45. package/dist/src/validation.js +0 -37
  46. package/docs/adding-a-classifier.md +132 -0
  47. package/docs/manifests.md +127 -0
  48. package/docs/resolver.md +104 -0
  49. package/docs/signals.md +102 -0
  50. package/downstream-models.json +124 -0
  51. package/open-classify.config.example.json +5 -1
  52. package/package.json +3 -1
  53. package/dist/src/classifiers/stock/prompts/security-output.md +0 -8
  54. package/dist/src/classifiers/stock/prompts/security.md +0 -26
  55. package/dist/src/classifiers/stock/security/manifest.json +0 -12
@@ -0,0 +1,22 @@
1
+ import { type RunClassifier } from "./classifiers.js";
2
+ import { type OpenClassifyConfig } from "./config.js";
3
+ import type { AggregatorConfig, Catalog, PipelineResult } from "./manifest.js";
4
+ import type { OpenClassifyInput } from "./types.js";
5
+ export type Classifier = (input: OpenClassifyInput, options?: {
6
+ signal?: AbortSignal;
7
+ }) => Promise<PipelineResult>;
8
+ export interface CreateClassifierOptions {
9
+ runClassifier?: RunClassifier;
10
+ catalog?: Catalog;
11
+ config?: OpenClassifyConfig;
12
+ configPath?: string;
13
+ catalogPath?: string;
14
+ skipResourceCheck?: boolean;
15
+ minAvailableMemoryBytes?: number;
16
+ minTotalMemoryBytes?: number;
17
+ fetch?: typeof fetch;
18
+ classifierTimeoutMs?: number;
19
+ classifierRetryCount?: number;
20
+ aggregator?: AggregatorConfig;
21
+ }
22
+ export declare function createClassifier(options?: CreateClassifierOptions): Classifier;
@@ -0,0 +1,50 @@
1
+ // High-level facade for the pipeline. Builds the runner and catalog once,
2
+ // then returns a closure callers can invoke many times without re-loading
3
+ // config or the catalog from disk. Backend-agnostic: pass a custom
4
+ // `runClassifier` to bypass the bundled Ollama runner entirely.
5
+ import { loadCatalog } from "./catalog.js";
6
+ import { classifierModelsFromConfig, loadOpenClassifyConfig, } from "./config.js";
7
+ import { assertOllamaResources, createOllamaClassifierRunner, OLLAMA_DEFAULT_CATALOG_PATH, } from "./ollama.js";
8
+ import { classifyOpenClassifyInput } from "./pipeline.js";
9
+ export function createClassifier(options = {}) {
10
+ const fileConfig = options.config ??
11
+ loadOpenClassifyConfig(options.configPath, {
12
+ optional: options.configPath === undefined &&
13
+ process.env.OPEN_CLASSIFY_CONFIG === undefined,
14
+ });
15
+ // When we own the runner, hoist the resource check to the wrapper so a
16
+ // failure surfaces as a top-level rejection — the per-classifier fallback
17
+ // path would otherwise mask it as five "classifier failed" entries.
18
+ const ownsRunner = options.runClassifier === undefined;
19
+ const needsResourceCheck = ownsRunner && !options.skipResourceCheck;
20
+ const runClassifier = options.runClassifier ??
21
+ createOllamaClassifierRunner({
22
+ host: fileConfig?.runner?.host,
23
+ defaultModel: fileConfig?.runner?.defaultModel,
24
+ models: classifierModelsFromConfig(fileConfig),
25
+ options: fileConfig?.runner?.options,
26
+ skipResourceCheck: needsResourceCheck ? true : options.skipResourceCheck,
27
+ fetch: options.fetch,
28
+ });
29
+ const catalog = options.catalog ??
30
+ loadCatalog(options.catalogPath ?? fileConfig?.catalog ?? OLLAMA_DEFAULT_CATALOG_PATH);
31
+ const aggregator = options.aggregator ?? fileConfig?.aggregator;
32
+ let resourceCheck;
33
+ return async (input, callOptions) => {
34
+ if (needsResourceCheck) {
35
+ resourceCheck ??= assertOllamaResources({
36
+ minTotalMemoryBytes: options.minTotalMemoryBytes,
37
+ minAvailableMemoryBytes: options.minAvailableMemoryBytes,
38
+ });
39
+ await resourceCheck;
40
+ }
41
+ return classifyOpenClassifyInput(input, {
42
+ runClassifier,
43
+ catalog,
44
+ classifierTimeoutMs: options.classifierTimeoutMs,
45
+ classifierRetryCount: options.classifierRetryCount,
46
+ aggregator,
47
+ signal: callOptions?.signal,
48
+ });
49
+ };
50
+ }
@@ -1,8 +1,10 @@
1
1
  import { type ClassifierName } from "./classifiers.js";
2
+ import { type AggregatorConfig } from "./manifest.js";
2
3
  export declare const DEFAULT_OPEN_CLASSIFY_CONFIG_PATH = "open-classify.config.json";
3
4
  export interface OpenClassifyConfig {
4
5
  readonly runner?: OllamaRunnerConfig;
5
6
  readonly catalog?: string;
7
+ readonly aggregator?: AggregatorConfig;
6
8
  }
7
9
  export interface OllamaRunnerConfig {
8
10
  readonly provider: "ollama";
@@ -1,5 +1,6 @@
1
1
  import { existsSync, readFileSync } from "node:fs";
2
2
  import { REGISTRY } from "./classifiers.js";
3
+ import { CERTAINTY_GATE_MODES, } from "./manifest.js";
3
4
  import { STOCK_CLASSIFIER_NAMES } from "./stock.js";
4
5
  import { isRecord } from "./validation.js";
5
6
  export const DEFAULT_OPEN_CLASSIFY_CONFIG_PATH = "open-classify.config.json";
@@ -37,10 +38,28 @@ export function validateOpenClassifyConfig(value, path = "open-classify config")
37
38
  if (!isRecord(value)) {
38
39
  throwConfig(path, "config must be a JSON object");
39
40
  }
40
- ensureAllowedKeys(value, ["runner", "catalog"], path, "<root>");
41
+ ensureAllowedKeys(value, ["runner", "catalog", "aggregator"], path, "<root>");
41
42
  return {
42
43
  ...(value.runner === undefined ? {} : { runner: validateRunner(value.runner, path) }),
43
44
  ...(value.catalog === undefined ? {} : { catalog: requireString(value.catalog, path, "catalog") }),
45
+ ...(value.aggregator === undefined ? {} : { aggregator: validateAggregator(value.aggregator, path) }),
46
+ };
47
+ }
48
+ function validateAggregator(value, path) {
49
+ if (!isRecord(value)) {
50
+ throwConfig(path, "aggregator must be an object");
51
+ }
52
+ ensureAllowedKeys(value, ["certaintyThreshold", "confidenceThreshold", "certaintyGate"], path, "aggregator");
53
+ return {
54
+ ...(value.certaintyThreshold === undefined
55
+ ? {}
56
+ : { certaintyThreshold: requireUnitFloat(value.certaintyThreshold, path, "aggregator.certaintyThreshold") }),
57
+ ...(value.confidenceThreshold === undefined
58
+ ? {}
59
+ : { confidenceThreshold: requireUnitFloat(value.confidenceThreshold, path, "aggregator.confidenceThreshold") }),
60
+ ...(value.certaintyGate === undefined
61
+ ? {}
62
+ : { certaintyGate: requireCertaintyGateMode(value.certaintyGate, path, "aggregator.certaintyGate") }),
44
63
  };
45
64
  }
46
65
  function validateRunner(value, path) {
@@ -131,6 +150,19 @@ function requireNumber(value, path, field) {
131
150
  }
132
151
  return value;
133
152
  }
153
+ function requireUnitFloat(value, path, field) {
154
+ const number = requireNumber(value, path, field);
155
+ if (number < 0 || number > 1) {
156
+ throwConfig(path, `${field} must be a finite number between 0 and 1 inclusive`);
157
+ }
158
+ return number;
159
+ }
160
+ function requireCertaintyGateMode(value, path, field) {
161
+ if (typeof value !== "string" || !CERTAINTY_GATE_MODES.includes(value)) {
162
+ throwConfig(path, `${field} must be one of ${CERTAINTY_GATE_MODES.join(", ")}`);
163
+ }
164
+ return value;
165
+ }
134
166
  function ensureAllowedKeys(value, allowedKeys, path, field) {
135
167
  const allowed = new Set(allowedKeys);
136
168
  for (const key of Object.keys(value)) {
@@ -1,10 +1,6 @@
1
1
  export declare const DOWNSTREAM_MODEL_TIER_VALUES: readonly ["local_fast", "local_small", "local_strong", "local_coding", "frontier_fast", "frontier_strong", "frontier_coding"];
2
2
  export type DownstreamModelTier = (typeof DOWNSTREAM_MODEL_TIER_VALUES)[number];
3
- export declare const MODEL_SPECIALIZATION_VALUES: readonly ["agentic_coding", "agentic_workflows", "chat", "code_fixing", "code_reasoning", "code_review", "writing", "reasoning", "planning", "coding", "computer_use", "debugging", "instruction_following", "question_answering", "subagents", "summarization", "tool_assisted_coding", "vision_input"];
3
+ export declare const MODEL_SPECIALIZATION_VALUES: readonly ["chat", "reasoning", "planning", "writing", "summarization", "coding", "tool_use", "computer_use", "vision"];
4
4
  export type ModelSpecialization = (typeof MODEL_SPECIALIZATION_VALUES)[number];
5
- export declare const SECURITY_DECISION_VALUES: readonly ["allow", "block", "needs_review"];
6
- export type SecurityDecision = (typeof SECURITY_DECISION_VALUES)[number];
7
- export declare const SECURITY_RISK_LEVEL_VALUES: readonly ["normal", "suspicious", "high_risk", "unknown"];
8
- export type SecurityRiskLevel = (typeof SECURITY_RISK_LEVEL_VALUES)[number];
9
- export declare const SECURITY_SIGNAL_VALUES: readonly ["instruction_attack", "secret_or_private_data_risk", "unsafe_tool_or_action", "untrusted_content_or_code", "injection_or_obfuscation"];
10
- export type SecuritySignal = (typeof SECURITY_SIGNAL_VALUES)[number];
5
+ export declare const PROMPT_INJECTION_RISK_LEVEL_VALUES: readonly ["normal", "suspicious", "high_risk", "unknown"];
6
+ export type PromptInjectionRiskLevel = (typeof PROMPT_INJECTION_RISK_LEVEL_VALUES)[number];
package/dist/src/enums.js CHANGED
@@ -19,44 +19,21 @@ export const DOWNSTREAM_MODEL_TIER_VALUES = [
19
19
  // Which kind of model/prompt specialization fits the request best. Combined
20
20
  // with the tier to look up a concrete model in the catalog.
21
21
  export const MODEL_SPECIALIZATION_VALUES = [
22
- "agentic_coding",
23
- "agentic_workflows",
24
22
  "chat",
25
- "code_fixing",
26
- "code_reasoning",
27
- "code_review",
28
- "writing",
29
23
  "reasoning",
30
24
  "planning",
25
+ "writing",
26
+ "summarization",
31
27
  "coding",
28
+ "tool_use",
32
29
  "computer_use",
33
- "debugging",
34
- "instruction_following",
35
- "question_answering",
36
- "subagents",
37
- "summarization",
38
- "tool_assisted_coding",
39
- "vision_input",
40
- ];
41
- export const SECURITY_DECISION_VALUES = [
42
- "allow",
43
- "block",
44
- "needs_review",
30
+ "vision",
45
31
  ];
46
- // Overall safety posture on the latest user message. Security short-circuiting
47
- // is driven by safety.decision, not risk level alone.
48
- export const SECURITY_RISK_LEVEL_VALUES = [
32
+ // Prompt-injection posture on the latest user message. The pipeline blocks
33
+ // confident high_risk and unknown prompt-injection outputs.
34
+ export const PROMPT_INJECTION_RISK_LEVEL_VALUES = [
49
35
  "normal",
50
36
  "suspicious",
51
37
  "high_risk",
52
38
  "unknown",
53
39
  ];
54
- // Specific safety concerns the security classifier can flag. These are
55
- // advisory; safety.decision controls whether the pipeline blocks or needs review.
56
- export const SECURITY_SIGNAL_VALUES = [
57
- "instruction_attack",
58
- "secret_or_private_data_risk",
59
- "unsafe_tool_or_action",
60
- "untrusted_content_or_code",
61
- "injection_or_obfuscation",
62
- ];
@@ -1,6 +1,7 @@
1
1
  export * from "./aggregator.js";
2
2
  export * from "./catalog.js";
3
3
  export * from "./classifiers.js";
4
+ export * from "./classify.js";
4
5
  export * from "./config.js";
5
6
  export * from "./enums.js";
6
7
  export * from "./input.js";
package/dist/src/index.js CHANGED
@@ -1,11 +1,12 @@
1
1
  // Public barrel for the Open Classify package. Everything an external caller
2
2
  // would need — input types, enums, the registry, the pipeline, the Ollama
3
- // runner, the catalog loader, the aggregator's confidence threshold — is
3
+ // runner, the catalog loader, the aggregator's certainty threshold — is
4
4
  // re-exported here. The build emits a single `index.js` that downstream
5
5
  // consumers can import from `open-classify`.
6
6
  export * from "./aggregator.js";
7
7
  export * from "./catalog.js";
8
8
  export * from "./classifiers.js";
9
+ export * from "./classify.js";
9
10
  export * from "./config.js";
10
11
  export * from "./enums.js";
11
12
  export * from "./input.js";
package/dist/src/input.js CHANGED
@@ -9,7 +9,7 @@ import { createHash } from "node:crypto";
9
9
  * Gemma 4 E4B supports a native 131,072-token (128K) context window. Open
10
10
  * Classify does not use that full window in the reference local runtime: it
11
11
  * runs the classifier set in parallel with a configured 4,096-token context.
12
- * The largest fixed classifier prompt is security at about 1,748 estimated
12
+ * The largest fixed classifier prompt is prompt_injection at roughly 1,700 estimated
13
13
  * tokens using the same 3 chars/token heuristic as the Ollama packer. We round
14
14
  * that up to 2,000 fixed-prompt tokens, reserve roughly 400 tokens for output,
15
15
  * chat-template variance, and estimation error, then spend the remainder on
@@ -1,8 +1,10 @@
1
- import type { AckReplySignal, ClassifierOutput, CustomClassifierOutput, FinalReplySignal, RoutingSignal, RuntimeClassifierManifest, SafetySignal, ToolsSignal } from "./stock.js";
1
+ import type { AckReplySignal, ClassifierOutput, CustomClassifierOutput, FinalReplySignal, PromptInjectionSignal, RoutingSignal, RuntimeClassifierManifest, ToolsSignal } from "./stock.js";
2
2
  import type { ClassifierInput, ClassifierRunStatus } from "./types.js";
3
3
  import type { DownstreamModelTier, ModelSpecialization } from "./enums.js";
4
4
  export type ClassifierName = string;
5
5
  export type ClassifierResults = Record<ClassifierName, ClassifierOutput>;
6
+ export declare const CERTAINTY_GATE_MODES: readonly ["min_score", "avg_score", "off"];
7
+ export type CertaintyGateMode = (typeof CERTAINTY_GATE_MODES)[number];
6
8
  export type RunClassifier = (name: ClassifierName, input: ClassifierInput, signal: AbortSignal) => Promise<ClassifierOutput>;
7
9
  export interface CatalogEntry {
8
10
  readonly id: string;
@@ -46,7 +48,7 @@ export interface Envelope {
46
48
  readonly ack_reply?: AckReplySignal;
47
49
  readonly routing?: RoutingSignal;
48
50
  readonly tools?: ToolsSignal;
49
- readonly safety?: SafetySignal;
51
+ readonly prompt_injection?: PromptInjectionSignal;
50
52
  readonly custom_outputs: ReadonlyArray<CustomClassifierOutput>;
51
53
  readonly model_recommendation: ModelRecommendation;
52
54
  }
@@ -71,35 +73,38 @@ export interface PipelineMeta {
71
73
  export interface PipelineAudit extends Envelope {
72
74
  readonly meta: PipelineMeta;
73
75
  readonly fired_by?: string;
76
+ readonly certainty_gate?: LowCertaintyBlockReason;
74
77
  }
75
- export type AnswerPipelineResult = {
76
- readonly action: "answer";
78
+ export type BlockReason = PromptInjectionBlockReason | LowCertaintyBlockReason;
79
+ export interface PromptInjectionBlockReason {
80
+ readonly kind: "prompt_injection";
81
+ readonly risk_level: PromptInjectionSignal["risk_level"];
82
+ }
83
+ export interface LowCertaintyBlockReason {
84
+ readonly kind: "low_certainty";
85
+ readonly mode: Exclude<CertaintyGateMode, "off">;
86
+ readonly threshold: number;
87
+ readonly score: number;
88
+ readonly classifier_scores: Readonly<Record<string, number>>;
89
+ readonly low_classifiers: ReadonlyArray<string>;
90
+ }
91
+ export type ReplyPipelineResult = {
92
+ readonly action: "reply";
77
93
  readonly message_id: string;
78
- readonly final_reply: FinalReplySignal;
79
- readonly reason: "already_answered";
94
+ readonly reply: {
95
+ readonly text: string;
96
+ };
97
+ readonly reason: "preflight_reply";
80
98
  readonly classifier_outputs: ClassifierCustomOutputs;
81
99
  readonly audit: Pick<PipelineAudit, "final_reply" | "meta" | "fired_by">;
82
100
  };
83
101
  export type BlockPipelineResult = {
84
102
  readonly action: "block";
85
103
  readonly message_id: string;
86
- readonly reason: {
87
- readonly risk_level?: SafetySignal["risk_level"];
88
- readonly signals?: ReadonlyArray<string>;
89
- };
90
- readonly classifier_outputs: ClassifierCustomOutputs;
91
- readonly audit: Pick<PipelineAudit, "safety" | "meta" | "fired_by">;
92
- };
93
- export type NeedsReviewPipelineResult = {
94
- readonly action: "needs_review";
95
- readonly message_id: string;
96
- readonly fired_by: string;
97
- readonly reason: {
98
- readonly risk_level?: SafetySignal["risk_level"];
99
- readonly signals?: ReadonlyArray<string>;
100
- };
104
+ readonly fired_by?: string;
105
+ readonly reason: BlockReason;
101
106
  readonly classifier_outputs: ClassifierCustomOutputs;
102
- readonly audit: Pick<PipelineAudit, "safety" | "meta" | "fired_by">;
107
+ readonly audit: Pick<PipelineAudit, "prompt_injection" | "meta" | "fired_by" | "certainty_gate">;
103
108
  };
104
109
  export type RoutePipelineResult = {
105
110
  readonly action: "route";
@@ -108,8 +113,11 @@ export type RoutePipelineResult = {
108
113
  readonly classifier_outputs: ClassifierCustomOutputs;
109
114
  readonly audit: PipelineAudit;
110
115
  };
111
- export type PipelineResult = AnswerPipelineResult | BlockPipelineResult | NeedsReviewPipelineResult | RoutePipelineResult;
116
+ export type PipelineResult = ReplyPipelineResult | BlockPipelineResult | RoutePipelineResult;
112
117
  export interface AggregatorConfig {
118
+ readonly certaintyThreshold?: number;
119
+ /** @deprecated Use certaintyThreshold. */
113
120
  readonly confidenceThreshold?: number;
121
+ readonly certaintyGate?: CertaintyGateMode;
114
122
  }
115
123
  export type ClassifierRegistry = ReadonlyArray<RuntimeClassifierManifest>;
@@ -1 +1,5 @@
1
- export {};
1
+ export const CERTAINTY_GATE_MODES = [
2
+ "min_score",
3
+ "avg_score",
4
+ "off",
5
+ ];
@@ -1,8 +1,4 @@
1
1
  import { type ClassifierName, type RunClassifier } from "./classifiers.js";
2
- import { type OpenClassifyConfig } from "./config.js";
3
- import { classifyOpenClassifyInput } from "./pipeline.js";
4
- import type { Catalog } from "./manifest.js";
5
- import type { OpenClassifyInput } from "./types.js";
6
2
  export declare const OLLAMA_DEFAULT_HOST = "http://localhost:11434";
7
3
  export declare const OLLAMA_BASE_MODEL = "gemma4:e4b-it-q4_K_M";
8
4
  export declare const OLLAMA_BASE_MODEL_NATIVE_CONTEXT_LENGTH = 131072;
@@ -28,12 +24,6 @@ export interface OllamaClassifierRunnerConfig {
28
24
  minAvailableMemoryBytes?: number;
29
25
  minTotalMemoryBytes?: number;
30
26
  }
31
- export interface ClassifyWithOllamaConfig extends OllamaClassifierRunnerConfig {
32
- catalog?: Catalog;
33
- catalogPath?: string;
34
- configPath?: string;
35
- openClassifyConfig?: OpenClassifyConfig;
36
- }
37
27
  export declare class OllamaClassifierError extends Error {
38
28
  readonly classifier: ClassifierName;
39
29
  readonly model: string;
@@ -51,4 +41,3 @@ export declare function assertOllamaResources(options?: {
51
41
  minTotalMemoryBytes?: number;
52
42
  minAvailableMemoryBytes?: number;
53
43
  }): Promise<void>;
54
- export declare function classifyWithOllama(input: OpenClassifyInput, config?: ClassifyWithOllamaConfig): ReturnType<typeof classifyOpenClassifyInput>;
@@ -10,10 +10,7 @@
10
10
  // `classifyOpenClassifyInput` — you don't have to use this module at all.
11
11
  import { execFile } from "node:child_process";
12
12
  import { promisify } from "node:util";
13
- import { loadCatalog } from "./catalog.js";
14
13
  import { CLASSIFIER_NAMES, MODULES_BY_NAME, validateClassifierOutput, } from "./classifiers.js";
15
- import { classifierModelsFromConfig, loadOpenClassifyConfig, } from "./config.js";
16
- import { classifyOpenClassifyInput } from "./pipeline.js";
17
14
  import { ClassifierValidationError, isRecord, } from "./validation.js";
18
15
  export const OLLAMA_DEFAULT_HOST = "http://localhost:11434";
19
16
  export const OLLAMA_BASE_MODEL = "gemma4:e4b-it-q4_K_M";
@@ -93,39 +90,6 @@ export async function assertOllamaResources(options = {}) {
93
90
  throw new OllamaResourceError(totalMemoryBytes, availableMemoryBytes, minTotalMemoryBytes, minAvailableMemoryBytes);
94
91
  }
95
92
  }
96
- export async function classifyWithOllama(input, config = {}) {
97
- const fileConfig = config.openClassifyConfig ?? loadOpenClassifyConfig(config.configPath, {
98
- optional: config.configPath === undefined && process.env.OPEN_CLASSIFY_CONFIG === undefined,
99
- });
100
- const runnerFileConfig = fileConfig?.runner;
101
- const runnerConfig = {
102
- ...config,
103
- host: config.host ?? runnerFileConfig?.host,
104
- defaultModel: config.defaultModel ?? runnerFileConfig?.defaultModel,
105
- models: {
106
- ...classifierModelsFromConfig(fileConfig),
107
- ...config.models,
108
- },
109
- options: {
110
- ...runnerFileConfig?.options,
111
- ...config.options,
112
- },
113
- };
114
- if (!runnerConfig.skipResourceCheck) {
115
- await assertOllamaResources({
116
- minTotalMemoryBytes: runnerConfig.minTotalMemoryBytes,
117
- minAvailableMemoryBytes: runnerConfig.minAvailableMemoryBytes,
118
- });
119
- Object.assign(runnerConfig, {
120
- skipResourceCheck: true,
121
- });
122
- }
123
- const catalog = config.catalog ?? loadCatalog(config.catalogPath ?? fileConfig?.catalog ?? OLLAMA_DEFAULT_CATALOG_PATH);
124
- return classifyOpenClassifyInput(input, {
125
- runClassifier: createOllamaClassifierRunner(runnerConfig),
126
- catalog,
127
- });
128
- }
129
93
  async function runOllamaClassifier(name, input, signal, fetchImpl, host, model, options, allowManifestModel) {
130
94
  const module_ = MODULES_BY_NAME[name];
131
95
  const systemPrompt = module_.systemPrompt;
@@ -3,6 +3,7 @@ import type { AggregatorConfig, Catalog, PipelineResult } from "./manifest.js";
3
3
  import type { OpenClassifyInput } from "./types.js";
4
4
  export declare const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15000;
5
5
  export declare const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
6
+ export declare const DEFAULT_CERTAINTY_GATE = "min_score";
6
7
  export declare class OpenClassifyNormalizationError extends Error {
7
8
  constructor(cause: unknown);
8
9
  }
@@ -1,9 +1,10 @@
1
- import { composeEnvelope } from "./aggregator.js";
1
+ import { certaintyThreshold, composeEnvelope } from "./aggregator.js";
2
2
  import { CLASSIFIER_NAMES, MODULES_BY_NAME, REGISTRY, } from "./classifiers.js";
3
3
  import { normalizeOpenClassifyInput, toClassifierInput } from "./input.js";
4
- import { isCustomManifest } from "./stock.js";
4
+ import { certaintyScore, isCustomManifest } from "./stock.js";
5
5
  export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15_000;
6
6
  export const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
7
+ export const DEFAULT_CERTAINTY_GATE = "min_score";
7
8
  export class OpenClassifyNormalizationError extends Error {
8
9
  constructor(cause) {
9
10
  super(errorMessage(cause), { cause });
@@ -11,10 +12,10 @@ export class OpenClassifyNormalizationError extends Error {
11
12
  }
12
13
  }
13
14
  // Short-circuit gates are intrinsic to specific stock signals — not configured
14
- // per-manifest. preflight.final_reply ⇒ answer; security.decision in
15
- // {block, needs_review} ⇒ block / needs_review. Order matters: preflight is
15
+ // per-manifest. preflight.final_reply ⇒ reply; confident high_risk or unknown
16
+ // prompt-injection risk ⇒ block. Order matters: preflight is
16
17
  // cheaper to evaluate, so we check it first.
17
- const SHORT_CIRCUIT_GATES = ["preflight", "security"];
18
+ const SHORT_CIRCUIT_GATES = ["preflight", "prompt_injection"];
18
19
  export async function classifyOpenClassifyInput(input, options) {
19
20
  let request;
20
21
  try {
@@ -36,7 +37,7 @@ export async function classifyOpenClassifyInput(input, options) {
36
37
  const classifierInput = toClassifierInput(request);
37
38
  const classifierTimeoutMs = options.classifierTimeoutMs ?? DEFAULT_CLASSIFIER_TIMEOUT_MS;
38
39
  const classifierRetryCount = options.classifierRetryCount ?? DEFAULT_CLASSIFIER_RETRY_COUNT;
39
- const threshold = options.aggregator?.confidenceThreshold ?? 0.6;
40
+ const threshold = certaintyThreshold(options.aggregator);
40
41
  const runs = new Map(CLASSIFIER_NAMES.map((name) => [
41
42
  name,
42
43
  runClassifierWithRetry(name, classifierInput, options.runClassifier, controller.signal, classifierTimeoutMs, classifierRetryCount),
@@ -65,6 +66,10 @@ export async function classifyOpenClassifyInput(input, options) {
65
66
  input: classifierInput,
66
67
  config: options.aggregator,
67
68
  });
69
+ const certaintyGate = certaintyGateBlock(options.aggregator, results);
70
+ if (certaintyGate) {
71
+ return buildCertaintyGateBlockResult(request, envelope, results, meta, certaintyGate);
72
+ }
68
73
  return buildRouteResult(request, envelope, results, meta);
69
74
  }
70
75
  finally {
@@ -72,38 +77,67 @@ export async function classifyOpenClassifyInput(input, options) {
72
77
  }
73
78
  }
74
79
  function shortCircuitVerdict(gate, result, threshold) {
75
- const confidence = result.confidence ?? 0;
76
- if (confidence < threshold)
80
+ const score = scoreCertainty(result.certainty);
81
+ if (score < threshold)
77
82
  return null;
78
83
  if (gate === "preflight") {
79
84
  const preflight = result;
80
85
  if (preflight.final_reply !== undefined) {
81
- return { kind: "answer", final_reply: preflight.final_reply };
86
+ return { kind: "reply", final_reply: preflight.final_reply };
82
87
  }
83
88
  return null;
84
89
  }
85
- if (gate === "security") {
86
- const security = result;
87
- if (security.decision === "block") {
90
+ if (gate === "prompt_injection") {
91
+ const promptInjection = result;
92
+ if (promptInjection.risk_level === "high_risk" || promptInjection.risk_level === "unknown") {
93
+ const promptInjectionSignal = extractPromptInjection(promptInjection);
88
94
  return {
89
95
  kind: "block",
90
- safety: extractSafety(security),
91
- };
92
- }
93
- if (security.decision === "needs_review") {
94
- return {
95
- kind: "needs_review",
96
- safety: extractSafety(security),
96
+ prompt_injection: promptInjectionSignal,
97
+ reason: {
98
+ kind: "prompt_injection",
99
+ risk_level: promptInjectionSignal.risk_level,
100
+ },
97
101
  };
98
102
  }
99
103
  }
100
104
  return null;
101
105
  }
102
- function extractSafety(value) {
106
+ function certaintyGateBlock(config, results) {
107
+ const mode = config?.certaintyGate ?? DEFAULT_CERTAINTY_GATE;
108
+ if (mode === "off")
109
+ return undefined;
110
+ const threshold = certaintyThreshold(config);
111
+ const classifier_scores = classifierScores(results);
112
+ const scores = Object.values(classifier_scores);
113
+ const score = mode === "min_score"
114
+ ? Math.min(...scores)
115
+ : scores.reduce((sum, value) => sum + value, 0) / scores.length;
116
+ if (score >= threshold)
117
+ return undefined;
118
+ return {
119
+ kind: "low_certainty",
120
+ mode,
121
+ threshold,
122
+ score,
123
+ classifier_scores,
124
+ low_classifiers: Object.entries(classifier_scores)
125
+ .filter(([, value]) => value < threshold)
126
+ .map(([name]) => name),
127
+ };
128
+ }
129
+ function classifierScores(results) {
130
+ return Object.fromEntries(REGISTRY.map((manifest) => [
131
+ manifest.name,
132
+ scoreCertainty(results[manifest.name]?.certainty),
133
+ ]));
134
+ }
135
+ function scoreCertainty(certainty) {
136
+ return certainty === undefined ? 0 : certaintyScore[certainty];
137
+ }
138
+ function extractPromptInjection(value) {
103
139
  return {
104
- ...(value.decision === undefined ? {} : { decision: value.decision }),
105
140
  risk_level: value.risk_level,
106
- signals: value.signals,
107
141
  };
108
142
  }
109
143
  function buildShortCircuitResult(name, verdict, settled, target_message_hash) {
@@ -116,13 +150,13 @@ function buildShortCircuitResult(name, verdict, settled, target_message_hash) {
116
150
  };
117
151
  const meta = { classifiers: { [name]: entry } };
118
152
  const classifier_outputs = classifierCustomOutputs({ [name]: value });
119
- if (verdict.kind === "answer") {
153
+ if (verdict.kind === "reply") {
120
154
  const preflight = value;
121
155
  return {
122
- action: "answer",
156
+ action: "reply",
123
157
  message_id: target_message_hash,
124
- final_reply: verdict.final_reply,
125
- reason: "already_answered",
158
+ reply: { text: verdict.final_reply.reply },
159
+ reason: "preflight_reply",
126
160
  classifier_outputs,
127
161
  audit: {
128
162
  fired_by: name,
@@ -131,34 +165,15 @@ function buildShortCircuitResult(name, verdict, settled, target_message_hash) {
131
165
  },
132
166
  };
133
167
  }
134
- if (verdict.kind === "needs_review") {
135
- return {
136
- action: "needs_review",
137
- message_id: target_message_hash,
138
- fired_by: name,
139
- reason: {
140
- risk_level: verdict.safety.risk_level,
141
- signals: verdict.safety.signals,
142
- },
143
- classifier_outputs,
144
- audit: {
145
- fired_by: name,
146
- safety: verdict.safety,
147
- meta,
148
- },
149
- };
150
- }
151
168
  return {
152
169
  action: "block",
153
170
  message_id: target_message_hash,
154
- reason: {
155
- risk_level: verdict.safety.risk_level,
156
- signals: verdict.safety.signals,
157
- },
171
+ fired_by: name,
172
+ reason: verdict.reason,
158
173
  classifier_outputs,
159
174
  audit: {
160
175
  fired_by: name,
161
- safety: verdict.safety,
176
+ prompt_injection: verdict.prompt_injection,
162
177
  meta,
163
178
  },
164
179
  };
@@ -199,6 +214,21 @@ function buildRouteResult(request, envelope, results, meta) {
199
214
  },
200
215
  };
201
216
  }
217
+ function buildCertaintyGateBlockResult(request, envelope, results, meta, certaintyGate) {
218
+ return {
219
+ action: "block",
220
+ message_id: request.target_message_hash,
221
+ fired_by: "certainty_gate",
222
+ reason: certaintyGate,
223
+ classifier_outputs: classifierCustomOutputs(results),
224
+ audit: {
225
+ ...envelope,
226
+ fired_by: "certainty_gate",
227
+ certainty_gate: certaintyGate,
228
+ meta,
229
+ },
230
+ };
231
+ }
202
232
  function classifierCustomOutputs(results) {
203
233
  const out = {};
204
234
  for (const manifest of REGISTRY) {
@@ -26,7 +26,7 @@ function stockSection(manifest) {
26
26
  allowed_tools: renderAllowedTools(manifest.tools),
27
27
  preflight_output: promptMarkdown("preflight-output.md"),
28
28
  routing_output: promptMarkdown("routing-output.md"),
29
- security_output: promptMarkdown("security-output.md"),
29
+ prompt_injection_output: promptMarkdown("prompt-injection-output.md"),
30
30
  specialty: promptMarkdown("specialty.md"),
31
31
  tier: promptMarkdown("tier.md"),
32
32
  tools_output: promptMarkdown("tools-output.md"),