open-classify 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -35
- package/dist/src/aggregator.d.ts +4 -1
- package/dist/src/aggregator.js +25 -15
- package/dist/src/classifiers/custom/context_shift/manifest.json +31 -0
- package/dist/src/classifiers/custom/context_shift/prompt.md +12 -0
- package/dist/src/classifiers/custom/{conversation_diegest → conversation_digest}/manifest.json +3 -1
- package/dist/src/classifiers/custom/{conversation_diegest → conversation_digest}/prompt.md +1 -1
- package/dist/src/classifiers/custom/memory_retrieval_queries/manifest.json +2 -0
- package/dist/src/classifiers/stock/model_specialization/manifest.json +4 -1
- package/dist/src/classifiers/stock/preflight/manifest.json +4 -1
- package/dist/src/classifiers/stock/prompt_injection/manifest.json +12 -0
- package/dist/src/classifiers/stock/prompts/confidence.md +3 -3
- package/dist/src/classifiers/stock/prompts/custom-output.md +7 -1
- package/dist/src/classifiers/stock/prompts/preflight.md +7 -7
- package/dist/src/classifiers/stock/prompts/prompt-injection-output.md +5 -0
- package/dist/src/classifiers/stock/prompts/prompt_injection.md +24 -0
- package/dist/src/classifiers/stock/prompts/reason.md +1 -1
- package/dist/src/classifiers/stock/prompts/specialty.md +8 -6
- package/dist/src/classifiers/stock/prompts/tier.md +1 -1
- package/dist/src/classifiers/stock/prompts/tools-output.md +4 -0
- package/dist/src/classifiers/stock/routing/manifest.json +4 -1
- package/dist/src/classifiers/stock/tools/manifest.json +2 -0
- package/dist/src/classify.d.ts +22 -0
- package/dist/src/classify.js +50 -0
- package/dist/src/config.d.ts +2 -0
- package/dist/src/config.js +33 -1
- package/dist/src/enums.d.ts +3 -7
- package/dist/src/enums.js +7 -30
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +2 -1
- package/dist/src/input.js +1 -1
- package/dist/src/manifest.d.ts +31 -23
- package/dist/src/manifest.js +5 -1
- package/dist/src/ollama.d.ts +0 -11
- package/dist/src/ollama.js +0 -36
- package/dist/src/pipeline.d.ts +1 -0
- package/dist/src/pipeline.js +78 -48
- package/dist/src/stock-prompt.js +1 -1
- package/dist/src/stock-validation.d.ts +1 -2
- package/dist/src/stock-validation.js +23 -40
- package/dist/src/stock.d.ts +12 -11
- package/dist/src/stock.js +21 -1
- package/dist/src/ui-server.js +12 -5
- package/dist/src/validation.d.ts +0 -1
- package/dist/src/validation.js +0 -37
- package/docs/adding-a-classifier.md +132 -0
- package/docs/manifests.md +127 -0
- package/docs/resolver.md +104 -0
- package/docs/signals.md +102 -0
- package/downstream-models.json +124 -0
- package/open-classify.config.example.json +5 -1
- package/package.json +3 -1
- package/dist/src/classifiers/stock/prompts/security-output.md +0 -8
- package/dist/src/classifiers/stock/prompts/security.md +0 -26
- package/dist/src/classifiers/stock/security/manifest.json +0 -12
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type RunClassifier } from "./classifiers.js";
|
|
2
|
+
import { type OpenClassifyConfig } from "./config.js";
|
|
3
|
+
import type { AggregatorConfig, Catalog, PipelineResult } from "./manifest.js";
|
|
4
|
+
import type { OpenClassifyInput } from "./types.js";
|
|
5
|
+
export type Classifier = (input: OpenClassifyInput, options?: {
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
}) => Promise<PipelineResult>;
|
|
8
|
+
export interface CreateClassifierOptions {
|
|
9
|
+
runClassifier?: RunClassifier;
|
|
10
|
+
catalog?: Catalog;
|
|
11
|
+
config?: OpenClassifyConfig;
|
|
12
|
+
configPath?: string;
|
|
13
|
+
catalogPath?: string;
|
|
14
|
+
skipResourceCheck?: boolean;
|
|
15
|
+
minAvailableMemoryBytes?: number;
|
|
16
|
+
minTotalMemoryBytes?: number;
|
|
17
|
+
fetch?: typeof fetch;
|
|
18
|
+
classifierTimeoutMs?: number;
|
|
19
|
+
classifierRetryCount?: number;
|
|
20
|
+
aggregator?: AggregatorConfig;
|
|
21
|
+
}
|
|
22
|
+
export declare function createClassifier(options?: CreateClassifierOptions): Classifier;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// High-level facade for the pipeline. Builds the runner and catalog once,
|
|
2
|
+
// then returns a closure callers can invoke many times without re-loading
|
|
3
|
+
// config or the catalog from disk. Backend-agnostic: pass a custom
|
|
4
|
+
// `runClassifier` to bypass the bundled Ollama runner entirely.
|
|
5
|
+
import { loadCatalog } from "./catalog.js";
|
|
6
|
+
import { classifierModelsFromConfig, loadOpenClassifyConfig, } from "./config.js";
|
|
7
|
+
import { assertOllamaResources, createOllamaClassifierRunner, OLLAMA_DEFAULT_CATALOG_PATH, } from "./ollama.js";
|
|
8
|
+
import { classifyOpenClassifyInput } from "./pipeline.js";
|
|
9
|
+
export function createClassifier(options = {}) {
|
|
10
|
+
const fileConfig = options.config ??
|
|
11
|
+
loadOpenClassifyConfig(options.configPath, {
|
|
12
|
+
optional: options.configPath === undefined &&
|
|
13
|
+
process.env.OPEN_CLASSIFY_CONFIG === undefined,
|
|
14
|
+
});
|
|
15
|
+
// When we own the runner, hoist the resource check to the wrapper so a
|
|
16
|
+
// failure surfaces as a top-level rejection — the per-classifier fallback
|
|
17
|
+
// path would otherwise mask it as five "classifier failed" entries.
|
|
18
|
+
const ownsRunner = options.runClassifier === undefined;
|
|
19
|
+
const needsResourceCheck = ownsRunner && !options.skipResourceCheck;
|
|
20
|
+
const runClassifier = options.runClassifier ??
|
|
21
|
+
createOllamaClassifierRunner({
|
|
22
|
+
host: fileConfig?.runner?.host,
|
|
23
|
+
defaultModel: fileConfig?.runner?.defaultModel,
|
|
24
|
+
models: classifierModelsFromConfig(fileConfig),
|
|
25
|
+
options: fileConfig?.runner?.options,
|
|
26
|
+
skipResourceCheck: needsResourceCheck ? true : options.skipResourceCheck,
|
|
27
|
+
fetch: options.fetch,
|
|
28
|
+
});
|
|
29
|
+
const catalog = options.catalog ??
|
|
30
|
+
loadCatalog(options.catalogPath ?? fileConfig?.catalog ?? OLLAMA_DEFAULT_CATALOG_PATH);
|
|
31
|
+
const aggregator = options.aggregator ?? fileConfig?.aggregator;
|
|
32
|
+
let resourceCheck;
|
|
33
|
+
return async (input, callOptions) => {
|
|
34
|
+
if (needsResourceCheck) {
|
|
35
|
+
resourceCheck ??= assertOllamaResources({
|
|
36
|
+
minTotalMemoryBytes: options.minTotalMemoryBytes,
|
|
37
|
+
minAvailableMemoryBytes: options.minAvailableMemoryBytes,
|
|
38
|
+
});
|
|
39
|
+
await resourceCheck;
|
|
40
|
+
}
|
|
41
|
+
return classifyOpenClassifyInput(input, {
|
|
42
|
+
runClassifier,
|
|
43
|
+
catalog,
|
|
44
|
+
classifierTimeoutMs: options.classifierTimeoutMs,
|
|
45
|
+
classifierRetryCount: options.classifierRetryCount,
|
|
46
|
+
aggregator,
|
|
47
|
+
signal: callOptions?.signal,
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
}
|
package/dist/src/config.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { type ClassifierName } from "./classifiers.js";
|
|
2
|
+
import { type AggregatorConfig } from "./manifest.js";
|
|
2
3
|
export declare const DEFAULT_OPEN_CLASSIFY_CONFIG_PATH = "open-classify.config.json";
|
|
3
4
|
export interface OpenClassifyConfig {
|
|
4
5
|
readonly runner?: OllamaRunnerConfig;
|
|
5
6
|
readonly catalog?: string;
|
|
7
|
+
readonly aggregator?: AggregatorConfig;
|
|
6
8
|
}
|
|
7
9
|
export interface OllamaRunnerConfig {
|
|
8
10
|
readonly provider: "ollama";
|
package/dist/src/config.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { REGISTRY } from "./classifiers.js";
|
|
3
|
+
import { CERTAINTY_GATE_MODES, } from "./manifest.js";
|
|
3
4
|
import { STOCK_CLASSIFIER_NAMES } from "./stock.js";
|
|
4
5
|
import { isRecord } from "./validation.js";
|
|
5
6
|
export const DEFAULT_OPEN_CLASSIFY_CONFIG_PATH = "open-classify.config.json";
|
|
@@ -37,10 +38,28 @@ export function validateOpenClassifyConfig(value, path = "open-classify config")
|
|
|
37
38
|
if (!isRecord(value)) {
|
|
38
39
|
throwConfig(path, "config must be a JSON object");
|
|
39
40
|
}
|
|
40
|
-
ensureAllowedKeys(value, ["runner", "catalog"], path, "<root>");
|
|
41
|
+
ensureAllowedKeys(value, ["runner", "catalog", "aggregator"], path, "<root>");
|
|
41
42
|
return {
|
|
42
43
|
...(value.runner === undefined ? {} : { runner: validateRunner(value.runner, path) }),
|
|
43
44
|
...(value.catalog === undefined ? {} : { catalog: requireString(value.catalog, path, "catalog") }),
|
|
45
|
+
...(value.aggregator === undefined ? {} : { aggregator: validateAggregator(value.aggregator, path) }),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function validateAggregator(value, path) {
|
|
49
|
+
if (!isRecord(value)) {
|
|
50
|
+
throwConfig(path, "aggregator must be an object");
|
|
51
|
+
}
|
|
52
|
+
ensureAllowedKeys(value, ["certaintyThreshold", "confidenceThreshold", "certaintyGate"], path, "aggregator");
|
|
53
|
+
return {
|
|
54
|
+
...(value.certaintyThreshold === undefined
|
|
55
|
+
? {}
|
|
56
|
+
: { certaintyThreshold: requireUnitFloat(value.certaintyThreshold, path, "aggregator.certaintyThreshold") }),
|
|
57
|
+
...(value.confidenceThreshold === undefined
|
|
58
|
+
? {}
|
|
59
|
+
: { confidenceThreshold: requireUnitFloat(value.confidenceThreshold, path, "aggregator.confidenceThreshold") }),
|
|
60
|
+
...(value.certaintyGate === undefined
|
|
61
|
+
? {}
|
|
62
|
+
: { certaintyGate: requireCertaintyGateMode(value.certaintyGate, path, "aggregator.certaintyGate") }),
|
|
44
63
|
};
|
|
45
64
|
}
|
|
46
65
|
function validateRunner(value, path) {
|
|
@@ -131,6 +150,19 @@ function requireNumber(value, path, field) {
|
|
|
131
150
|
}
|
|
132
151
|
return value;
|
|
133
152
|
}
|
|
153
|
+
function requireUnitFloat(value, path, field) {
|
|
154
|
+
const number = requireNumber(value, path, field);
|
|
155
|
+
if (number < 0 || number > 1) {
|
|
156
|
+
throwConfig(path, `${field} must be a finite number between 0 and 1 inclusive`);
|
|
157
|
+
}
|
|
158
|
+
return number;
|
|
159
|
+
}
|
|
160
|
+
function requireCertaintyGateMode(value, path, field) {
|
|
161
|
+
if (typeof value !== "string" || !CERTAINTY_GATE_MODES.includes(value)) {
|
|
162
|
+
throwConfig(path, `${field} must be one of ${CERTAINTY_GATE_MODES.join(", ")}`);
|
|
163
|
+
}
|
|
164
|
+
return value;
|
|
165
|
+
}
|
|
134
166
|
function ensureAllowedKeys(value, allowedKeys, path, field) {
|
|
135
167
|
const allowed = new Set(allowedKeys);
|
|
136
168
|
for (const key of Object.keys(value)) {
|
package/dist/src/enums.d.ts
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
export declare const DOWNSTREAM_MODEL_TIER_VALUES: readonly ["local_fast", "local_small", "local_strong", "local_coding", "frontier_fast", "frontier_strong", "frontier_coding"];
|
|
2
2
|
export type DownstreamModelTier = (typeof DOWNSTREAM_MODEL_TIER_VALUES)[number];
|
|
3
|
-
export declare const MODEL_SPECIALIZATION_VALUES: readonly ["
|
|
3
|
+
export declare const MODEL_SPECIALIZATION_VALUES: readonly ["chat", "reasoning", "planning", "writing", "summarization", "coding", "tool_use", "computer_use", "vision"];
|
|
4
4
|
export type ModelSpecialization = (typeof MODEL_SPECIALIZATION_VALUES)[number];
|
|
5
|
-
export declare const
|
|
6
|
-
export type
|
|
7
|
-
export declare const SECURITY_RISK_LEVEL_VALUES: readonly ["normal", "suspicious", "high_risk", "unknown"];
|
|
8
|
-
export type SecurityRiskLevel = (typeof SECURITY_RISK_LEVEL_VALUES)[number];
|
|
9
|
-
export declare const SECURITY_SIGNAL_VALUES: readonly ["instruction_attack", "secret_or_private_data_risk", "unsafe_tool_or_action", "untrusted_content_or_code", "injection_or_obfuscation"];
|
|
10
|
-
export type SecuritySignal = (typeof SECURITY_SIGNAL_VALUES)[number];
|
|
5
|
+
export declare const PROMPT_INJECTION_RISK_LEVEL_VALUES: readonly ["normal", "suspicious", "high_risk", "unknown"];
|
|
6
|
+
export type PromptInjectionRiskLevel = (typeof PROMPT_INJECTION_RISK_LEVEL_VALUES)[number];
|
package/dist/src/enums.js
CHANGED
|
@@ -19,44 +19,21 @@ export const DOWNSTREAM_MODEL_TIER_VALUES = [
|
|
|
19
19
|
// Which kind of model/prompt specialization fits the request best. Combined
|
|
20
20
|
// with the tier to look up a concrete model in the catalog.
|
|
21
21
|
export const MODEL_SPECIALIZATION_VALUES = [
|
|
22
|
-
"agentic_coding",
|
|
23
|
-
"agentic_workflows",
|
|
24
22
|
"chat",
|
|
25
|
-
"code_fixing",
|
|
26
|
-
"code_reasoning",
|
|
27
|
-
"code_review",
|
|
28
|
-
"writing",
|
|
29
23
|
"reasoning",
|
|
30
24
|
"planning",
|
|
25
|
+
"writing",
|
|
26
|
+
"summarization",
|
|
31
27
|
"coding",
|
|
28
|
+
"tool_use",
|
|
32
29
|
"computer_use",
|
|
33
|
-
"
|
|
34
|
-
"instruction_following",
|
|
35
|
-
"question_answering",
|
|
36
|
-
"subagents",
|
|
37
|
-
"summarization",
|
|
38
|
-
"tool_assisted_coding",
|
|
39
|
-
"vision_input",
|
|
40
|
-
];
|
|
41
|
-
export const SECURITY_DECISION_VALUES = [
|
|
42
|
-
"allow",
|
|
43
|
-
"block",
|
|
44
|
-
"needs_review",
|
|
30
|
+
"vision",
|
|
45
31
|
];
|
|
46
|
-
//
|
|
47
|
-
//
|
|
48
|
-
export const
|
|
32
|
+
// Prompt-injection posture on the latest user message. The pipeline blocks
|
|
33
|
+
// confident high_risk and unknown prompt-injection outputs.
|
|
34
|
+
export const PROMPT_INJECTION_RISK_LEVEL_VALUES = [
|
|
49
35
|
"normal",
|
|
50
36
|
"suspicious",
|
|
51
37
|
"high_risk",
|
|
52
38
|
"unknown",
|
|
53
39
|
];
|
|
54
|
-
// Specific safety concerns the security classifier can flag. These are
|
|
55
|
-
// advisory; safety.decision controls whether the pipeline blocks or needs review.
|
|
56
|
-
export const SECURITY_SIGNAL_VALUES = [
|
|
57
|
-
"instruction_attack",
|
|
58
|
-
"secret_or_private_data_risk",
|
|
59
|
-
"unsafe_tool_or_action",
|
|
60
|
-
"untrusted_content_or_code",
|
|
61
|
-
"injection_or_obfuscation",
|
|
62
|
-
];
|
package/dist/src/index.d.ts
CHANGED
package/dist/src/index.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// Public barrel for the Open Classify package. Everything an external caller
|
|
2
2
|
// would need — input types, enums, the registry, the pipeline, the Ollama
|
|
3
|
-
// runner, the catalog loader, the aggregator's
|
|
3
|
+
// runner, the catalog loader, the aggregator's certainty threshold — is
|
|
4
4
|
// re-exported here. The build emits a single `index.js` that downstream
|
|
5
5
|
// consumers can import from `open-classify`.
|
|
6
6
|
export * from "./aggregator.js";
|
|
7
7
|
export * from "./catalog.js";
|
|
8
8
|
export * from "./classifiers.js";
|
|
9
|
+
export * from "./classify.js";
|
|
9
10
|
export * from "./config.js";
|
|
10
11
|
export * from "./enums.js";
|
|
11
12
|
export * from "./input.js";
|
package/dist/src/input.js
CHANGED
|
@@ -9,7 +9,7 @@ import { createHash } from "node:crypto";
|
|
|
9
9
|
* Gemma 4 E4B supports a native 131,072-token (128K) context window. Open
|
|
10
10
|
* Classify does not use that full window in the reference local runtime: it
|
|
11
11
|
* runs the classifier set in parallel with a configured 4,096-token context.
|
|
12
|
-
* The largest fixed classifier prompt is
|
|
12
|
+
* The largest fixed classifier prompt is prompt_injection at roughly 1,700 estimated
|
|
13
13
|
* tokens using the same 3 chars/token heuristic as the Ollama packer. We round
|
|
14
14
|
* that up to 2,000 fixed-prompt tokens, reserve roughly 400 tokens for output,
|
|
15
15
|
* chat-template variance, and estimation error, then spend the remainder on
|
package/dist/src/manifest.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import type { AckReplySignal, ClassifierOutput, CustomClassifierOutput, FinalReplySignal, RoutingSignal, RuntimeClassifierManifest,
|
|
1
|
+
import type { AckReplySignal, ClassifierOutput, CustomClassifierOutput, FinalReplySignal, PromptInjectionSignal, RoutingSignal, RuntimeClassifierManifest, ToolsSignal } from "./stock.js";
|
|
2
2
|
import type { ClassifierInput, ClassifierRunStatus } from "./types.js";
|
|
3
3
|
import type { DownstreamModelTier, ModelSpecialization } from "./enums.js";
|
|
4
4
|
export type ClassifierName = string;
|
|
5
5
|
export type ClassifierResults = Record<ClassifierName, ClassifierOutput>;
|
|
6
|
+
export declare const CERTAINTY_GATE_MODES: readonly ["min_score", "avg_score", "off"];
|
|
7
|
+
export type CertaintyGateMode = (typeof CERTAINTY_GATE_MODES)[number];
|
|
6
8
|
export type RunClassifier = (name: ClassifierName, input: ClassifierInput, signal: AbortSignal) => Promise<ClassifierOutput>;
|
|
7
9
|
export interface CatalogEntry {
|
|
8
10
|
readonly id: string;
|
|
@@ -46,7 +48,7 @@ export interface Envelope {
|
|
|
46
48
|
readonly ack_reply?: AckReplySignal;
|
|
47
49
|
readonly routing?: RoutingSignal;
|
|
48
50
|
readonly tools?: ToolsSignal;
|
|
49
|
-
readonly
|
|
51
|
+
readonly prompt_injection?: PromptInjectionSignal;
|
|
50
52
|
readonly custom_outputs: ReadonlyArray<CustomClassifierOutput>;
|
|
51
53
|
readonly model_recommendation: ModelRecommendation;
|
|
52
54
|
}
|
|
@@ -71,35 +73,38 @@ export interface PipelineMeta {
|
|
|
71
73
|
export interface PipelineAudit extends Envelope {
|
|
72
74
|
readonly meta: PipelineMeta;
|
|
73
75
|
readonly fired_by?: string;
|
|
76
|
+
readonly certainty_gate?: LowCertaintyBlockReason;
|
|
74
77
|
}
|
|
75
|
-
export type
|
|
76
|
-
|
|
78
|
+
export type BlockReason = PromptInjectionBlockReason | LowCertaintyBlockReason;
|
|
79
|
+
export interface PromptInjectionBlockReason {
|
|
80
|
+
readonly kind: "prompt_injection";
|
|
81
|
+
readonly risk_level: PromptInjectionSignal["risk_level"];
|
|
82
|
+
}
|
|
83
|
+
export interface LowCertaintyBlockReason {
|
|
84
|
+
readonly kind: "low_certainty";
|
|
85
|
+
readonly mode: Exclude<CertaintyGateMode, "off">;
|
|
86
|
+
readonly threshold: number;
|
|
87
|
+
readonly score: number;
|
|
88
|
+
readonly classifier_scores: Readonly<Record<string, number>>;
|
|
89
|
+
readonly low_classifiers: ReadonlyArray<string>;
|
|
90
|
+
}
|
|
91
|
+
export type ReplyPipelineResult = {
|
|
92
|
+
readonly action: "reply";
|
|
77
93
|
readonly message_id: string;
|
|
78
|
-
readonly
|
|
79
|
-
|
|
94
|
+
readonly reply: {
|
|
95
|
+
readonly text: string;
|
|
96
|
+
};
|
|
97
|
+
readonly reason: "preflight_reply";
|
|
80
98
|
readonly classifier_outputs: ClassifierCustomOutputs;
|
|
81
99
|
readonly audit: Pick<PipelineAudit, "final_reply" | "meta" | "fired_by">;
|
|
82
100
|
};
|
|
83
101
|
export type BlockPipelineResult = {
|
|
84
102
|
readonly action: "block";
|
|
85
103
|
readonly message_id: string;
|
|
86
|
-
readonly
|
|
87
|
-
|
|
88
|
-
readonly signals?: ReadonlyArray<string>;
|
|
89
|
-
};
|
|
90
|
-
readonly classifier_outputs: ClassifierCustomOutputs;
|
|
91
|
-
readonly audit: Pick<PipelineAudit, "safety" | "meta" | "fired_by">;
|
|
92
|
-
};
|
|
93
|
-
export type NeedsReviewPipelineResult = {
|
|
94
|
-
readonly action: "needs_review";
|
|
95
|
-
readonly message_id: string;
|
|
96
|
-
readonly fired_by: string;
|
|
97
|
-
readonly reason: {
|
|
98
|
-
readonly risk_level?: SafetySignal["risk_level"];
|
|
99
|
-
readonly signals?: ReadonlyArray<string>;
|
|
100
|
-
};
|
|
104
|
+
readonly fired_by?: string;
|
|
105
|
+
readonly reason: BlockReason;
|
|
101
106
|
readonly classifier_outputs: ClassifierCustomOutputs;
|
|
102
|
-
readonly audit: Pick<PipelineAudit, "
|
|
107
|
+
readonly audit: Pick<PipelineAudit, "prompt_injection" | "meta" | "fired_by" | "certainty_gate">;
|
|
103
108
|
};
|
|
104
109
|
export type RoutePipelineResult = {
|
|
105
110
|
readonly action: "route";
|
|
@@ -108,8 +113,11 @@ export type RoutePipelineResult = {
|
|
|
108
113
|
readonly classifier_outputs: ClassifierCustomOutputs;
|
|
109
114
|
readonly audit: PipelineAudit;
|
|
110
115
|
};
|
|
111
|
-
export type PipelineResult =
|
|
116
|
+
export type PipelineResult = ReplyPipelineResult | BlockPipelineResult | RoutePipelineResult;
|
|
112
117
|
export interface AggregatorConfig {
|
|
118
|
+
readonly certaintyThreshold?: number;
|
|
119
|
+
/** @deprecated Use certaintyThreshold. */
|
|
113
120
|
readonly confidenceThreshold?: number;
|
|
121
|
+
readonly certaintyGate?: CertaintyGateMode;
|
|
114
122
|
}
|
|
115
123
|
export type ClassifierRegistry = ReadonlyArray<RuntimeClassifierManifest>;
|
package/dist/src/manifest.js
CHANGED
package/dist/src/ollama.d.ts
CHANGED
|
@@ -1,8 +1,4 @@
|
|
|
1
1
|
import { type ClassifierName, type RunClassifier } from "./classifiers.js";
|
|
2
|
-
import { type OpenClassifyConfig } from "./config.js";
|
|
3
|
-
import { classifyOpenClassifyInput } from "./pipeline.js";
|
|
4
|
-
import type { Catalog } from "./manifest.js";
|
|
5
|
-
import type { OpenClassifyInput } from "./types.js";
|
|
6
2
|
export declare const OLLAMA_DEFAULT_HOST = "http://localhost:11434";
|
|
7
3
|
export declare const OLLAMA_BASE_MODEL = "gemma4:e4b-it-q4_K_M";
|
|
8
4
|
export declare const OLLAMA_BASE_MODEL_NATIVE_CONTEXT_LENGTH = 131072;
|
|
@@ -28,12 +24,6 @@ export interface OllamaClassifierRunnerConfig {
|
|
|
28
24
|
minAvailableMemoryBytes?: number;
|
|
29
25
|
minTotalMemoryBytes?: number;
|
|
30
26
|
}
|
|
31
|
-
export interface ClassifyWithOllamaConfig extends OllamaClassifierRunnerConfig {
|
|
32
|
-
catalog?: Catalog;
|
|
33
|
-
catalogPath?: string;
|
|
34
|
-
configPath?: string;
|
|
35
|
-
openClassifyConfig?: OpenClassifyConfig;
|
|
36
|
-
}
|
|
37
27
|
export declare class OllamaClassifierError extends Error {
|
|
38
28
|
readonly classifier: ClassifierName;
|
|
39
29
|
readonly model: string;
|
|
@@ -51,4 +41,3 @@ export declare function assertOllamaResources(options?: {
|
|
|
51
41
|
minTotalMemoryBytes?: number;
|
|
52
42
|
minAvailableMemoryBytes?: number;
|
|
53
43
|
}): Promise<void>;
|
|
54
|
-
export declare function classifyWithOllama(input: OpenClassifyInput, config?: ClassifyWithOllamaConfig): ReturnType<typeof classifyOpenClassifyInput>;
|
package/dist/src/ollama.js
CHANGED
|
@@ -10,10 +10,7 @@
|
|
|
10
10
|
// `classifyOpenClassifyInput` — you don't have to use this module at all.
|
|
11
11
|
import { execFile } from "node:child_process";
|
|
12
12
|
import { promisify } from "node:util";
|
|
13
|
-
import { loadCatalog } from "./catalog.js";
|
|
14
13
|
import { CLASSIFIER_NAMES, MODULES_BY_NAME, validateClassifierOutput, } from "./classifiers.js";
|
|
15
|
-
import { classifierModelsFromConfig, loadOpenClassifyConfig, } from "./config.js";
|
|
16
|
-
import { classifyOpenClassifyInput } from "./pipeline.js";
|
|
17
14
|
import { ClassifierValidationError, isRecord, } from "./validation.js";
|
|
18
15
|
export const OLLAMA_DEFAULT_HOST = "http://localhost:11434";
|
|
19
16
|
export const OLLAMA_BASE_MODEL = "gemma4:e4b-it-q4_K_M";
|
|
@@ -93,39 +90,6 @@ export async function assertOllamaResources(options = {}) {
|
|
|
93
90
|
throw new OllamaResourceError(totalMemoryBytes, availableMemoryBytes, minTotalMemoryBytes, minAvailableMemoryBytes);
|
|
94
91
|
}
|
|
95
92
|
}
|
|
96
|
-
export async function classifyWithOllama(input, config = {}) {
|
|
97
|
-
const fileConfig = config.openClassifyConfig ?? loadOpenClassifyConfig(config.configPath, {
|
|
98
|
-
optional: config.configPath === undefined && process.env.OPEN_CLASSIFY_CONFIG === undefined,
|
|
99
|
-
});
|
|
100
|
-
const runnerFileConfig = fileConfig?.runner;
|
|
101
|
-
const runnerConfig = {
|
|
102
|
-
...config,
|
|
103
|
-
host: config.host ?? runnerFileConfig?.host,
|
|
104
|
-
defaultModel: config.defaultModel ?? runnerFileConfig?.defaultModel,
|
|
105
|
-
models: {
|
|
106
|
-
...classifierModelsFromConfig(fileConfig),
|
|
107
|
-
...config.models,
|
|
108
|
-
},
|
|
109
|
-
options: {
|
|
110
|
-
...runnerFileConfig?.options,
|
|
111
|
-
...config.options,
|
|
112
|
-
},
|
|
113
|
-
};
|
|
114
|
-
if (!runnerConfig.skipResourceCheck) {
|
|
115
|
-
await assertOllamaResources({
|
|
116
|
-
minTotalMemoryBytes: runnerConfig.minTotalMemoryBytes,
|
|
117
|
-
minAvailableMemoryBytes: runnerConfig.minAvailableMemoryBytes,
|
|
118
|
-
});
|
|
119
|
-
Object.assign(runnerConfig, {
|
|
120
|
-
skipResourceCheck: true,
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
const catalog = config.catalog ?? loadCatalog(config.catalogPath ?? fileConfig?.catalog ?? OLLAMA_DEFAULT_CATALOG_PATH);
|
|
124
|
-
return classifyOpenClassifyInput(input, {
|
|
125
|
-
runClassifier: createOllamaClassifierRunner(runnerConfig),
|
|
126
|
-
catalog,
|
|
127
|
-
});
|
|
128
|
-
}
|
|
129
93
|
async function runOllamaClassifier(name, input, signal, fetchImpl, host, model, options, allowManifestModel) {
|
|
130
94
|
const module_ = MODULES_BY_NAME[name];
|
|
131
95
|
const systemPrompt = module_.systemPrompt;
|
package/dist/src/pipeline.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { AggregatorConfig, Catalog, PipelineResult } from "./manifest.js";
|
|
|
3
3
|
import type { OpenClassifyInput } from "./types.js";
|
|
4
4
|
export declare const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15000;
|
|
5
5
|
export declare const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
|
|
6
|
+
export declare const DEFAULT_CERTAINTY_GATE = "min_score";
|
|
6
7
|
export declare class OpenClassifyNormalizationError extends Error {
|
|
7
8
|
constructor(cause: unknown);
|
|
8
9
|
}
|
package/dist/src/pipeline.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { composeEnvelope } from "./aggregator.js";
|
|
1
|
+
import { certaintyThreshold, composeEnvelope } from "./aggregator.js";
|
|
2
2
|
import { CLASSIFIER_NAMES, MODULES_BY_NAME, REGISTRY, } from "./classifiers.js";
|
|
3
3
|
import { normalizeOpenClassifyInput, toClassifierInput } from "./input.js";
|
|
4
|
-
import { isCustomManifest } from "./stock.js";
|
|
4
|
+
import { certaintyScore, isCustomManifest } from "./stock.js";
|
|
5
5
|
export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15_000;
|
|
6
6
|
export const DEFAULT_CLASSIFIER_RETRY_COUNT = 1;
|
|
7
|
+
export const DEFAULT_CERTAINTY_GATE = "min_score";
|
|
7
8
|
export class OpenClassifyNormalizationError extends Error {
|
|
8
9
|
constructor(cause) {
|
|
9
10
|
super(errorMessage(cause), { cause });
|
|
@@ -11,10 +12,10 @@ export class OpenClassifyNormalizationError extends Error {
|
|
|
11
12
|
}
|
|
12
13
|
}
|
|
13
14
|
// Short-circuit gates are intrinsic to specific stock signals — not configured
|
|
14
|
-
// per-manifest. preflight.final_reply ⇒
|
|
15
|
-
//
|
|
15
|
+
// per-manifest. preflight.final_reply ⇒ reply; confident high_risk or unknown
|
|
16
|
+
// prompt-injection risk ⇒ block. Order matters: preflight is
|
|
16
17
|
// cheaper to evaluate, so we check it first.
|
|
17
|
-
const SHORT_CIRCUIT_GATES = ["preflight", "
|
|
18
|
+
const SHORT_CIRCUIT_GATES = ["preflight", "prompt_injection"];
|
|
18
19
|
export async function classifyOpenClassifyInput(input, options) {
|
|
19
20
|
let request;
|
|
20
21
|
try {
|
|
@@ -36,7 +37,7 @@ export async function classifyOpenClassifyInput(input, options) {
|
|
|
36
37
|
const classifierInput = toClassifierInput(request);
|
|
37
38
|
const classifierTimeoutMs = options.classifierTimeoutMs ?? DEFAULT_CLASSIFIER_TIMEOUT_MS;
|
|
38
39
|
const classifierRetryCount = options.classifierRetryCount ?? DEFAULT_CLASSIFIER_RETRY_COUNT;
|
|
39
|
-
const threshold = options.aggregator
|
|
40
|
+
const threshold = certaintyThreshold(options.aggregator);
|
|
40
41
|
const runs = new Map(CLASSIFIER_NAMES.map((name) => [
|
|
41
42
|
name,
|
|
42
43
|
runClassifierWithRetry(name, classifierInput, options.runClassifier, controller.signal, classifierTimeoutMs, classifierRetryCount),
|
|
@@ -65,6 +66,10 @@ export async function classifyOpenClassifyInput(input, options) {
|
|
|
65
66
|
input: classifierInput,
|
|
66
67
|
config: options.aggregator,
|
|
67
68
|
});
|
|
69
|
+
const certaintyGate = certaintyGateBlock(options.aggregator, results);
|
|
70
|
+
if (certaintyGate) {
|
|
71
|
+
return buildCertaintyGateBlockResult(request, envelope, results, meta, certaintyGate);
|
|
72
|
+
}
|
|
68
73
|
return buildRouteResult(request, envelope, results, meta);
|
|
69
74
|
}
|
|
70
75
|
finally {
|
|
@@ -72,38 +77,67 @@ export async function classifyOpenClassifyInput(input, options) {
|
|
|
72
77
|
}
|
|
73
78
|
}
|
|
74
79
|
function shortCircuitVerdict(gate, result, threshold) {
|
|
75
|
-
const
|
|
76
|
-
if (
|
|
80
|
+
const score = scoreCertainty(result.certainty);
|
|
81
|
+
if (score < threshold)
|
|
77
82
|
return null;
|
|
78
83
|
if (gate === "preflight") {
|
|
79
84
|
const preflight = result;
|
|
80
85
|
if (preflight.final_reply !== undefined) {
|
|
81
|
-
return { kind: "
|
|
86
|
+
return { kind: "reply", final_reply: preflight.final_reply };
|
|
82
87
|
}
|
|
83
88
|
return null;
|
|
84
89
|
}
|
|
85
|
-
if (gate === "
|
|
86
|
-
const
|
|
87
|
-
if (
|
|
90
|
+
if (gate === "prompt_injection") {
|
|
91
|
+
const promptInjection = result;
|
|
92
|
+
if (promptInjection.risk_level === "high_risk" || promptInjection.risk_level === "unknown") {
|
|
93
|
+
const promptInjectionSignal = extractPromptInjection(promptInjection);
|
|
88
94
|
return {
|
|
89
95
|
kind: "block",
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
kind: "needs_review",
|
|
96
|
-
safety: extractSafety(security),
|
|
96
|
+
prompt_injection: promptInjectionSignal,
|
|
97
|
+
reason: {
|
|
98
|
+
kind: "prompt_injection",
|
|
99
|
+
risk_level: promptInjectionSignal.risk_level,
|
|
100
|
+
},
|
|
97
101
|
};
|
|
98
102
|
}
|
|
99
103
|
}
|
|
100
104
|
return null;
|
|
101
105
|
}
|
|
102
|
-
function
|
|
106
|
+
function certaintyGateBlock(config, results) {
|
|
107
|
+
const mode = config?.certaintyGate ?? DEFAULT_CERTAINTY_GATE;
|
|
108
|
+
if (mode === "off")
|
|
109
|
+
return undefined;
|
|
110
|
+
const threshold = certaintyThreshold(config);
|
|
111
|
+
const classifier_scores = classifierScores(results);
|
|
112
|
+
const scores = Object.values(classifier_scores);
|
|
113
|
+
const score = mode === "min_score"
|
|
114
|
+
? Math.min(...scores)
|
|
115
|
+
: scores.reduce((sum, value) => sum + value, 0) / scores.length;
|
|
116
|
+
if (score >= threshold)
|
|
117
|
+
return undefined;
|
|
118
|
+
return {
|
|
119
|
+
kind: "low_certainty",
|
|
120
|
+
mode,
|
|
121
|
+
threshold,
|
|
122
|
+
score,
|
|
123
|
+
classifier_scores,
|
|
124
|
+
low_classifiers: Object.entries(classifier_scores)
|
|
125
|
+
.filter(([, value]) => value < threshold)
|
|
126
|
+
.map(([name]) => name),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function classifierScores(results) {
|
|
130
|
+
return Object.fromEntries(REGISTRY.map((manifest) => [
|
|
131
|
+
manifest.name,
|
|
132
|
+
scoreCertainty(results[manifest.name]?.certainty),
|
|
133
|
+
]));
|
|
134
|
+
}
|
|
135
|
+
function scoreCertainty(certainty) {
|
|
136
|
+
return certainty === undefined ? 0 : certaintyScore[certainty];
|
|
137
|
+
}
|
|
138
|
+
function extractPromptInjection(value) {
|
|
103
139
|
return {
|
|
104
|
-
...(value.decision === undefined ? {} : { decision: value.decision }),
|
|
105
140
|
risk_level: value.risk_level,
|
|
106
|
-
signals: value.signals,
|
|
107
141
|
};
|
|
108
142
|
}
|
|
109
143
|
function buildShortCircuitResult(name, verdict, settled, target_message_hash) {
|
|
@@ -116,13 +150,13 @@ function buildShortCircuitResult(name, verdict, settled, target_message_hash) {
|
|
|
116
150
|
};
|
|
117
151
|
const meta = { classifiers: { [name]: entry } };
|
|
118
152
|
const classifier_outputs = classifierCustomOutputs({ [name]: value });
|
|
119
|
-
if (verdict.kind === "
|
|
153
|
+
if (verdict.kind === "reply") {
|
|
120
154
|
const preflight = value;
|
|
121
155
|
return {
|
|
122
|
-
action: "
|
|
156
|
+
action: "reply",
|
|
123
157
|
message_id: target_message_hash,
|
|
124
|
-
|
|
125
|
-
reason: "
|
|
158
|
+
reply: { text: verdict.final_reply.reply },
|
|
159
|
+
reason: "preflight_reply",
|
|
126
160
|
classifier_outputs,
|
|
127
161
|
audit: {
|
|
128
162
|
fired_by: name,
|
|
@@ -131,34 +165,15 @@ function buildShortCircuitResult(name, verdict, settled, target_message_hash) {
|
|
|
131
165
|
},
|
|
132
166
|
};
|
|
133
167
|
}
|
|
134
|
-
if (verdict.kind === "needs_review") {
|
|
135
|
-
return {
|
|
136
|
-
action: "needs_review",
|
|
137
|
-
message_id: target_message_hash,
|
|
138
|
-
fired_by: name,
|
|
139
|
-
reason: {
|
|
140
|
-
risk_level: verdict.safety.risk_level,
|
|
141
|
-
signals: verdict.safety.signals,
|
|
142
|
-
},
|
|
143
|
-
classifier_outputs,
|
|
144
|
-
audit: {
|
|
145
|
-
fired_by: name,
|
|
146
|
-
safety: verdict.safety,
|
|
147
|
-
meta,
|
|
148
|
-
},
|
|
149
|
-
};
|
|
150
|
-
}
|
|
151
168
|
return {
|
|
152
169
|
action: "block",
|
|
153
170
|
message_id: target_message_hash,
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
signals: verdict.safety.signals,
|
|
157
|
-
},
|
|
171
|
+
fired_by: name,
|
|
172
|
+
reason: verdict.reason,
|
|
158
173
|
classifier_outputs,
|
|
159
174
|
audit: {
|
|
160
175
|
fired_by: name,
|
|
161
|
-
|
|
176
|
+
prompt_injection: verdict.prompt_injection,
|
|
162
177
|
meta,
|
|
163
178
|
},
|
|
164
179
|
};
|
|
@@ -199,6 +214,21 @@ function buildRouteResult(request, envelope, results, meta) {
|
|
|
199
214
|
},
|
|
200
215
|
};
|
|
201
216
|
}
|
|
217
|
+
function buildCertaintyGateBlockResult(request, envelope, results, meta, certaintyGate) {
|
|
218
|
+
return {
|
|
219
|
+
action: "block",
|
|
220
|
+
message_id: request.target_message_hash,
|
|
221
|
+
fired_by: "certainty_gate",
|
|
222
|
+
reason: certaintyGate,
|
|
223
|
+
classifier_outputs: classifierCustomOutputs(results),
|
|
224
|
+
audit: {
|
|
225
|
+
...envelope,
|
|
226
|
+
fired_by: "certainty_gate",
|
|
227
|
+
certainty_gate: certaintyGate,
|
|
228
|
+
meta,
|
|
229
|
+
},
|
|
230
|
+
};
|
|
231
|
+
}
|
|
202
232
|
function classifierCustomOutputs(results) {
|
|
203
233
|
const out = {};
|
|
204
234
|
for (const manifest of REGISTRY) {
|
package/dist/src/stock-prompt.js
CHANGED
|
@@ -26,7 +26,7 @@ function stockSection(manifest) {
|
|
|
26
26
|
allowed_tools: renderAllowedTools(manifest.tools),
|
|
27
27
|
preflight_output: promptMarkdown("preflight-output.md"),
|
|
28
28
|
routing_output: promptMarkdown("routing-output.md"),
|
|
29
|
-
|
|
29
|
+
prompt_injection_output: promptMarkdown("prompt-injection-output.md"),
|
|
30
30
|
specialty: promptMarkdown("specialty.md"),
|
|
31
31
|
tier: promptMarkdown("tier.md"),
|
|
32
32
|
tools_output: promptMarkdown("tools-output.md"),
|