@framers/agentos-ext-ml-classifiers 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +23 -0
- package/dist/ClassifierOrchestrator.d.ts +126 -0
- package/dist/ClassifierOrchestrator.d.ts.map +1 -0
- package/dist/ClassifierOrchestrator.js +239 -0
- package/dist/ClassifierOrchestrator.js.map +1 -0
- package/dist/IContentClassifier.d.ts +117 -0
- package/dist/IContentClassifier.d.ts.map +1 -0
- package/dist/IContentClassifier.js +22 -0
- package/dist/IContentClassifier.js.map +1 -0
- package/dist/MLClassifierGuardrail.d.ts +163 -0
- package/dist/MLClassifierGuardrail.d.ts.map +1 -0
- package/dist/MLClassifierGuardrail.js +335 -0
- package/dist/MLClassifierGuardrail.js.map +1 -0
- package/dist/SlidingWindowBuffer.d.ts +213 -0
- package/dist/SlidingWindowBuffer.d.ts.map +1 -0
- package/dist/SlidingWindowBuffer.js +246 -0
- package/dist/SlidingWindowBuffer.js.map +1 -0
- package/dist/classifiers/InjectionClassifier.d.ts +126 -0
- package/dist/classifiers/InjectionClassifier.d.ts.map +1 -0
- package/dist/classifiers/InjectionClassifier.js +210 -0
- package/dist/classifiers/InjectionClassifier.js.map +1 -0
- package/dist/classifiers/JailbreakClassifier.d.ts +124 -0
- package/dist/classifiers/JailbreakClassifier.d.ts.map +1 -0
- package/dist/classifiers/JailbreakClassifier.js +208 -0
- package/dist/classifiers/JailbreakClassifier.js.map +1 -0
- package/dist/classifiers/ToxicityClassifier.d.ts +125 -0
- package/dist/classifiers/ToxicityClassifier.d.ts.map +1 -0
- package/dist/classifiers/ToxicityClassifier.js +212 -0
- package/dist/classifiers/ToxicityClassifier.js.map +1 -0
- package/dist/classifiers/WorkerClassifierProxy.d.ts +158 -0
- package/dist/classifiers/WorkerClassifierProxy.d.ts.map +1 -0
- package/dist/classifiers/WorkerClassifierProxy.js +268 -0
- package/dist/classifiers/WorkerClassifierProxy.js.map +1 -0
- package/dist/index.d.ts +110 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +342 -0
- package/dist/index.js.map +1 -0
- package/dist/tools/ClassifyContentTool.d.ts +105 -0
- package/dist/tools/ClassifyContentTool.d.ts.map +1 -0
- package/dist/tools/ClassifyContentTool.js +149 -0
- package/dist/tools/ClassifyContentTool.js.map +1 -0
- package/dist/types.d.ts +319 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +62 -0
- package/dist/types.js.map +1 -0
- package/dist/worker/classifier-worker.d.ts +49 -0
- package/dist/worker/classifier-worker.d.ts.map +1 -0
- package/dist/worker/classifier-worker.js +180 -0
- package/dist/worker/classifier-worker.js.map +1 -0
- package/package.json +45 -0
- package/src/ClassifierOrchestrator.ts +290 -0
- package/src/IContentClassifier.ts +124 -0
- package/src/MLClassifierGuardrail.ts +419 -0
- package/src/SlidingWindowBuffer.ts +384 -0
- package/src/classifiers/InjectionClassifier.ts +261 -0
- package/src/classifiers/JailbreakClassifier.ts +259 -0
- package/src/classifiers/ToxicityClassifier.ts +263 -0
- package/src/classifiers/WorkerClassifierProxy.ts +366 -0
- package/src/index.ts +383 -0
- package/src/tools/ClassifyContentTool.ts +201 -0
- package/src/types.ts +391 -0
- package/src/worker/classifier-worker.ts +267 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Framers
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
23
|
+
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Orchestrator for parallel ML classifier execution with worst-wins aggregation.
|
|
3
|
+
*
|
|
4
|
+
* The `ClassifierOrchestrator` runs all registered {@link IContentClassifier}
|
|
5
|
+
* instances in parallel against a single text input and aggregates their
|
|
6
|
+
* results into a single {@link ChunkEvaluation}. The aggregation policy is
|
|
7
|
+
* **worst-wins**: if any classifier recommends BLOCK the overall result is
|
|
8
|
+
* BLOCK, even if every other classifier returned ALLOW.
|
|
9
|
+
*
|
|
10
|
+
* Priority order (descending):
|
|
11
|
+
* ```
|
|
12
|
+
* BLOCK > FLAG > SANITIZE > ALLOW
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Each classifier may have its own threshold overrides (via
|
|
16
|
+
* `perClassifierThresholds`), and individual labels can be mapped to
|
|
17
|
+
* hard-coded actions via `ClassifierConfig.labelActions`.
|
|
18
|
+
*
|
|
19
|
+
* @module agentos/extensions/packs/ml-classifiers/ClassifierOrchestrator
|
|
20
|
+
*/
|
|
21
|
+
import type { IContentClassifier } from './IContentClassifier';
|
|
22
|
+
import type { ChunkEvaluation, ClassifierThresholds } from './types';
|
|
23
|
+
/**
|
|
24
|
+
* Drives all registered ML classifiers in parallel and folds their results
|
|
25
|
+
* into a single {@link ChunkEvaluation} using worst-wins aggregation.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```typescript
|
|
29
|
+
* const orchestrator = new ClassifierOrchestrator(
|
|
30
|
+
* [toxicityClassifier, injectionClassifier],
|
|
31
|
+
* DEFAULT_THRESHOLDS,
|
|
32
|
+
* );
|
|
33
|
+
*
|
|
34
|
+
* const evaluation = await orchestrator.classifyAll('some user message');
|
|
35
|
+
* if (evaluation.recommendedAction === GuardrailAction.BLOCK) {
|
|
36
|
+
* // Terminate the interaction.
|
|
37
|
+
* }
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export declare class ClassifierOrchestrator {
|
|
41
|
+
/** Immutable list of classifiers to run on every `classifyAll()` call. */
|
|
42
|
+
private readonly classifiers;
|
|
43
|
+
/** Merged default thresholds (pack-level defaults + caller overrides). */
|
|
44
|
+
private readonly defaultThresholds;
|
|
45
|
+
/**
|
|
46
|
+
* Optional per-classifier threshold overrides, keyed by classifier ID.
|
|
47
|
+
* When a classifier's ID appears here, the partial thresholds are merged
|
|
48
|
+
* on top of {@link defaultThresholds} for that classifier only.
|
|
49
|
+
*/
|
|
50
|
+
private readonly perClassifierThresholds;
|
|
51
|
+
/**
|
|
52
|
+
* Create a new orchestrator.
|
|
53
|
+
*
|
|
54
|
+
* @param classifiers - Array of classifier instances to run in parallel.
|
|
55
|
+
* @param defaultThresholds - Pack-level threshold defaults applied to every classifier
|
|
56
|
+
* unless overridden by `perClassifierThresholds`.
|
|
57
|
+
* @param perClassifierThresholds - Optional map from classifier ID to partial threshold
|
|
58
|
+
* overrides. Missing fields fall back to `defaultThresholds`.
|
|
59
|
+
*/
|
|
60
|
+
constructor(classifiers: IContentClassifier[], defaultThresholds?: ClassifierThresholds, perClassifierThresholds?: Record<string, Partial<ClassifierThresholds>>);
|
|
61
|
+
/**
|
|
62
|
+
* Classify `text` against every registered classifier in parallel and
|
|
63
|
+
* return the aggregated {@link ChunkEvaluation}.
|
|
64
|
+
*
|
|
65
|
+
* Execution details:
|
|
66
|
+
* 1. All classifiers run concurrently via `Promise.allSettled`.
|
|
67
|
+
* 2. Fulfilled results are wrapped as {@link AnnotatedClassificationResult}
|
|
68
|
+
* with provenance metadata (`classifierId`, `latencyMs`).
|
|
69
|
+
* 3. Rejected promises log a warning and contribute an implicit ALLOW so
|
|
70
|
+
* a single broken classifier does not block all content.
|
|
71
|
+
* 4. Each result is mapped to a {@link GuardrailAction} using
|
|
72
|
+
* per-classifier thresholds (if configured) or the pack defaults.
|
|
73
|
+
* 5. The final `recommendedAction` is the most restrictive action across
|
|
74
|
+
* all classifiers (worst-wins).
|
|
75
|
+
*
|
|
76
|
+
* @param text - The text to evaluate. Must not be empty.
|
|
77
|
+
* @returns A promise resolving to the aggregated evaluation result.
|
|
78
|
+
*/
|
|
79
|
+
classifyAll(text: string): Promise<ChunkEvaluation>;
|
|
80
|
+
/**
|
|
81
|
+
* Dispose every registered classifier, releasing model weights and any
|
|
82
|
+
* other resources they hold.
|
|
83
|
+
*
|
|
84
|
+
* Calls each classifier's `dispose()` method (if present) and swallows
|
|
85
|
+
* errors so a single failing classifier does not prevent cleanup of the
|
|
86
|
+
* others.
|
|
87
|
+
*/
|
|
88
|
+
dispose(): Promise<void>;
|
|
89
|
+
/**
|
|
90
|
+
* Invoke a single classifier with wall-clock latency tracking.
|
|
91
|
+
*
|
|
92
|
+
* Wraps `classifier.classify(text)` and returns the raw
|
|
93
|
+
* {@link ClassificationResult} augmented with `classifierId` and
|
|
94
|
+
* `latencyMs` fields.
|
|
95
|
+
*
|
|
96
|
+
* @param classifier - The classifier to invoke.
|
|
97
|
+
* @param text - The text to classify.
|
|
98
|
+
* @returns An annotated result with provenance metadata.
|
|
99
|
+
*/
|
|
100
|
+
private timedClassify;
|
|
101
|
+
/**
|
|
102
|
+
* Map a classifier's confidence score to a {@link GuardrailAction}.
|
|
103
|
+
*
|
|
104
|
+
* The mapping checks `labelActions` first (from per-classifier config in
|
|
105
|
+
* thresholds), then falls back to numeric threshold comparison:
|
|
106
|
+
*
|
|
107
|
+
* 1. `confidence >= blockThreshold` -> BLOCK
|
|
108
|
+
* 2. `confidence >= flagThreshold` -> FLAG
|
|
109
|
+
* 3. `confidence >= warnThreshold` -> SANITIZE
|
|
110
|
+
* 4. otherwise -> ALLOW
|
|
111
|
+
*
|
|
112
|
+
* @param result - The annotated classification result.
|
|
113
|
+
* @param thresholds - Resolved thresholds for this classifier.
|
|
114
|
+
* @returns The appropriate guardrail action.
|
|
115
|
+
*/
|
|
116
|
+
private scoreToAction;
|
|
117
|
+
/**
|
|
118
|
+
* Resolve the effective thresholds for a given classifier by merging
|
|
119
|
+
* per-classifier overrides on top of the pack-level defaults.
|
|
120
|
+
*
|
|
121
|
+
* @param classifierId - ID of the classifier to resolve thresholds for.
|
|
122
|
+
* @returns Fully-resolved thresholds with no undefined fields.
|
|
123
|
+
*/
|
|
124
|
+
private resolveThresholds;
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=ClassifierOrchestrator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ClassifierOrchestrator.d.ts","sourceRoot":"","sources":["../src/ClassifierOrchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAEV,eAAe,EACf,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAwBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,sBAAsB;IAKjC,0EAA0E;IAC1E,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuB;IAEnD,0EAA0E;IAC1E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAuB;IAEzD;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAgD;IAMxF;;;;;;;;OAQG;gBAED,WAAW,EAAE,kBAAkB,EAAE,EACjC,iBAAiB,GAAE,oBAAyC,EAC5D,uBAAuB,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAM;IAW7E;;;;;;;;;;;;;;;;;OAiBG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAoDzD;;;;;;;OAOG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAc9B;;;;;;;;;;OAUG;YACW,aAAa;IAe3B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,aAAa;IAwBrB;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;CAY1B"}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Orchestrator for parallel ML classifier execution with worst-wins aggregation.
|
|
3
|
+
*
|
|
4
|
+
* The `ClassifierOrchestrator` runs all registered {@link IContentClassifier}
|
|
5
|
+
* instances in parallel against a single text input and aggregates their
|
|
6
|
+
* results into a single {@link ChunkEvaluation}. The aggregation policy is
|
|
7
|
+
* **worst-wins**: if any classifier recommends BLOCK the overall result is
|
|
8
|
+
* BLOCK, even if every other classifier returned ALLOW.
|
|
9
|
+
*
|
|
10
|
+
* Priority order (descending):
|
|
11
|
+
* ```
|
|
12
|
+
* BLOCK > FLAG > SANITIZE > ALLOW
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Each classifier may have its own threshold overrides (via
|
|
16
|
+
* `perClassifierThresholds`), and individual labels can be mapped to
|
|
17
|
+
* hard-coded actions via `ClassifierConfig.labelActions`.
|
|
18
|
+
*
|
|
19
|
+
* @module agentos/extensions/packs/ml-classifiers/ClassifierOrchestrator
|
|
20
|
+
*/
|
|
21
|
+
import { DEFAULT_THRESHOLDS } from './types';
|
|
22
|
+
import { GuardrailAction } from '@framers/agentos';
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
// Action severity ranking — used by worst-wins aggregation
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
/**
|
|
27
|
+
* Numeric severity for each {@link GuardrailAction}, where higher values
|
|
28
|
+
* represent more restrictive actions. Used to implement the worst-wins
|
|
29
|
+
* comparison without brittle string ordering.
|
|
30
|
+
*/
|
|
31
|
+
const ACTION_SEVERITY = {
|
|
32
|
+
[GuardrailAction.ALLOW]: 0,
|
|
33
|
+
[GuardrailAction.SANITIZE]: 1,
|
|
34
|
+
[GuardrailAction.FLAG]: 2,
|
|
35
|
+
[GuardrailAction.BLOCK]: 3,
|
|
36
|
+
};
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// ClassifierOrchestrator
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
/**
|
|
41
|
+
* Drives all registered ML classifiers in parallel and folds their results
|
|
42
|
+
* into a single {@link ChunkEvaluation} using worst-wins aggregation.
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* ```typescript
|
|
46
|
+
* const orchestrator = new ClassifierOrchestrator(
|
|
47
|
+
* [toxicityClassifier, injectionClassifier],
|
|
48
|
+
* DEFAULT_THRESHOLDS,
|
|
49
|
+
* );
|
|
50
|
+
*
|
|
51
|
+
* const evaluation = await orchestrator.classifyAll('some user message');
|
|
52
|
+
* if (evaluation.recommendedAction === GuardrailAction.BLOCK) {
|
|
53
|
+
* // Terminate the interaction.
|
|
54
|
+
* }
|
|
55
|
+
* ```
|
|
56
|
+
*/
|
|
57
|
+
export class ClassifierOrchestrator {
|
|
58
|
+
// -------------------------------------------------------------------------
|
|
59
|
+
// Private state
|
|
60
|
+
// -------------------------------------------------------------------------
|
|
61
|
+
/** Immutable list of classifiers to run on every `classifyAll()` call. */
|
|
62
|
+
classifiers;
|
|
63
|
+
/** Merged default thresholds (pack-level defaults + caller overrides). */
|
|
64
|
+
defaultThresholds;
|
|
65
|
+
/**
|
|
66
|
+
* Optional per-classifier threshold overrides, keyed by classifier ID.
|
|
67
|
+
* When a classifier's ID appears here, the partial thresholds are merged
|
|
68
|
+
* on top of {@link defaultThresholds} for that classifier only.
|
|
69
|
+
*/
|
|
70
|
+
perClassifierThresholds;
|
|
71
|
+
// -------------------------------------------------------------------------
|
|
72
|
+
// Constructor
|
|
73
|
+
// -------------------------------------------------------------------------
|
|
74
|
+
/**
|
|
75
|
+
* Create a new orchestrator.
|
|
76
|
+
*
|
|
77
|
+
* @param classifiers - Array of classifier instances to run in parallel.
|
|
78
|
+
* @param defaultThresholds - Pack-level threshold defaults applied to every classifier
|
|
79
|
+
* unless overridden by `perClassifierThresholds`.
|
|
80
|
+
* @param perClassifierThresholds - Optional map from classifier ID to partial threshold
|
|
81
|
+
* overrides. Missing fields fall back to `defaultThresholds`.
|
|
82
|
+
*/
|
|
83
|
+
constructor(classifiers, defaultThresholds = DEFAULT_THRESHOLDS, perClassifierThresholds = {}) {
|
|
84
|
+
this.classifiers = classifiers;
|
|
85
|
+
this.defaultThresholds = defaultThresholds;
|
|
86
|
+
this.perClassifierThresholds = perClassifierThresholds;
|
|
87
|
+
}
|
|
88
|
+
// -------------------------------------------------------------------------
|
|
89
|
+
// Public API
|
|
90
|
+
// -------------------------------------------------------------------------
|
|
91
|
+
/**
|
|
92
|
+
* Classify `text` against every registered classifier in parallel and
|
|
93
|
+
* return the aggregated {@link ChunkEvaluation}.
|
|
94
|
+
*
|
|
95
|
+
* Execution details:
|
|
96
|
+
* 1. All classifiers run concurrently via `Promise.allSettled`.
|
|
97
|
+
* 2. Fulfilled results are wrapped as {@link AnnotatedClassificationResult}
|
|
98
|
+
* with provenance metadata (`classifierId`, `latencyMs`).
|
|
99
|
+
* 3. Rejected promises log a warning and contribute an implicit ALLOW so
|
|
100
|
+
* a single broken classifier does not block all content.
|
|
101
|
+
* 4. Each result is mapped to a {@link GuardrailAction} using
|
|
102
|
+
* per-classifier thresholds (if configured) or the pack defaults.
|
|
103
|
+
* 5. The final `recommendedAction` is the most restrictive action across
|
|
104
|
+
* all classifiers (worst-wins).
|
|
105
|
+
*
|
|
106
|
+
* @param text - The text to evaluate. Must not be empty.
|
|
107
|
+
* @returns A promise resolving to the aggregated evaluation result.
|
|
108
|
+
*/
|
|
109
|
+
async classifyAll(text) {
|
|
110
|
+
// Record wall-clock start time so `totalLatencyMs` reflects the
|
|
111
|
+
// real-world time spent, not the sum of sequential latencies.
|
|
112
|
+
const wallStart = performance.now();
|
|
113
|
+
// Fire all classifiers in parallel and wait for every one to settle.
|
|
114
|
+
const settled = await Promise.allSettled(this.classifiers.map((c) => this.timedClassify(c, text)));
|
|
115
|
+
// Accumulate annotated results and track the worst action seen.
|
|
116
|
+
const results = [];
|
|
117
|
+
let worstAction = GuardrailAction.ALLOW;
|
|
118
|
+
let triggeredBy = null;
|
|
119
|
+
for (let i = 0; i < settled.length; i++) {
|
|
120
|
+
const outcome = settled[i];
|
|
121
|
+
const classifier = this.classifiers[i];
|
|
122
|
+
if (outcome.status === 'fulfilled') {
|
|
123
|
+
const annotated = outcome.value;
|
|
124
|
+
results.push(annotated);
|
|
125
|
+
// Resolve the thresholds for this specific classifier.
|
|
126
|
+
const thresholds = this.resolveThresholds(classifier.id);
|
|
127
|
+
// Map the raw confidence score to a guardrail action.
|
|
128
|
+
const action = this.scoreToAction(annotated, thresholds);
|
|
129
|
+
// Worst-wins: keep the most restrictive action.
|
|
130
|
+
if (ACTION_SEVERITY[action] > ACTION_SEVERITY[worstAction]) {
|
|
131
|
+
worstAction = action;
|
|
132
|
+
triggeredBy = classifier.id;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
// Classifier failed — log and contribute an implicit ALLOW.
|
|
137
|
+
console.warn(`[ClassifierOrchestrator] Classifier "${classifier.id}" failed: ${outcome.reason}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const wallEnd = performance.now();
|
|
141
|
+
return {
|
|
142
|
+
results,
|
|
143
|
+
recommendedAction: worstAction,
|
|
144
|
+
triggeredBy,
|
|
145
|
+
totalLatencyMs: Math.round(wallEnd - wallStart),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Dispose every registered classifier, releasing model weights and any
|
|
150
|
+
* other resources they hold.
|
|
151
|
+
*
|
|
152
|
+
* Calls each classifier's `dispose()` method (if present) and swallows
|
|
153
|
+
* errors so a single failing classifier does not prevent cleanup of the
|
|
154
|
+
* others.
|
|
155
|
+
*/
|
|
156
|
+
async dispose() {
|
|
157
|
+
await Promise.allSettled(this.classifiers.map(async (c) => {
|
|
158
|
+
if (c.dispose) {
|
|
159
|
+
await c.dispose();
|
|
160
|
+
}
|
|
161
|
+
}));
|
|
162
|
+
}
|
|
163
|
+
// -------------------------------------------------------------------------
|
|
164
|
+
// Private helpers
|
|
165
|
+
// -------------------------------------------------------------------------
|
|
166
|
+
/**
|
|
167
|
+
* Invoke a single classifier with wall-clock latency tracking.
|
|
168
|
+
*
|
|
169
|
+
* Wraps `classifier.classify(text)` and returns the raw
|
|
170
|
+
* {@link ClassificationResult} augmented with `classifierId` and
|
|
171
|
+
* `latencyMs` fields.
|
|
172
|
+
*
|
|
173
|
+
* @param classifier - The classifier to invoke.
|
|
174
|
+
* @param text - The text to classify.
|
|
175
|
+
* @returns An annotated result with provenance metadata.
|
|
176
|
+
*/
|
|
177
|
+
async timedClassify(classifier, text) {
|
|
178
|
+
const start = performance.now();
|
|
179
|
+
const result = await classifier.classify(text);
|
|
180
|
+
const latencyMs = Math.round(performance.now() - start);
|
|
181
|
+
return {
|
|
182
|
+
...result,
|
|
183
|
+
classifierId: classifier.id,
|
|
184
|
+
latencyMs,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Map a classifier's confidence score to a {@link GuardrailAction}.
|
|
189
|
+
*
|
|
190
|
+
* The mapping checks `labelActions` first (from per-classifier config in
|
|
191
|
+
* thresholds), then falls back to numeric threshold comparison:
|
|
192
|
+
*
|
|
193
|
+
* 1. `confidence >= blockThreshold` -> BLOCK
|
|
194
|
+
* 2. `confidence >= flagThreshold` -> FLAG
|
|
195
|
+
* 3. `confidence >= warnThreshold` -> SANITIZE
|
|
196
|
+
* 4. otherwise -> ALLOW
|
|
197
|
+
*
|
|
198
|
+
* @param result - The annotated classification result.
|
|
199
|
+
* @param thresholds - Resolved thresholds for this classifier.
|
|
200
|
+
* @returns The appropriate guardrail action.
|
|
201
|
+
*/
|
|
202
|
+
scoreToAction(result, thresholds) {
|
|
203
|
+
// Extract the confidence as a single number.
|
|
204
|
+
// ClassificationResult.confidence may be number | number[]; normalise.
|
|
205
|
+
const confidence = Array.isArray(result.confidence)
|
|
206
|
+
? result.confidence[0] ?? 0
|
|
207
|
+
: result.confidence;
|
|
208
|
+
// Threshold comparison — checked in descending severity order.
|
|
209
|
+
if (confidence >= thresholds.blockThreshold) {
|
|
210
|
+
return GuardrailAction.BLOCK;
|
|
211
|
+
}
|
|
212
|
+
if (confidence >= thresholds.flagThreshold) {
|
|
213
|
+
return GuardrailAction.FLAG;
|
|
214
|
+
}
|
|
215
|
+
if (confidence >= thresholds.warnThreshold) {
|
|
216
|
+
return GuardrailAction.SANITIZE;
|
|
217
|
+
}
|
|
218
|
+
return GuardrailAction.ALLOW;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Resolve the effective thresholds for a given classifier by merging
|
|
222
|
+
* per-classifier overrides on top of the pack-level defaults.
|
|
223
|
+
*
|
|
224
|
+
* @param classifierId - ID of the classifier to resolve thresholds for.
|
|
225
|
+
* @returns Fully-resolved thresholds with no undefined fields.
|
|
226
|
+
*/
|
|
227
|
+
resolveThresholds(classifierId) {
|
|
228
|
+
const overrides = this.perClassifierThresholds[classifierId];
|
|
229
|
+
if (!overrides) {
|
|
230
|
+
return this.defaultThresholds;
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
blockThreshold: overrides.blockThreshold ?? this.defaultThresholds.blockThreshold,
|
|
234
|
+
flagThreshold: overrides.flagThreshold ?? this.defaultThresholds.flagThreshold,
|
|
235
|
+
warnThreshold: overrides.warnThreshold ?? this.defaultThresholds.warnThreshold,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
//# sourceMappingURL=ClassifierOrchestrator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ClassifierOrchestrator.js","sourceRoot":"","sources":["../src/ClassifierOrchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AASH,OAAO,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,8EAA8E;AAC9E,2DAA2D;AAC3D,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,eAAe,GAAoC;IACvD,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;IAC1B,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC7B,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;IACzB,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;CAC3B,CAAC;AAEF,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,sBAAsB;IACjC,4EAA4E;IAC5E,gBAAgB;IAChB,4EAA4E;IAE5E,0EAA0E;IACzD,WAAW,CAAuB;IAEnD,0EAA0E;IACzD,iBAAiB,CAAuB;IAEzD;;;;OAIG;IACc,uBAAuB,CAAgD;IAExF,4EAA4E;IAC5E,cAAc;IACd,4EAA4E;IAE5E;;;;;;;;OAQG;IACH,YACE,WAAiC,EACjC,oBAA0C,kBAAkB,EAC5D,0BAAyE,EAAE;QAE3E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;IACzD,CAAC;IAED,4EAA4E;IAC5E,aAAa;IACb,4EAA4E;IAE5E;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,WAAW,CAAC,IAAY;QAC5B,gEAAgE;QAChE,8DAA8D;QAC9D,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,qEAAqE;QACrE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CACzD,CAAC;QAEF,gEAAgE;QAChE,MAAM,OAAO,GAAoC,EAAE,CAAC;QACpD,IAAI,WAAW,GAAG,eAAe,CAAC,KAAK,CAAC;QACxC,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAEvC,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;gBAChC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAExB,uDAAuD;gBACvD,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBAEzD,sDAAsD;gBACtD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBAEzD,gDAAgD;gBAChD,IAAI,eAAe,CAAC,MAAM,CAAC,GAAG,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,WAAW,GAAG,MAAM,CAAC;oBACrB,WAAW,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC9B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,OAAO,CAAC,IAAI,CACV,wCAAwC,UAAU,CAAC,EAAE,aAAa,OAAO,CAAC,MAAM,EAAE,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAElC,OAAO;YACL,OAAO;YACP,iBAAiB,EAAE,WAAW;YAC9B,WAAW;YACX,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,SAAS,CAAC;SAChD,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,CAAC,UAAU,CACtB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC/B,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;YACpB,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E;;;;;;;;;;OAUG;IACK,KAAK,CAAC,aAAa,CACzB,UAA8B,EAC9B,IAAY;QAEZ,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;QAExD,OAAO;YACL,GAAG,MAAM;YACT,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,SAAS;SACV,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,aAAa,CACnB,MAAqC,EACrC,UAAgC;QAEhC,6CAA6C;QAC7C,uEAAuE;QACvE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC;YACjD,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YAC3B,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;QAEtB,+DAA+D;QAC/D,IAAI,UAAU,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;YAC5C,OAAO,eAAe,CAAC,KAAK,CAAC;QAC/B,CAAC;QACD,IAAI,UAAU,IAAI,UAAU,CAAC,aAAa,EAAE,CAAC;YAC3C,OAAO,eAAe,CAAC,IAAI,CAAC;QAC9B,CAAC;QACD,IAAI,UAAU,IAAI,UAAU,CAAC,aAAa,EAAE,CAAC;YAC3C,OAAO,eAAe,CAAC,QAAQ,CAAC;QAClC,CAAC;QAED,OAAO,eAAe,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACK,iBAAiB,CAAC,YAAoB;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,uBAAuB,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,iBAAiB,CAAC;QAChC,CAAC;QAED,OAAO;YACL,cAAc,EAAE,SAAS,CAAC,cAAc,IAAI,IAAI,CAAC,iBAAiB,CAAC,cAAc;YACjF,aAAa,EAAE,SAAS,CAAC,aAAa,IAAI,IAAI,CAAC,iBAAiB,CAAC,aAAa;YAC9E,aAAa,EAAE,SAAS,CAAC,aAAa,IAAI,IAAI,CAAC,iBAAiB,CAAC,aAAa;SAC/E,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Interface contract for ML-backed content classifiers.
|
|
3
|
+
*
|
|
4
|
+
* An `IContentClassifier` represents a single model pipeline that accepts
|
|
5
|
+
* arbitrary text and returns a {@link ClassificationResult} containing the
|
|
6
|
+
* winning label and confidence scores for all candidate classes.
|
|
7
|
+
*
|
|
8
|
+
* Built-in implementations (toxicity, injection, jailbreak) each implement
|
|
9
|
+
* this interface. Third-party classifiers may be registered via the
|
|
10
|
+
* `customClassifiers` option of {@link MLClassifierPackOptions}.
|
|
11
|
+
*
|
|
12
|
+
* Lifecycle
|
|
13
|
+
* ---------
|
|
14
|
+
* 1. The pack initialises each classifier (model loading, warm-up).
|
|
15
|
+
* 2. The guardrail pipeline calls `classify()` for every text chunk.
|
|
16
|
+
* 3. On pack teardown, `dispose()` is called (if present) to release GPU/
|
|
17
|
+
* WASM memory.
|
|
18
|
+
*
|
|
19
|
+
* @module agentos/extensions/packs/ml-classifiers/IContentClassifier
|
|
20
|
+
*/
|
|
21
|
+
import type { ClassificationResult } from '@framers/agentos';
|
|
22
|
+
/**
|
|
23
|
+
* Contract for a single ML content classifier.
|
|
24
|
+
*
|
|
25
|
+
* Implementations back one model pipeline and expose a narrow classify/dispose
|
|
26
|
+
* API so the guardrail orchestrator can drive them uniformly regardless of the
|
|
27
|
+
* underlying runtime (Node.js ONNX, browser WASM, remote inference endpoint).
|
|
28
|
+
*
|
|
29
|
+
* @example Minimal custom classifier
|
|
30
|
+
* ```typescript
|
|
31
|
+
* class SarcasmClassifier implements IContentClassifier {
|
|
32
|
+
* readonly id = 'custom:sarcasm-detector';
|
|
33
|
+
* readonly displayName = 'Sarcasm Detector';
|
|
34
|
+
* readonly description = 'Detects sarcastic or ironic statements.';
|
|
35
|
+
* readonly modelId = 'my-org/sarcasm-bert';
|
|
36
|
+
* isLoaded = false;
|
|
37
|
+
*
|
|
38
|
+
* async classify(text: string): Promise<ClassificationResult> {
|
|
39
|
+
* // … run inference …
|
|
40
|
+
* return { bestClass: 'NOT_SARCASTIC', confidence: 0.8, allScores: [] };
|
|
41
|
+
* }
|
|
42
|
+
*
|
|
43
|
+
* async dispose(): Promise<void> {
|
|
44
|
+
* // Free resources.
|
|
45
|
+
* }
|
|
46
|
+
* }
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
export interface IContentClassifier {
|
|
50
|
+
/**
|
|
51
|
+
* Unique service identifier for this classifier.
|
|
52
|
+
*
|
|
53
|
+
* Must follow the `agentos:<domain>:<name>` convention so it can be
|
|
54
|
+
* registered with the AgentOS shared service registry.
|
|
55
|
+
*
|
|
56
|
+
* @example `'agentos:ml-classifiers:toxicity-pipeline'`
|
|
57
|
+
*/
|
|
58
|
+
readonly id: string;
|
|
59
|
+
/**
|
|
60
|
+
* Human-readable name displayed in logs and dashboards.
|
|
61
|
+
*
|
|
62
|
+
* @example `'Toxicity Pipeline'`
|
|
63
|
+
*/
|
|
64
|
+
readonly displayName: string;
|
|
65
|
+
/**
|
|
66
|
+
* Short prose description of what this classifier detects.
|
|
67
|
+
*
|
|
68
|
+
* @example `'Detects toxic, hateful, or abusive language in text.'`
|
|
69
|
+
*/
|
|
70
|
+
readonly description: string;
|
|
71
|
+
/**
|
|
72
|
+
* Identifier of the underlying model being used, typically a Hugging Face
|
|
73
|
+
* model ID or a local filesystem path.
|
|
74
|
+
*
|
|
75
|
+
* @example `'Xenova/toxic-bert'`
|
|
76
|
+
*/
|
|
77
|
+
readonly modelId: string;
|
|
78
|
+
/**
|
|
79
|
+
* Whether the model weights have been fully loaded into memory and the
|
|
80
|
+
* classifier is ready to accept `classify()` calls.
|
|
81
|
+
*
|
|
82
|
+
* The pack initialiser sets this to `true` after the warm-up inference
|
|
83
|
+
* succeeds. Callers can check this flag before calling `classify()` to
|
|
84
|
+
* avoid queueing calls during a slow model download.
|
|
85
|
+
*/
|
|
86
|
+
isLoaded: boolean;
|
|
87
|
+
/**
|
|
88
|
+
* Classify the provided text and return confidence scores for all candidate
|
|
89
|
+
* labels.
|
|
90
|
+
*
|
|
91
|
+
* The classifier is responsible for mapping raw model output to the
|
|
92
|
+
* {@link ClassificationResult} shape. It should NOT apply thresholds or
|
|
93
|
+
* guardrail actions — that is the responsibility of the pack orchestrator.
|
|
94
|
+
*
|
|
95
|
+
* @param text - The text to classify. May be a short chunk from a streaming
|
|
96
|
+
* response or a complete message. Must not be empty.
|
|
97
|
+
* @returns A promise that resolves with the classification result, including
|
|
98
|
+
* the winning label (`bestClass`), its `confidence`, and `allScores` for
|
|
99
|
+
* every label the model evaluated.
|
|
100
|
+
* @throws {Error} If the model is not loaded (`isLoaded === false`) or if
|
|
101
|
+
* inference fails for an unrecoverable reason.
|
|
102
|
+
*/
|
|
103
|
+
classify(text: string): Promise<ClassificationResult>;
|
|
104
|
+
/**
|
|
105
|
+
* Release all resources held by this classifier (model weights, WASM
|
|
106
|
+
* module, GPU buffers, worker threads, etc.).
|
|
107
|
+
*
|
|
108
|
+
* Called by the pack orchestrator during AgentOS shutdown or when the pack
|
|
109
|
+
* is unloaded. Implementations should be idempotent — calling `dispose()`
|
|
110
|
+
* multiple times must not throw.
|
|
111
|
+
*
|
|
112
|
+
* @optional Classifiers that hold no persistent resources may omit this
|
|
113
|
+
* method.
|
|
114
|
+
*/
|
|
115
|
+
dispose?(): Promise<void>;
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=IContentClassifier.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IContentClassifier.d.ts","sourceRoot":"","sources":["../src/IContentClassifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB;;;;;;;OAOG;IACH,QAAQ,EAAE,OAAO,CAAC;IAElB;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEtD;;;;;;;;;;OAUG;IACH,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Interface contract for ML-backed content classifiers.
|
|
3
|
+
*
|
|
4
|
+
* An `IContentClassifier` represents a single model pipeline that accepts
|
|
5
|
+
* arbitrary text and returns a {@link ClassificationResult} containing the
|
|
6
|
+
* winning label and confidence scores for all candidate classes.
|
|
7
|
+
*
|
|
8
|
+
* Built-in implementations (toxicity, injection, jailbreak) each implement
|
|
9
|
+
* this interface. Third-party classifiers may be registered via the
|
|
10
|
+
* `customClassifiers` option of {@link MLClassifierPackOptions}.
|
|
11
|
+
*
|
|
12
|
+
* Lifecycle
|
|
13
|
+
* ---------
|
|
14
|
+
* 1. The pack initialises each classifier (model loading, warm-up).
|
|
15
|
+
* 2. The guardrail pipeline calls `classify()` for every text chunk.
|
|
16
|
+
* 3. On pack teardown, `dispose()` is called (if present) to release GPU/
|
|
17
|
+
* WASM memory.
|
|
18
|
+
*
|
|
19
|
+
* @module agentos/extensions/packs/ml-classifiers/IContentClassifier
|
|
20
|
+
*/
|
|
21
|
+
export {};
|
|
22
|
+
//# sourceMappingURL=IContentClassifier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IContentClassifier.js","sourceRoot":"","sources":["../src/IContentClassifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG"}
|