@framers/agentos 0.1.54 → 0.1.56

package/dist/extensions/packs/pii-redaction/recognizers/NerModelRecognizer.js

@@ -0,0 +1,299 @@
+/**
+ * @file NerModelRecognizer.ts
+ * @description Tier 3 NER-model recogniser that uses a HuggingFace
+ * Transformers pipeline for high-accuracy named-entity recognition.
+ *
+ * This recogniser loads a pre-trained BERT-style NER model via the
+ * `@huggingface/transformers` library and maps BIO-tagged outputs
+ * (B-PER, I-PER, B-LOC, I-LOC, B-ORG, I-ORG, B-MISC, I-MISC) to the
+ * pipeline's {@link PiiEntityType} values.
+ *
+ * The model is lazily loaded through the {@link ISharedServiceRegistry} so
+ * that only one instance exists per agent, and it is shared across any
+ * extensions that need NER capabilities.
+ *
+ * @module pii-redaction/recognizers
+ */
+// ---------------------------------------------------------------------------
+// Service identity for the shared NER pipeline
+// ---------------------------------------------------------------------------
+/**
+ * Stable service ID for the HuggingFace NER pipeline stored in the shared
+ * service registry.
+ */
+const NER_PIPELINE_SERVICE_ID = 'agentos:nlp:ner-pipeline';
+// ---------------------------------------------------------------------------
+// BIO label → PiiEntityType mapping
+// ---------------------------------------------------------------------------
+/**
+ * Maps BERT NER BIO-tag prefixes (without the B-/I- prefix) to our
+ * canonical {@link PiiEntityType} values.
+ *
+ * | NER label | PiiEntityType   |
+ * |-----------|-----------------|
+ * | PER       | PERSON          |
+ * | LOC       | LOCATION        |
+ * | ORG       | ORGANIZATION    |
+ * | MISC      | UNKNOWN_PII     |
+ */
+const NER_LABEL_MAP = {
+    PER: 'PERSON',
+    LOC: 'LOCATION',
+    ORG: 'ORGANIZATION',
+    MISC: 'UNKNOWN_PII',
+};
+// ---------------------------------------------------------------------------
+// NerModelRecognizer
+// ---------------------------------------------------------------------------
+/**
+ * Tier 3 entity recogniser that runs a HuggingFace BERT NER model for
+ * high-accuracy named-entity recognition.
+ *
+ * ### How it works
+ * 1. On first `recognize()` call, the `@huggingface/transformers` library is
+ *    loaded and a `token-classification` pipeline is created via the shared
+ *    service registry.
+ * 2. The pipeline tokenises the input and runs it through the NER model,
+ *    returning BIO-tagged token predictions.
+ * 3. Contiguous BIO tokens are merged: a `B-PER` followed by `I-PER` tokens
+ *    becomes a single PERSON entity.  The final score is the average of the
+ *    constituent token scores.
+ * 4. Merged entities are mapped to {@link PiiEntity} objects.
+ *
+ * ### Graceful degradation
+ * If `@huggingface/transformers` is not installed or the model fails to load,
+ * the recogniser sets `unavailable = true` and returns empty arrays on all
+ * subsequent calls, ensuring the pipeline degrades without crashing.
+ *
+ * @example
+ * ```ts
+ * const registry = new SharedServiceRegistry();
+ * const recognizer = new NerModelRecognizer(registry);
+ * const entities = await recognizer.recognize('John Smith lives in London');
+ * // entities: [{ entityType: 'PERSON', text: 'John Smith', ... },
+ * //            { entityType: 'LOCATION', text: 'London', ... }]
+ * ```
+ */
+export class NerModelRecognizer {
+    /**
+     * Construct a new NerModelRecognizer.
+     *
+     * @param services - Shared service registry for lazy-loading the
+     *   HuggingFace NER pipeline.
+     */
+    constructor(services) {
+        /** @inheritdoc */
+        this.name = 'NerModelRecognizer';
+        /** @inheritdoc */
+        this.supportedEntities = [
+            'PERSON',
+            'LOCATION',
+            'ORGANIZATION',
+            'UNKNOWN_PII', // Mapped from MISC entities
+        ];
+        /**
+         * When `true`, the transformers library or model failed to load and all
+         * future calls will return empty arrays.
+         */
+        this.unavailable = false;
+        this.services = services;
+    }
+    /**
+     * Scan the input text for named entities using a BERT NER model.
+     *
+     * BIO-tagged tokens are merged into contiguous entity spans and mapped
+     * to {@link PiiEntity} objects.
+     *
+     * @param input   - Raw text to analyse.
+     * @param options - Optional filtering and context hints.
+     * @returns Array of detected {@link PiiEntity} objects.
+     */
+    async recognize(input, options) {
+        // If the model previously failed to load, bail out immediately.
+        if (this.unavailable)
+            return [];
+        // Determine which entity types the caller wants.
+        const wantedTypes = this.resolveWantedTypes(options?.entityTypes);
+        if (wantedTypes.size === 0)
+            return [];
+        // Lazily load the NER pipeline via the shared service registry.
+        let pipeline;
+        try {
+            pipeline = await this.services.getOrCreate(NER_PIPELINE_SERVICE_ID, async () => {
+                // Dynamic import so @huggingface/transformers is optional.
+                const transformers = await import('@huggingface/transformers');
+                // Create a token-classification pipeline with a small NER model.
+                const pipe = await transformers.pipeline('token-classification', 'Xenova/bert-base-NER');
+                return pipe;
+            });
+        }
+        catch {
+            // Transformers not installed or model download failed.
+            this.unavailable = true;
+            return [];
+        }
+        // Run the NER pipeline on the input text.
+        let tokens;
+        try {
+            tokens = await pipeline(input);
+        }
+        catch {
+            // Runtime inference error — degrade gracefully for this call.
+            return [];
+        }
+        // Merge BIO tokens into contiguous entity spans.
+        const merged = this.mergeBioTokens(tokens);
+        // Map merged spans to PiiEntity objects, filtering by wanted types.
+        return this.mapToEntities(merged, wantedTypes);
+    }
+    /** @inheritdoc */
+    async dispose() {
+        // The pipeline is owned by the shared service registry and will be
+        // cleaned up when the registry is released.
+    }
+    // -----------------------------------------------------------------------
+    // Private helpers
+    // -----------------------------------------------------------------------
+    /**
+     * Determines which of our supported entity types the caller wants.
+     *
+     * @param entityTypes - Optional entity-type filter from the caller.
+     * @returns Set of wanted types intersected with our supported types.
+     */
+    resolveWantedTypes(entityTypes) {
+        if (!entityTypes || entityTypes.length === 0) {
+            return new Set(this.supportedEntities);
+        }
+        const supported = new Set(this.supportedEntities);
+        return new Set(entityTypes.filter((t) => supported.has(t)));
+    }
+    /**
+     * Merges BIO-tagged tokens into contiguous entity spans.
+     *
+     * The BIO tagging scheme works as follows:
+     * - `B-XXX` — Beginning of a new entity of type XXX.
+     * - `I-XXX` — Inside/continuation of the current entity of type XXX.
+     * - `O`     — Outside any entity (ignored).
+     *
+     * A `B-PER` followed by one or more `I-PER` tokens produces a single
+     * merged span.  When a `B-XXX` appears while another entity is open,
+     * the previous entity is flushed and a new one begins.
+     *
+     * @param tokens - Raw BIO-tagged token array from the NER pipeline.
+     * @returns Array of merged entity spans with aggregated metadata.
+     */
+    mergeBioTokens(tokens) {
+        const result = [];
+        let current = null;
+        for (const token of tokens) {
+            const { tag, label } = this.parseBioLabel(token.entity);
+            // Skip 'O' (outside) tokens.
+            if (tag === 'O' || !label) {
+                // Flush any in-progress entity.
+                if (current) {
+                    result.push(current);
+                    current = null;
+                }
+                continue;
+            }
+            if (tag === 'B') {
+                // Beginning of a new entity — flush the previous one if any.
+                if (current)
+                    result.push(current);
+                current = {
+                    label,
+                    text: token.word,
+                    start: token.start,
+                    end: token.end,
+                    scores: [token.score],
+                };
+            }
+            else if (tag === 'I' && current && current.label === label) {
+                // Continuation of the current entity — extend the span.
+                current.text += token.word.startsWith('##')
+                    ? token.word.slice(2) // WordPiece sub-token: strip '##' prefix
+                    : ` ${token.word}`; // Regular token: add space separator
+                current.end = token.end;
+                current.scores.push(token.score);
+            }
+            else {
+                // I-tag without a matching B-tag or different label: treat as
+                // a new entity beginning (common with imperfect models).
+                if (current)
+                    result.push(current);
+                current = {
+                    label,
+                    text: token.word,
+                    start: token.start,
+                    end: token.end,
+                    scores: [token.score],
+                };
+            }
+        }
+        // Flush the last entity if still open.
+        if (current)
+            result.push(current);
+        return result;
+    }
+    /**
+     * Parses a BIO label string like `'B-PER'` or `'I-LOC'` into its
+     * tag component (`'B'`, `'I'`, `'O'`) and entity label (`'PER'`, `'LOC'`).
+     *
+     * @param bioLabel - The raw BIO label from the NER model.
+     * @returns Parsed tag and label.
+     */
+    parseBioLabel(bioLabel) {
+        if (bioLabel === 'O')
+            return { tag: 'O', label: null };
+        const dashIdx = bioLabel.indexOf('-');
+        if (dashIdx === -1) {
+            // Non-standard label without B-/I- prefix — treat as beginning.
+            return { tag: 'B', label: bioLabel };
+        }
+        return {
+            tag: bioLabel.slice(0, dashIdx),
+            label: bioLabel.slice(dashIdx + 1),
+        };
+    }
+    /**
+     * Maps merged entity spans to {@link PiiEntity} objects, filtering by
+     * the set of wanted entity types.
+     *
+     * The score for each entity is the arithmetic mean of its constituent
+     * token scores, reflecting the model's average confidence across the
+     * full span.
+     *
+     * @param merged     - Array of merged BIO entity spans.
+     * @param wantedTypes - Set of entity types the caller is interested in.
+     * @returns Filtered array of {@link PiiEntity} objects.
+     */
+    mapToEntities(merged, wantedTypes) {
+        const entities = [];
+        for (const span of merged) {
+            // Map the NER label to our PiiEntityType.
+            const entityType = NER_LABEL_MAP[span.label];
+            if (!entityType)
+                continue; // Unknown NER label — skip.
+            // Apply entity-type filter.
+            if (!wantedTypes.has(entityType))
+                continue;
+            // Compute average score across all constituent tokens.
+            const avgScore = span.scores.reduce((sum, s) => sum + s, 0) / span.scores.length;
+            entities.push({
+                entityType,
+                text: span.text,
+                start: span.start,
+                end: span.end,
+                score: avgScore,
+                source: 'ner-model',
+                metadata: {
+                    nerLabel: span.label,
+                    nerModel: 'Xenova/bert-base-NER',
+                    tokenCount: span.scores.length,
+                },
+            });
+        }
+        return entities;
+    }
+}
+//# sourceMappingURL=NerModelRecognizer.js.map

package/dist/extensions/packs/pii-redaction/recognizers/NerModelRecognizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NerModelRecognizer.js","sourceRoot":"","sources":["../../../../../src/extensions/packs/pii-redaction/recognizers/NerModelRecognizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,uBAAuB,GAAG,0BAA0B,CAAC;AAE3D,8EAA8E;AAC9E,oCAAoC;AACpC,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,aAAa,GAAkC;IACnD,GAAG,EAAE,QAAQ;IACb,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,cAAc;IACnB,IAAI,EAAE,aAAa;CACpB,CAAC;AA6BF,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,kBAAkB;IAwB7B;;;;;OAKG;IACH,YAAY,QAAgC;QA7B5C,kBAAkB;QACF,SAAI,GAAG,oBAAoB,CAAC;QAE5C,kBAAkB;QACF,sBAAiB,GAAoB;YACnD,QAAQ;YACR,UAAU;YACV,cAAc;YACd,aAAa,EAAE,4BAA4B;SAC5C,CAAC;QAEF;;;WAGG;QACK,gBAAW,GAAG,KAAK,CAAC;QAe1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,OAA0B;QAC9D,gEAAgE;QAChE,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAEhC,iDAAiD;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAClE,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,gEAAgE;QAChE,IAAI,QAAuB,CAAC;QAC5B,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CACxC,uBAAuB,EACvB,KAAK,IAAI,EAAE;gBACT,2DAA2D;gBAC3D,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;gBAC/D,iEAAiE;gBACjE,MAAM,IAAI,GAAG,MAAO,YAElB,CAAC,QAAQ,CACT,sBAAsB,EACtB,sBAAsB,CACvB,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC,CACF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;YACvD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,0CAA0C;QAC1C,IAAI,MAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,iDAAiD;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAE3C,oEAAoE;QACpE,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACjD,CAAC;IAED,kBAAkB;IACX,KAAK,CAAC,OAAO;QAClB,mEAAmE;QACnE,4CAA4C;IAC9C,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E;;;;;OAKG;IACK,kBAAkB,CAAC,WAA6B;QACtD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAClD,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,cAAc,CAAC,MAAkB;QACvC,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,IAAI,OAAO,GAAwB,IAAI,CAAC;QAExC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAExD,6BAA6B;YAC7B,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC1B,gCAAgC;gBAChC,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACrB,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;gBAChB,6DAA6D;gBAC7D,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,OAAO,GAAG;oBACR,KAAK;oBACL,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;iBACtB,CAAC;YACJ,CAAC;iBAAM,IAAI,GAAG,KAAK,GAAG,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;gBAC7D,wDAAwD;gBACxD,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;oBACzC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,yCAAyC;oBAChE,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAI,qCAAqC;gBAC9D,OAAO,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;gBACxB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,8DAA8D;gBAC9D,yDAAyD;gBACzD,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClC,OAAO,GAAG;oBACR,KAAK;oBACL,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;iBACtB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CAAC,QAAgB;QACpC,IAAI,QAAQ,KAAK,GAAG;YAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAEvD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;YACnB,gEAAgE;YAChE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QACvC,CAAC;QAED,OAAO;YACL,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;YAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;SACnC,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;OAWG;IACK,aAAa,CACnB,MAAsB,EACtB,WAA+B;QAE/B,MAAM,QAAQ,GAAgB,EAAE,CAAC;QAEjC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,0CAA0C;YAC1C,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC,UAAU;gBAAE,SAAS,CAAC,4BAA4B;YAEvD,4BAA4B;YAC5B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,SAAS;YAE3C,uDAAuD;YACvD,MAAM,QAAQ,GACZ,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YAElE,QAAQ,CAAC,IAAI,CAAC;gBACZ,UAAU;gBACV,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,QAAQ;gBACf,MAAM,EAAE,WAAW;gBACnB,QAAQ,EAAE;oBACR,QAAQ,EAAE,IAAI,CAAC,KAAK;oBACpB,QAAQ,EAAE,sBAAsB;oBAChC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;iBAC/B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.d.ts

@@ -0,0 +1,102 @@
+/**
+ * @file NlpPrefilterRecognizer.ts
+ * @description Tier 2 NLP-based pre-filter recogniser that uses the
+ * `compromise` library for lightweight named-entity extraction.
+ *
+ * This recogniser is designed as a low-confidence pre-filter: it catches
+ * person names, places, and organisations that regex patterns typically miss,
+ * then flags them at a low score (0.3–0.6) so that higher tiers (NER model,
+ * LLM judge) can confirm or discard them.
+ *
+ * The `compromise` library is loaded lazily via the {@link ISharedServiceRegistry}
+ * so that the heavyweight module is only initialised once and shared across
+ * any extensions that need it.
+ *
+ * @module pii-redaction/recognizers
+ */
+import type { PiiEntity, PiiEntityType } from '../types';
+import type { IEntityRecognizer, RecognizeOptions } from './IEntityRecognizer';
+import type { ISharedServiceRegistry } from '../../../ISharedServiceRegistry';
+/**
+ * Tier 2 NLP pre-filter recogniser that uses the `compromise` library to
+ * detect person names, places, and organisations.
+ *
+ * ### Design rationale
+ * Compromise is a rule-based NLP library (~200 KB) that is much lighter than
+ * a full transformer model.  It provides reasonable recall for English-language
+ * named entities at the cost of lower precision.  This recogniser intentionally
+ * assigns low confidence scores (0.3–0.6) so that its results serve as
+ * *candidates* for higher-tier confirmation rather than final detections.
+ *
+ * ### Graceful degradation
+ * If the compromise module fails to load (e.g. not installed, wrong platform),
+ * the recogniser sets an internal `unavailable` flag and returns empty results
+ * on every subsequent call without throwing.  This ensures the pipeline
+ * continues to function with just the regex and/or LLM tiers.
+ *
+ * ### Shared service pattern
+ * The compromise module is loaded via the {@link ISharedServiceRegistry} so
+ * that multiple extensions in the same agent can share a single instance.
+ *
+ * @example
+ * ```ts
+ * const registry = new SharedServiceRegistry();
+ * const recognizer = new NlpPrefilterRecognizer(registry);
+ * const entities = await recognizer.recognize('John Smith lives in London');
+ * // entities might include PERSON "John Smith" and LOCATION "London"
+ * ```
+ */
+export declare class NlpPrefilterRecognizer implements IEntityRecognizer {
+    /** @inheritdoc */
+    readonly name = "NlpPrefilterRecognizer";
+    /** @inheritdoc */
+    readonly supportedEntities: PiiEntityType[];
+    /**
+     * When `true`, the compromise module failed to load and all future
+     * `recognize()` calls will return empty arrays silently.
+     */
+    private unavailable;
+    /**
+     * Reference to the shared service registry used for lazy-loading the
+     * compromise NLP module.
+     */
+    private readonly services;
+    /**
+     * Construct a new NlpPrefilterRecognizer.
+     *
+     * @param services - Shared service registry for lazy-loading compromise.
+     */
+    constructor(services: ISharedServiceRegistry);
+    /**
+     * Scan the input text for person names, places, and organisations using
+     * the compromise NLP library.
+     *
+     * Results are returned with low confidence scores (0.3–0.6) to indicate
+     * they are pre-filter candidates requiring higher-tier confirmation.
+     *
+     * @param input   - Raw text to analyse.
+     * @param options - Optional filtering and context hints.
+     * @returns Array of low-confidence {@link PiiEntity} candidates.
+     */
+    recognize(input: string, options?: RecognizeOptions): Promise<PiiEntity[]>;
+    /** @inheritdoc */
+    dispose(): Promise<void>;
+    /**
+     * Determines which of our supported entity types the caller wants.
+     *
+     * @param entityTypes - Optional entity-type filter from the caller.
+     * @returns Set of wanted types intersected with our supported types.
+     */
+    private resolveWantedTypes;
+    /**
+     * Computes a contiguous [start, end) span from a compromise offset result.
+     *
+     * Compromise returns individual term offsets; this method merges them into
+     * a single span covering the full match text.
+     *
+     * @param match - A compromise offset result with term-level positions.
+     * @returns Object with `start` and `end` offsets, or `null` if no terms.
+     */
+    private computeSpan;
+}
+//# sourceMappingURL=NlpPrefilterRecognizer.d.ts.map

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"NlpPrefilterRecognizer.d.ts","sourceRoot":"","sources":["../../../../../src/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzD,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAwF9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,sBAAuB,YAAW,iBAAiB;IAC9D,kBAAkB;IAClB,SAAgB,IAAI,4BAA4B;IAEhD,kBAAkB;IAClB,SAAgB,iBAAiB,EAAE,aAAa,EAAE,CAIhD;IAEF;;;OAGG;IACH,OAAO,CAAC,WAAW,CAAS;IAE5B;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;IAElD;;;;OAIG;gBACS,QAAQ,EAAE,sBAAsB;IAI5C;;;;;;;;;;OAUG;IACU,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IA4FvF,kBAAkB;IACL,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IASrC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAQ1B;;;;;;;;OAQG;IACH,OAAO,CAAC,WAAW;CAepB"}

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.js

@@ -0,0 +1,228 @@
+/**
+ * @file NlpPrefilterRecognizer.ts
+ * @description Tier 2 NLP-based pre-filter recogniser that uses the
+ * `compromise` library for lightweight named-entity extraction.
+ *
+ * This recogniser is designed as a low-confidence pre-filter: it catches
+ * person names, places, and organisations that regex patterns typically miss,
+ * then flags them at a low score (0.3–0.6) so that higher tiers (NER model,
+ * LLM judge) can confirm or discard them.
+ *
+ * The `compromise` library is loaded lazily via the {@link ISharedServiceRegistry}
+ * so that the heavyweight module is only initialised once and shared across
+ * any extensions that need it.
+ *
+ * @module pii-redaction/recognizers
+ */
+// ---------------------------------------------------------------------------
+// Service identity constant for the shared compromise instance
+// ---------------------------------------------------------------------------
+/**
+ * Stable service ID used to store the compromise module in the shared
+ * service registry.  Other extensions that need compromise can reuse the
+ * same cached instance.
+ */
+const COMPROMISE_SERVICE_ID = 'agentos:nlp:compromise';
+// ---------------------------------------------------------------------------
+// Score constants
+// ---------------------------------------------------------------------------
+/**
+ * Score assigned to person name detections.  Higher than places/orgs because
+ * compromise's person-name heuristics tend to be slightly more reliable.
+ */
+const PERSON_SCORE = 0.55;
+/** Score assigned to location/place detections. */
+const LOCATION_SCORE = 0.45;
+/** Score assigned to organisation detections. */
+const ORG_SCORE = 0.4;
+// ---------------------------------------------------------------------------
+// NlpPrefilterRecognizer
+// ---------------------------------------------------------------------------
+/**
+ * Tier 2 NLP pre-filter recogniser that uses the `compromise` library to
+ * detect person names, places, and organisations.
+ *
+ * ### Design rationale
+ * Compromise is a rule-based NLP library (~200 KB) that is much lighter than
+ * a full transformer model.  It provides reasonable recall for English-language
+ * named entities at the cost of lower precision.  This recogniser intentionally
+ * assigns low confidence scores (0.3–0.6) so that its results serve as
+ * *candidates* for higher-tier confirmation rather than final detections.
+ *
+ * ### Graceful degradation
+ * If the compromise module fails to load (e.g. not installed, wrong platform),
+ * the recogniser sets an internal `unavailable` flag and returns empty results
+ * on every subsequent call without throwing.  This ensures the pipeline
+ * continues to function with just the regex and/or LLM tiers.
+ *
+ * ### Shared service pattern
+ * The compromise module is loaded via the {@link ISharedServiceRegistry} so
+ * that multiple extensions in the same agent can share a single instance.
+ *
+ * @example
+ * ```ts
+ * const registry = new SharedServiceRegistry();
+ * const recognizer = new NlpPrefilterRecognizer(registry);
+ * const entities = await recognizer.recognize('John Smith lives in London');
+ * // entities might include PERSON "John Smith" and LOCATION "London"
+ * ```
+ */
+export class NlpPrefilterRecognizer {
+    /**
+     * Construct a new NlpPrefilterRecognizer.
+     *
+     * @param services - Shared service registry for lazy-loading compromise.
+     */
+    constructor(services) {
+        /** @inheritdoc */
+        this.name = 'NlpPrefilterRecognizer';
+        /** @inheritdoc */
+        this.supportedEntities = [
+            'PERSON',
+            'LOCATION',
+            'ORGANIZATION',
+        ];
+        /**
+         * When `true`, the compromise module failed to load and all future
+         * `recognize()` calls will return empty arrays silently.
+         */
+        this.unavailable = false;
+        this.services = services;
+    }
+    /**
+     * Scan the input text for person names, places, and organisations using
+     * the compromise NLP library.
+     *
+     * Results are returned with low confidence scores (0.3–0.6) to indicate
+     * they are pre-filter candidates requiring higher-tier confirmation.
+     *
+     * @param input   - Raw text to analyse.
+     * @param options - Optional filtering and context hints.
+     * @returns Array of low-confidence {@link PiiEntity} candidates.
+     */
+    async recognize(input, options) {
+        // If compromise failed to load previously, bail out silently.
+        if (this.unavailable)
+            return [];
+        // Determine which entity types the caller wants from us.
+        const wantedTypes = this.resolveWantedTypes(options?.entityTypes);
+        if (wantedTypes.size === 0)
+            return [];
+        // Lazily load compromise via the shared service registry.
+        let nlp;
+        try {
+            nlp = await this.services.getOrCreate(COMPROMISE_SERVICE_ID, async () => {
+                // Dynamic import so compromise is optional at the module level.
+                const mod = await import('compromise');
+                // compromise's default export may be the function directly or
+                // wrapped in { default: ... } depending on the bundler.
+                return mod.default ?? mod;
+            });
+        }
+        catch {
+            // compromise is not installed or failed to load — degrade gracefully.
+            this.unavailable = true;
+            return [];
+        }
+        // Parse the input text with compromise.
+        const doc = nlp(input);
+        const entities = [];
+        // Extract people (PERSON entities).
+        if (wantedTypes.has('PERSON')) {
+            const people = doc.people().out('offset');
+            for (const match of people) {
+                const span = this.computeSpan(match);
+                if (span) {
+                    entities.push({
+                        entityType: 'PERSON',
+                        text: match.text,
+                        start: span.start,
+                        end: span.end,
+                        score: PERSON_SCORE,
+                        source: 'nlp-prefilter',
+                        metadata: { recognizer: 'compromise', method: 'people' },
+                    });
+                }
+            }
+        }
+        // Extract places (LOCATION entities).
+        if (wantedTypes.has('LOCATION')) {
+            const places = doc.places().out('offset');
+            for (const match of places) {
+                const span = this.computeSpan(match);
+                if (span) {
+                    entities.push({
+                        entityType: 'LOCATION',
+                        text: match.text,
+                        start: span.start,
+                        end: span.end,
+                        score: LOCATION_SCORE,
+                        source: 'nlp-prefilter',
+                        metadata: { recognizer: 'compromise', method: 'places' },
+                    });
+                }
+            }
+        }
+        // Extract organisations (ORGANIZATION entities).
+        if (wantedTypes.has('ORGANIZATION')) {
+            const orgs = doc.organizations().out('offset');
+            for (const match of orgs) {
+                const span = this.computeSpan(match);
+                if (span) {
+                    entities.push({
+                        entityType: 'ORGANIZATION',
+                        text: match.text,
+                        start: span.start,
+                        end: span.end,
+                        score: ORG_SCORE,
+                        source: 'nlp-prefilter',
+                        metadata: { recognizer: 'compromise', method: 'organizations' },
+                    });
+                }
+            }
+        }
+        return entities;
+    }
+    /** @inheritdoc */
+    async dispose() {
+        // The compromise instance is owned by the shared service registry and
+        // will be cleaned up when the registry is released.  Nothing to do here.
+    }
+    // -----------------------------------------------------------------------
+    // Private helpers
+    // -----------------------------------------------------------------------
+    /**
+     * Determines which of our supported entity types the caller wants.
+     *
+     * @param entityTypes - Optional entity-type filter from the caller.
+     * @returns Set of wanted types intersected with our supported types.
+     */
+    resolveWantedTypes(entityTypes) {
+        if (!entityTypes || entityTypes.length === 0) {
+            return new Set(this.supportedEntities);
+        }
+        const supported = new Set(this.supportedEntities);
+        return new Set(entityTypes.filter((t) => supported.has(t)));
+    }
+    /**
+     * Computes a contiguous [start, end) span from a compromise offset result.
+     *
+     * Compromise returns individual term offsets; this method merges them into
+     * a single span covering the full match text.
+     *
+     * @param match - A compromise offset result with term-level positions.
+     * @returns Object with `start` and `end` offsets, or `null` if no terms.
+     */
+    computeSpan(match) {
+        if (!match.terms || match.terms.length === 0)
+            return null;
+        // Start is the offset of the first term.
+        const firstTerm = match.terms[0];
+        const start = firstTerm.offset.start;
+        // End is the offset + length of the last term.
+        const lastTerm = match.terms[match.terms.length - 1];
+        const end = lastTerm.offset.start + lastTerm.offset.length;
+        return { start, end };
+    }
+}
+//# sourceMappingURL=NlpPrefilterRecognizer.js.map

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NlpPrefilterRecognizer.js","sourceRoot":"","sources":["../../../../../src/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,8EAA8E;AAC9E,+DAA+D;AAC/D,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,wBAAwB,CAAC;AAyDvD,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,mDAAmD;AACnD,MAAM,cAAc,GAAG,IAAI,CAAC;AAE5B,iDAAiD;AACjD,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,sBAAsB;IAuBjC;;;;OAIG;IACH,YAAY,QAAgC;QA3B5C,kBAAkB;QACF,SAAI,GAAG,wBAAwB,CAAC;QAEhD,kBAAkB;QACF,sBAAiB,GAAoB;YACnD,QAAQ;YACR,UAAU;YACV,cAAc;SACf,CAAC;QAEF;;;WAGG;QACK,gBAAW,GAAG,KAAK,CAAC;QAc1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;;;;;;;;OAUG;IACI,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,OAA0B;QAC9D,8DAA8D;QAC9D,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAEhC,yDAAyD;QACzD,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAClE,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,0DAA0D;QAC1D,IAAI,GAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CACnC,qBAAqB,EACrB,KAAK,IAAI,EAAE;gBACT,gEAAgE;gBAChE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;gBACvC,8DAA8D;gBAC9D,wDAAwD;gBACxD,OAAQ,GAA+B,CAAC,OAA2B,IAAI,GAAkC,CAAC;YAC5G,CAAC,CACF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,sEAAsE;YACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,wCAAwC;QACxC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;QAEvB,MAAM,QAAQ,GAAgB,EAAE,CAAC;QAEjC,oCAAoC;QACpC,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,UAAU,EAAE,QAAQ;wBACpB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,KAAK,EAAE,YAAY;wBACnB,MAAM,EAAE,eAAe;wBACvB,QAAQ,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE;qBACzD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,UAAU,EAAE,UAAU;wBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,KAAK,EAAE,cAAc;wBACrB,MAAM,EAAE,eAAe;wBACvB,QAAQ,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE;qBACzD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,iDAAiD;QACjD,IAAI,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/C,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,UAAU,EAAE,cAAc;wBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,KAAK,EAAE,SAAS;wBAChB,MAAM,EAAE,eAAe;wBACvB,QAAQ,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,eAAe,EAAE;qBAChE,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,kBAAkB;IACX,KAAK,CAAC,OAAO;QAClB,sEAAsE;QACtE,yEAAyE;IAC3E,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E;;;;;OAKG;IACK,kBAAkB,CAAC,WAA6B;QACtD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAClD,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;;;;OAQG;IACK,WAAW,CACjB,KAA6B;QAE7B,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAE1D,yCAAyC;QACzC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;QAErC,+CAA+C;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;QAE3D,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACxB,CAAC;CACF"}