@framers/agentos 0.1.56 → 0.1.58

package/dist/extensions/packs/pii-redaction/recognizers/NerModelRecognizer.js

@@ -1,299 +0,0 @@
-/**
- * @file NerModelRecognizer.ts
- * @description Tier 3 NER-model recogniser that uses a HuggingFace
- * Transformers pipeline for high-accuracy named-entity recognition.
- *
- * This recogniser loads a pre-trained BERT-style NER model via the
- * `@huggingface/transformers` library and maps BIO-tagged outputs
- * (B-PER, I-PER, B-LOC, I-LOC, B-ORG, I-ORG, B-MISC, I-MISC) to the
- * pipeline's {@link PiiEntityType} values.
- *
- * The model is lazily loaded through the {@link ISharedServiceRegistry} so
- * that only one instance exists per agent, and it is shared across any
- * extensions that need NER capabilities.
- *
- * @module pii-redaction/recognizers
- */
-// ---------------------------------------------------------------------------
-// Service identity for the shared NER pipeline
-// ---------------------------------------------------------------------------
-/**
- * Stable service ID for the HuggingFace NER pipeline stored in the shared
- * service registry.
- */
-const NER_PIPELINE_SERVICE_ID = 'agentos:nlp:ner-pipeline';
-// ---------------------------------------------------------------------------
-// BIO label → PiiEntityType mapping
-// ---------------------------------------------------------------------------
-/**
- * Maps BERT NER BIO-tag prefixes (without the B-/I- prefix) to our
- * canonical {@link PiiEntityType} values.
- *
- * | NER label | PiiEntityType   |
- * |-----------|-----------------|
- * | PER       | PERSON          |
- * | LOC       | LOCATION        |
- * | ORG       | ORGANIZATION    |
- * | MISC      | UNKNOWN_PII     |
- */
-const NER_LABEL_MAP = {
-    PER: 'PERSON',
-    LOC: 'LOCATION',
-    ORG: 'ORGANIZATION',
-    MISC: 'UNKNOWN_PII',
-};
-// ---------------------------------------------------------------------------
-// NerModelRecognizer
-// ---------------------------------------------------------------------------
-/**
- * Tier 3 entity recogniser that runs a HuggingFace BERT NER model for
- * high-accuracy named-entity recognition.
- *
- * ### How it works
- * 1. On first `recognize()` call, the `@huggingface/transformers` library is
- *    loaded and a `token-classification` pipeline is created via the shared
- *    service registry.
- * 2. The pipeline tokenises the input and runs it through the NER model,
- *    returning BIO-tagged token predictions.
- * 3. Contiguous BIO tokens are merged: a `B-PER` followed by `I-PER` tokens
- *    becomes a single PERSON entity.  The final score is the average of the
- *    constituent token scores.
- * 4. Merged entities are mapped to {@link PiiEntity} objects.
- *
- * ### Graceful degradation
- * If `@huggingface/transformers` is not installed or the model fails to load,
- * the recogniser sets `unavailable = true` and returns empty arrays on all
- * subsequent calls, ensuring the pipeline degrades without crashing.
- *
- * @example
- * ```ts
- * const registry = new SharedServiceRegistry();
- * const recognizer = new NerModelRecognizer(registry);
- * const entities = await recognizer.recognize('John Smith lives in London');
- * // entities: [{ entityType: 'PERSON', text: 'John Smith', ... },
- * //            { entityType: 'LOCATION', text: 'London', ... }]
- * ```
- */
-export class NerModelRecognizer {
-    /**
-     * Construct a new NerModelRecognizer.
-     *
-     * @param services - Shared service registry for lazy-loading the
-     *   HuggingFace NER pipeline.
-     */
-    constructor(services) {
-        /** @inheritdoc */
-        this.name = 'NerModelRecognizer';
-        /** @inheritdoc */
-        this.supportedEntities = [
-            'PERSON',
-            'LOCATION',
-            'ORGANIZATION',
-            'UNKNOWN_PII', // Mapped from MISC entities
-        ];
-        /**
-         * When `true`, the transformers library or model failed to load and all
-         * future calls will return empty arrays.
-         */
-        this.unavailable = false;
-        this.services = services;
-    }
-    /**
-     * Scan the input text for named entities using a BERT NER model.
-     *
-     * BIO-tagged tokens are merged into contiguous entity spans and mapped
-     * to {@link PiiEntity} objects.
-     *
-     * @param input   - Raw text to analyse.
-     * @param options - Optional filtering and context hints.
-     * @returns Array of detected {@link PiiEntity} objects.
-     */
-    async recognize(input, options) {
-        // If the model previously failed to load, bail out immediately.
-        if (this.unavailable)
-            return [];
-        // Determine which entity types the caller wants.
-        const wantedTypes = this.resolveWantedTypes(options?.entityTypes);
-        if (wantedTypes.size === 0)
-            return [];
-        // Lazily load the NER pipeline via the shared service registry.
-        let pipeline;
-        try {
-            pipeline = await this.services.getOrCreate(NER_PIPELINE_SERVICE_ID, async () => {
-                // Dynamic import so @huggingface/transformers is optional.
-                const transformers = await import('@huggingface/transformers');
-                // Create a token-classification pipeline with a small NER model.
-                const pipe = await transformers.pipeline('token-classification', 'Xenova/bert-base-NER');
-                return pipe;
-            });
-        }
-        catch {
-            // Transformers not installed or model download failed.
-            this.unavailable = true;
-            return [];
-        }
-        // Run the NER pipeline on the input text.
-        let tokens;
-        try {
-            tokens = await pipeline(input);
-        }
-        catch {
-            // Runtime inference error — degrade gracefully for this call.
-            return [];
-        }
-        // Merge BIO tokens into contiguous entity spans.
-        const merged = this.mergeBioTokens(tokens);
-        // Map merged spans to PiiEntity objects, filtering by wanted types.
-        return this.mapToEntities(merged, wantedTypes);
-    }
-    /** @inheritdoc */
-    async dispose() {
-        // The pipeline is owned by the shared service registry and will be
-        // cleaned up when the registry is released.
-    }
-    // -----------------------------------------------------------------------
-    // Private helpers
-    // -----------------------------------------------------------------------
-    /**
-     * Determines which of our supported entity types the caller wants.
-     *
-     * @param entityTypes - Optional entity-type filter from the caller.
-     * @returns Set of wanted types intersected with our supported types.
-     */
-    resolveWantedTypes(entityTypes) {
-        if (!entityTypes || entityTypes.length === 0) {
-            return new Set(this.supportedEntities);
-        }
-        const supported = new Set(this.supportedEntities);
-        return new Set(entityTypes.filter((t) => supported.has(t)));
-    }
-    /**
-     * Merges BIO-tagged tokens into contiguous entity spans.
-     *
-     * The BIO tagging scheme works as follows:
-     * - `B-XXX` — Beginning of a new entity of type XXX.
-     * - `I-XXX` — Inside/continuation of the current entity of type XXX.
-     * - `O`     — Outside any entity (ignored).
-     *
-     * A `B-PER` followed by one or more `I-PER` tokens produces a single
-     * merged span.  When a `B-XXX` appears while another entity is open,
-     * the previous entity is flushed and a new one begins.
-     *
-     * @param tokens - Raw BIO-tagged token array from the NER pipeline.
-     * @returns Array of merged entity spans with aggregated metadata.
-     */
-    mergeBioTokens(tokens) {
-        const result = [];
-        let current = null;
-        for (const token of tokens) {
-            const { tag, label } = this.parseBioLabel(token.entity);
-            // Skip 'O' (outside) tokens.
-            if (tag === 'O' || !label) {
-                // Flush any in-progress entity.
-                if (current) {
-                    result.push(current);
-                    current = null;
-                }
-                continue;
-            }
-            if (tag === 'B') {
-                // Beginning of a new entity — flush the previous one if any.
-                if (current)
-                    result.push(current);
-                current = {
-                    label,
-                    text: token.word,
-                    start: token.start,
-                    end: token.end,
-                    scores: [token.score],
-                };
-            }
-            else if (tag === 'I' && current && current.label === label) {
-                // Continuation of the current entity — extend the span.
-                current.text += token.word.startsWith('##')
-                    ? token.word.slice(2) // WordPiece sub-token: strip '##' prefix
-                    : ` ${token.word}`; // Regular token: add space separator
-                current.end = token.end;
-                current.scores.push(token.score);
-            }
-            else {
-                // I-tag without a matching B-tag or different label: treat as
-                // a new entity beginning (common with imperfect models).
-                if (current)
-                    result.push(current);
-                current = {
-                    label,
-                    text: token.word,
-                    start: token.start,
-                    end: token.end,
-                    scores: [token.score],
-                };
-            }
-        }
-        // Flush the last entity if still open.
-        if (current)
-            result.push(current);
-        return result;
-    }
-    /**
-     * Parses a BIO label string like `'B-PER'` or `'I-LOC'` into its
-     * tag component (`'B'`, `'I'`, `'O'`) and entity label (`'PER'`, `'LOC'`).
-     *
-     * @param bioLabel - The raw BIO label from the NER model.
-     * @returns Parsed tag and label.
-     */
-    parseBioLabel(bioLabel) {
-        if (bioLabel === 'O')
-            return { tag: 'O', label: null };
-        const dashIdx = bioLabel.indexOf('-');
-        if (dashIdx === -1) {
-            // Non-standard label without B-/I- prefix — treat as beginning.
-            return { tag: 'B', label: bioLabel };
-        }
-        return {
-            tag: bioLabel.slice(0, dashIdx),
-            label: bioLabel.slice(dashIdx + 1),
-        };
-    }
-    /**
-     * Maps merged entity spans to {@link PiiEntity} objects, filtering by
-     * the set of wanted entity types.
-     *
-     * The score for each entity is the arithmetic mean of its constituent
-     * token scores, reflecting the model's average confidence across the
-     * full span.
-     *
-     * @param merged     - Array of merged BIO entity spans.
-     * @param wantedTypes - Set of entity types the caller is interested in.
-     * @returns Filtered array of {@link PiiEntity} objects.
-     */
-    mapToEntities(merged, wantedTypes) {
-        const entities = [];
-        for (const span of merged) {
-            // Map the NER label to our PiiEntityType.
-            const entityType = NER_LABEL_MAP[span.label];
-            if (!entityType)
-                continue; // Unknown NER label — skip.
-            // Apply entity-type filter.
-            if (!wantedTypes.has(entityType))
-                continue;
-            // Compute average score across all constituent tokens.
-            const avgScore = span.scores.reduce((sum, s) => sum + s, 0) / span.scores.length;
-            entities.push({
-                entityType,
-                text: span.text,
-                start: span.start,
-                end: span.end,
-                score: avgScore,
-                source: 'ner-model',
-                metadata: {
-                    nerLabel: span.label,
-                    nerModel: 'Xenova/bert-base-NER',
-                    tokenCount: span.scores.length,
-                },
-            });
-        }
-        return entities;
-    }
-}
-//# sourceMappingURL=NerModelRecognizer.js.map

package/dist/extensions/packs/pii-redaction/recognizers/NerModelRecognizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"NerModelRecognizer.js","sourceRoot":"","sources":["../../../../../src/extensions/packs/pii-redaction/recognizers/NerModelRecognizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,uBAAuB,GAAG,0BAA0B,CAAC;AAE3D,8EAA8E;AAC9E,oCAAoC;AACpC,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,aAAa,GAAkC;IACnD,GAAG,EAAE,QAAQ;IACb,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,cAAc;IACnB,IAAI,EAAE,aAAa;CACpB,CAAC;AA6BF,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,kBAAkB;IAwB7B;;;;;OAKG;IACH,YAAY,QAAgC;QA7B5C,kBAAkB;QACF,SAAI,GAAG,oBAAoB,CAAC;QAE5C,kBAAkB;QACF,sBAAiB,GAAoB;YACnD,QAAQ;YACR,UAAU;YACV,cAAc;YACd,aAAa,EAAE,4BAA4B;SAC5C,CAAC;QAEF;;;WAGG;QACK,gBAAW,GAAG,KAAK,CAAC;QAe1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,OAA0B;QAC9D,gEAAgE;QAChE,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAEhC,iDAAiD;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAClE,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,gEAAgE;QAChE,IAAI,QAAuB,CAAC;QAC5B,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CACxC,uBAAuB,EACvB,KAAK,IAAI,EAAE;gBACT,2DAA2D;gBAC3D,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;gBAC/D,iEAAiE;gBACjE,MAAM,IAAI,GAAG,MAAO,YAElB,CAAC,QAAQ,CACT,sBAAsB,EACtB,sBAAsB,CACvB,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC,CACF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;YACvD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,0CAA0C;QAC1C,IAAI,MAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,iDAAiD;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAE3C,oEAAoE;QACpE,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACjD,CAAC;IAED,kBAAkB;IACX,KAAK,CAAC,OAAO;QAClB,mEAAmE;QACnE,4CAA4C;IAC9C,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E;;;;;OAKG;IACK,kBAAkB,CAAC,WAA6B;QACtD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAClD,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,cAAc,CAAC,MAAkB;QACvC,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,IAAI,OAAO,GAAwB,IAAI,CAAC;QAExC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAExD,6BAA6B;YAC7B,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC1B,gCAAgC;gBAChC,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACrB,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;gBAChB,6DAA6D;gBAC7D,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,OAAO,GAAG;oBACR,KAAK;oBACL,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;iBACtB,CAAC;YACJ,CAAC;iBAAM,IAAI,GAAG,KAAK,GAAG,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;gBAC7D,wDAAwD;gBACxD,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;oBACzC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,yCAAyC;oBAChE,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAI,qCAAqC;gBAC9D,OAAO,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;gBACxB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,8DAA8D;gBAC9D,yDAAyD;gBACzD,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClC,OAAO,GAAG;oBACR,KAAK;oBACL,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;iBACtB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CAAC,QAAgB;QACpC,IAAI,QAAQ,KAAK,GAAG;YAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAEvD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;YACnB,gEAAgE;YAChE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QACvC,CAAC;QAED,OAAO;YACL,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;YAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;SACnC,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;OAWG;IACK,aAAa,CACnB,MAAsB,EACtB,WAA+B;QAE/B,MAAM,QAAQ,GAAgB,EAAE,CAAC;QAEjC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,0CAA0C;YAC1C,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC,UAAU;gBAAE,SAAS,CAAC,4BAA4B;YAEvD,4BAA4B;YAC5B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,SAAS;YAE3C,uDAAuD;YACvD,MAAM,QAAQ,GACZ,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YAElE,QAAQ,CAAC,IAAI,CAAC;gBACZ,UAAU;gBACV,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,QAAQ;gBACf,MAAM,EAAE,WAAW;gBACnB,QAAQ,EAAE;oBACR,QAAQ,EAAE,IAAI,CAAC,KAAK;oBACpB,QAAQ,EAAE,sBAAsB;oBAChC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;iBAC/B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.d.ts

@@ -1,102 +0,0 @@
-/**
- * @file NlpPrefilterRecognizer.ts
- * @description Tier 2 NLP-based pre-filter recogniser that uses the
- * `compromise` library for lightweight named-entity extraction.
- *
- * This recogniser is designed as a low-confidence pre-filter: it catches
- * person names, places, and organisations that regex patterns typically miss,
- * then flags them at a low score (0.3–0.6) so that higher tiers (NER model,
- * LLM judge) can confirm or discard them.
- *
- * The `compromise` library is loaded lazily via the {@link ISharedServiceRegistry}
- * so that the heavyweight module is only initialised once and shared across
- * any extensions that need it.
- *
- * @module pii-redaction/recognizers
- */
-import type { PiiEntity, PiiEntityType } from '../types';
-import type { IEntityRecognizer, RecognizeOptions } from './IEntityRecognizer';
-import type { ISharedServiceRegistry } from '../../../ISharedServiceRegistry';
-/**
- * Tier 2 NLP pre-filter recogniser that uses the `compromise` library to
- * detect person names, places, and organisations.
- *
- * ### Design rationale
- * Compromise is a rule-based NLP library (~200 KB) that is much lighter than
- * a full transformer model.  It provides reasonable recall for English-language
- * named entities at the cost of lower precision.  This recogniser intentionally
- * assigns low confidence scores (0.3–0.6) so that its results serve as
- * *candidates* for higher-tier confirmation rather than final detections.
- *
- * ### Graceful degradation
- * If the compromise module fails to load (e.g. not installed, wrong platform),
- * the recogniser sets an internal `unavailable` flag and returns empty results
- * on every subsequent call without throwing.  This ensures the pipeline
- * continues to function with just the regex and/or LLM tiers.
- *
- * ### Shared service pattern
- * The compromise module is loaded via the {@link ISharedServiceRegistry} so
- * that multiple extensions in the same agent can share a single instance.
- *
- * @example
- * ```ts
- * const registry = new SharedServiceRegistry();
- * const recognizer = new NlpPrefilterRecognizer(registry);
- * const entities = await recognizer.recognize('John Smith lives in London');
- * // entities might include PERSON "John Smith" and LOCATION "London"
- * ```
- */
-export declare class NlpPrefilterRecognizer implements IEntityRecognizer {
-    /** @inheritdoc */
-    readonly name = "NlpPrefilterRecognizer";
-    /** @inheritdoc */
-    readonly supportedEntities: PiiEntityType[];
-    /**
-     * When `true`, the compromise module failed to load and all future
-     * `recognize()` calls will return empty arrays silently.
-     */
-    private unavailable;
-    /**
-     * Reference to the shared service registry used for lazy-loading the
-     * compromise NLP module.
-     */
-    private readonly services;
-    /**
-     * Construct a new NlpPrefilterRecognizer.
-     *
-     * @param services - Shared service registry for lazy-loading compromise.
-     */
-    constructor(services: ISharedServiceRegistry);
-    /**
-     * Scan the input text for person names, places, and organisations using
-     * the compromise NLP library.
-     *
-     * Results are returned with low confidence scores (0.3–0.6) to indicate
-     * they are pre-filter candidates requiring higher-tier confirmation.
-     *
-     * @param input   - Raw text to analyse.
-     * @param options - Optional filtering and context hints.
-     * @returns Array of low-confidence {@link PiiEntity} candidates.
-     */
-    recognize(input: string, options?: RecognizeOptions): Promise<PiiEntity[]>;
-    /** @inheritdoc */
-    dispose(): Promise<void>;
-    /**
-     * Determines which of our supported entity types the caller wants.
-     *
-     * @param entityTypes - Optional entity-type filter from the caller.
-     * @returns Set of wanted types intersected with our supported types.
-     */
-    private resolveWantedTypes;
-    /**
-     * Computes a contiguous [start, end) span from a compromise offset result.
-     *
-     * Compromise returns individual term offsets; this method merges them into
-     * a single span covering the full match text.
-     *
-     * @param match - A compromise offset result with term-level positions.
-     * @returns Object with `start` and `end` offsets, or `null` if no terms.
-     */
-    private computeSpan;
-}
-//# sourceMappingURL=NlpPrefilterRecognizer.d.ts.map

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"NlpPrefilterRecognizer.d.ts","sourceRoot":"","sources":["../../../../../src/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzD,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAwF9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,sBAAuB,YAAW,iBAAiB;IAC9D,kBAAkB;IAClB,SAAgB,IAAI,4BAA4B;IAEhD,kBAAkB;IAClB,SAAgB,iBAAiB,EAAE,aAAa,EAAE,CAIhD;IAEF;;;OAGG;IACH,OAAO,CAAC,WAAW,CAAS;IAE5B;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;IAElD;;;;OAIG;gBACS,QAAQ,EAAE,sBAAsB;IAI5C;;;;;;;;;;OAUG;IACU,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IA4FvF,kBAAkB;IACL,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IASrC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAQ1B;;;;;;;;OAQG;IACH,OAAO,CAAC,WAAW;CAepB"}

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.js

@@ -1,228 +0,0 @@
-/**
- * @file NlpPrefilterRecognizer.ts
- * @description Tier 2 NLP-based pre-filter recogniser that uses the
- * `compromise` library for lightweight named-entity extraction.
- *
- * This recogniser is designed as a low-confidence pre-filter: it catches
- * person names, places, and organisations that regex patterns typically miss,
- * then flags them at a low score (0.3–0.6) so that higher tiers (NER model,
- * LLM judge) can confirm or discard them.
- *
- * The `compromise` library is loaded lazily via the {@link ISharedServiceRegistry}
- * so that the heavyweight module is only initialised once and shared across
- * any extensions that need it.
- *
- * @module pii-redaction/recognizers
- */
-// ---------------------------------------------------------------------------
-// Service identity constant for the shared compromise instance
-// ---------------------------------------------------------------------------
-/**
- * Stable service ID used to store the compromise module in the shared
- * service registry.  Other extensions that need compromise can reuse the
- * same cached instance.
- */
-const COMPROMISE_SERVICE_ID = 'agentos:nlp:compromise';
-// ---------------------------------------------------------------------------
-// Score constants
-// ---------------------------------------------------------------------------
-/**
- * Score assigned to person name detections.  Higher than places/orgs because
- * compromise's person-name heuristics tend to be slightly more reliable.
- */
-const PERSON_SCORE = 0.55;
-/** Score assigned to location/place detections. */
-const LOCATION_SCORE = 0.45;
-/** Score assigned to organisation detections. */
-const ORG_SCORE = 0.4;
-// ---------------------------------------------------------------------------
-// NlpPrefilterRecognizer
-// ---------------------------------------------------------------------------
-/**
- * Tier 2 NLP pre-filter recogniser that uses the `compromise` library to
- * detect person names, places, and organisations.
- *
- * ### Design rationale
- * Compromise is a rule-based NLP library (~200 KB) that is much lighter than
- * a full transformer model.  It provides reasonable recall for English-language
- * named entities at the cost of lower precision.  This recogniser intentionally
- * assigns low confidence scores (0.3–0.6) so that its results serve as
- * *candidates* for higher-tier confirmation rather than final detections.
- *
- * ### Graceful degradation
- * If the compromise module fails to load (e.g. not installed, wrong platform),
- * the recogniser sets an internal `unavailable` flag and returns empty results
- * on every subsequent call without throwing.  This ensures the pipeline
- * continues to function with just the regex and/or LLM tiers.
- *
- * ### Shared service pattern
- * The compromise module is loaded via the {@link ISharedServiceRegistry} so
- * that multiple extensions in the same agent can share a single instance.
- *
- * @example
- * ```ts
- * const registry = new SharedServiceRegistry();
- * const recognizer = new NlpPrefilterRecognizer(registry);
- * const entities = await recognizer.recognize('John Smith lives in London');
- * // entities might include PERSON "John Smith" and LOCATION "London"
- * ```
- */
-export class NlpPrefilterRecognizer {
-    /**
-     * Construct a new NlpPrefilterRecognizer.
-     *
-     * @param services - Shared service registry for lazy-loading compromise.
-     */
-    constructor(services) {
-        /** @inheritdoc */
-        this.name = 'NlpPrefilterRecognizer';
-        /** @inheritdoc */
-        this.supportedEntities = [
-            'PERSON',
-            'LOCATION',
-            'ORGANIZATION',
-        ];
-        /**
-         * When `true`, the compromise module failed to load and all future
-         * `recognize()` calls will return empty arrays silently.
-         */
-        this.unavailable = false;
-        this.services = services;
-    }
-    /**
-     * Scan the input text for person names, places, and organisations using
-     * the compromise NLP library.
-     *
-     * Results are returned with low confidence scores (0.3–0.6) to indicate
-     * they are pre-filter candidates requiring higher-tier confirmation.
-     *
-     * @param input   - Raw text to analyse.
-     * @param options - Optional filtering and context hints.
-     * @returns Array of low-confidence {@link PiiEntity} candidates.
-     */
-    async recognize(input, options) {
-        // If compromise failed to load previously, bail out silently.
-        if (this.unavailable)
-            return [];
-        // Determine which entity types the caller wants from us.
-        const wantedTypes = this.resolveWantedTypes(options?.entityTypes);
-        if (wantedTypes.size === 0)
-            return [];
-        // Lazily load compromise via the shared service registry.
-        let nlp;
-        try {
-            nlp = await this.services.getOrCreate(COMPROMISE_SERVICE_ID, async () => {
-                // Dynamic import so compromise is optional at the module level.
-                const mod = await import('compromise');
-                // compromise's default export may be the function directly or
-                // wrapped in { default: ... } depending on the bundler.
-                return mod.default ?? mod;
-            });
-        }
-        catch {
-            // compromise is not installed or failed to load — degrade gracefully.
-            this.unavailable = true;
-            return [];
-        }
-        // Parse the input text with compromise.
-        const doc = nlp(input);
-        const entities = [];
-        // Extract people (PERSON entities).
-        if (wantedTypes.has('PERSON')) {
-            const people = doc.people().out('offset');
-            for (const match of people) {
-                const span = this.computeSpan(match);
-                if (span) {
-                    entities.push({
-                        entityType: 'PERSON',
-                        text: match.text,
-                        start: span.start,
-                        end: span.end,
-                        score: PERSON_SCORE,
-                        source: 'nlp-prefilter',
-                        metadata: { recognizer: 'compromise', method: 'people' },
-                    });
-                }
-            }
-        }
-        // Extract places (LOCATION entities).
-        if (wantedTypes.has('LOCATION')) {
-            const places = doc.places().out('offset');
-            for (const match of places) {
-                const span = this.computeSpan(match);
-                if (span) {
-                    entities.push({
-                        entityType: 'LOCATION',
-                        text: match.text,
-                        start: span.start,
-                        end: span.end,
-                        score: LOCATION_SCORE,
-                        source: 'nlp-prefilter',
-                        metadata: { recognizer: 'compromise', method: 'places' },
-                    });
-                }
-            }
-        }
-        // Extract organisations (ORGANIZATION entities).
-        if (wantedTypes.has('ORGANIZATION')) {
-            const orgs = doc.organizations().out('offset');
-            for (const match of orgs) {
-                const span = this.computeSpan(match);
-                if (span) {
-                    entities.push({
-                        entityType: 'ORGANIZATION',
-                        text: match.text,
-                        start: span.start,
-                        end: span.end,
-                        score: ORG_SCORE,
-                        source: 'nlp-prefilter',
-                        metadata: { recognizer: 'compromise', method: 'organizations' },
-                    });
-                }
-            }
-        }
-        return entities;
-    }
-    /** @inheritdoc */
-    async dispose() {
-        // The compromise instance is owned by the shared service registry and
-        // will be cleaned up when the registry is released.  Nothing to do here.
-    }
-    // -----------------------------------------------------------------------
-    // Private helpers
-    // -----------------------------------------------------------------------
-    /**
-     * Determines which of our supported entity types the caller wants.
-     *
-     * @param entityTypes - Optional entity-type filter from the caller.
-     * @returns Set of wanted types intersected with our supported types.
-     */
-    resolveWantedTypes(entityTypes) {
-        if (!entityTypes || entityTypes.length === 0) {
-            return new Set(this.supportedEntities);
-        }
-        const supported = new Set(this.supportedEntities);
-        return new Set(entityTypes.filter((t) => supported.has(t)));
-    }
-    /**
-     * Computes a contiguous [start, end) span from a compromise offset result.
-     *
-     * Compromise returns individual term offsets; this method merges them into
-     * a single span covering the full match text.
-     *
-     * @param match - A compromise offset result with term-level positions.
-     * @returns Object with `start` and `end` offsets, or `null` if no terms.
-     */
-    computeSpan(match) {
-        if (!match.terms || match.terms.length === 0)
-            return null;
-        // Start is the offset of the first term.
-        const firstTerm = match.terms[0];
-        const start = firstTerm.offset.start;
-        // End is the offset + length of the last term.
-        const lastTerm = match.terms[match.terms.length - 1];
-        const end = lastTerm.offset.start + lastTerm.offset.length;
-        return { start, end };
-    }
-}
-//# sourceMappingURL=NlpPrefilterRecognizer.js.map

package/dist/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"NlpPrefilterRecognizer.js","sourceRoot":"","sources":["../../../../../src/extensions/packs/pii-redaction/recognizers/NlpPrefilterRecognizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,8EAA8E;AAC9E,+DAA+D;AAC/D,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,wBAAwB,CAAC;AAyDvD,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,mDAAmD;AACnD,MAAM,cAAc,GAAG,IAAI,CAAC;AAE5B,iDAAiD;AACjD,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,sBAAsB;IAuBjC;;;;OAIG;IACH,YAAY,QAAgC;QA3B5C,kBAAkB;QACF,SAAI,GAAG,wBAAwB,CAAC;QAEhD,kBAAkB;QACF,sBAAiB,GAAoB;YACnD,QAAQ;YACR,UAAU;YACV,cAAc;SACf,CAAC;QAEF;;;WAGG;QACK,gBAAW,GAAG,KAAK,CAAC;QAc1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;;;;;;;;OAUG;IACI,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,OAA0B;QAC9D,8DAA8D;QAC9D,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAEhC,yDAAyD;QACzD,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAClE,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,0DAA0D;QAC1D,IAAI,GAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CACnC,qBAAqB,EACrB,KAAK,IAAI,EAAE;gBACT,gEAAgE;gBAChE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;gBACvC,8DAA8D;gBAC9D,wDAAwD;gBACxD,OAAQ,GAA+B,CAAC,OAA2B,IAAI,GAAkC,CAAC;YAC5G,CAAC,CACF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,sEAAsE;YACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,wCAAwC;QACxC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;QAEvB,MAAM,QAAQ,GAAgB,EAAE,CAAC;QAEjC,oCAAoC;QACpC,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,UAAU,EAAE,QAAQ;wBACpB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,KAAK,EAAE,YAAY;wBACnB,MAAM,EAAE,eAAe;wBACvB,QAAQ,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE;qBACzD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,UAAU,EAAE,UAAU;wBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,KAAK,EAAE,cAAc;wBACrB,MAAM,EAAE,eAAe;wBACvB,QAAQ,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE;qBACzD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,iDAAiD;QACjD,IAAI,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/C,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,UAAU,EAAE,cAAc;wBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,KAAK,EAAE,SAAS;wBAChB,MAAM,EAAE,eAAe;wBACvB,QAAQ,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,eAAe,EAAE;qBAChE,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,kBAAkB;IACX,KAAK,CAAC,OAAO;QAClB,sEAAsE;QACtE,yEAAyE;IAC3E,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E;;;;;OAKG;IACK,kBAAkB,CAAC,WAA6B;QACtD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAClD,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;;;;OAQG;IACK,WAAW,CACjB,KAA6B;QAE7B,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAE1D,yCAAyC;QACzC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;QAErC,+CAA+C;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;QAE3D,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACxB,CAAC;CACF"}