@framers/agentos 0.1.124 → 0.1.126

package/dist/core/text-processing/TextProcessingPipeline.d.ts

@@ -0,0 +1,49 @@
+/**
+ * @fileoverview Configurable text processing pipeline.
+ * Chains a tokenizer + N processors to produce processed tokens from raw text.
+ *
+ * @module agentos/core/text-processing/TextProcessingPipeline
+ */
+import type { Token } from './types';
+import type { ITextProcessor, ITokenizer } from './ITextProcessor';
+/**
+ * Orchestrates a chain of text processors: tokenizer → processors → output.
+ *
+ * @example
+ * const pipeline = new TextProcessingPipeline(new StandardTokenizer())
+ *   .add(new LowercaseNormalizer())
+ *   .add(new StopWordFilter(ENGLISH_STOP_WORDS))
+ *   .add(new PorterStemmer());
+ *
+ * const tokens = pipeline.process('The quick brown foxes are running');
+ * // tokens[0].text === 'quick', tokens[0].stem === 'quick'
+ * // tokens[1].text === 'brown', ...
+ */
+export declare class TextProcessingPipeline {
+    private tokenizer;
+    private processors;
+    /**
+     * @param tokenizer — the first stage that splits raw text into tokens
+     */
+    constructor(tokenizer: ITokenizer);
+    /** Add a processing stage to the pipeline. Returns `this` for chaining. */
+    add(processor: ITextProcessor): this;
+    /**
+     * Process raw text through the full pipeline.
+     *
+     * @param text — raw input text
+     * @returns array of processed tokens with position and linguistic annotations
+     */
+    process(text: string): Token[];
+    /**
+     * Convenience: process text and return just the token strings.
+     * Useful for BM25 indexing and FTS where only the text values are needed.
+     *
+     * @param text — raw input text
+     * @returns array of processed token strings
+     */
+    processToStrings(text: string): string[];
+    /** Get the names of all stages for debugging. */
+    getStageNames(): string[];
+}
+//# sourceMappingURL=TextProcessingPipeline.d.ts.map

package/dist/core/text-processing/TextProcessingPipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextProcessingPipeline.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/TextProcessingPipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;;;;;;;GAYG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,UAAU,CAAwB;IAE1C;;OAEG;gBACS,SAAS,EAAE,UAAU;IAIjC,2EAA2E;IAC3E,GAAG,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI;IAKpC;;;;;OAKG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE;IAU9B;;;;;;OAMG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAIxC,iDAAiD;IACjD,aAAa,IAAI,MAAM,EAAE;CAG1B"}

package/dist/core/text-processing/TextProcessingPipeline.js

@@ -0,0 +1,61 @@
+/**
+ * @fileoverview Configurable text processing pipeline.
+ * Chains a tokenizer + N processors to produce processed tokens from raw text.
+ *
+ * @module agentos/core/text-processing/TextProcessingPipeline
+ */
+/**
+ * Orchestrates a chain of text processors: tokenizer → processors → output.
+ *
+ * @example
+ * const pipeline = new TextProcessingPipeline(new StandardTokenizer())
+ *   .add(new LowercaseNormalizer())
+ *   .add(new StopWordFilter(ENGLISH_STOP_WORDS))
+ *   .add(new PorterStemmer());
+ *
+ * const tokens = pipeline.process('The quick brown foxes are running');
+ * // tokens[0].text === 'quick', tokens[0].stem === 'quick'
+ * // tokens[1].text === 'brown', ...
+ */
+export class TextProcessingPipeline {
+    /**
+     * @param tokenizer — the first stage that splits raw text into tokens
+     */
+    constructor(tokenizer) {
+        this.processors = [];
+        this.tokenizer = tokenizer;
+    }
+    /** Add a processing stage to the pipeline. Returns `this` for chaining. */
+    add(processor) {
+        this.processors.push(processor);
+        return this;
+    }
+    /**
+     * Process raw text through the full pipeline.
+     *
+     * @param text — raw input text
+     * @returns array of processed tokens with position and linguistic annotations
+     */
+    process(text) {
+        let tokens = this.tokenizer.tokenize(text);
+        for (const processor of this.processors) {
+            tokens = processor.process(tokens);
+        }
+        return tokens;
+    }
+    /**
+     * Convenience: process text and return just the token strings.
+     * Useful for BM25 indexing and FTS where only the text values are needed.
+     *
+     * @param text — raw input text
+     * @returns array of processed token strings
+     */
+    processToStrings(text) {
+        return this.process(text).map(t => t.text);
+    }
+    /** Get the names of all stages for debugging. */
+    getStageNames() {
+        return [this.tokenizer.name, ...this.processors.map(p => p.name)];
+    }
+}
+//# sourceMappingURL=TextProcessingPipeline.js.map

package/dist/core/text-processing/TextProcessingPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextProcessingPipeline.js","sourceRoot":"","sources":["../../../src/core/text-processing/TextProcessingPipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,sBAAsB;IAIjC;;OAEG;IACH,YAAY,SAAqB;QALzB,eAAU,GAAqB,EAAE,CAAC;QAMxC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,2EAA2E;IAC3E,GAAG,CAAC,SAAyB;QAC3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,IAAY;QAClB,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE3C,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,gBAAgB,CAAC,IAAY;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,iDAAiD;IACjD,aAAa;QACX,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,CAAC;CACF"}

package/dist/core/text-processing/filters/StopWordFilter.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @fileoverview Removes tokens that match a configurable stop word list.
+ * @module agentos/core/text-processing/filters/StopWordFilter
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+/** ~120 common English stop words. */
+export declare const ENGLISH_STOP_WORDS: ReadonlySet<string>;
+/**
+ * Stop words for code search. These are noise in code but NOT programming keywords.
+ * Note: `class`, `function`, `import`, `const`, `let`, `var`, `return`, `this`
+ * are deliberately NOT in this list — they're meaningful in code search.
+ */
+export declare const CODE_STOP_WORDS: ReadonlySet<string>;
+export declare function getNaturalStopWords(): ReadonlySet<string>;
+/**
+ * Filters tokens whose `.text` appears in the provided stop word set.
+ * Case-sensitive — apply after LowercaseNormalizer for case-insensitive filtering.
+ */
+export declare class StopWordFilter implements ITextProcessor {
+    readonly name = "StopWordFilter";
+    private stopWords;
+    /**
+     * @param stopWords — stop word set to filter against. Defaults to `natural`'s
+     * 170-word list when available, falls back to the built-in 120-word ENGLISH_STOP_WORDS.
+     */
+    constructor(stopWords?: ReadonlySet<string>);
+    process(tokens: Token[]): Token[];
+}
+//# sourceMappingURL=StopWordFilter.d.ts.map

package/dist/core/text-processing/filters/StopWordFilter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StopWordFilter.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/filters/StopWordFilter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,sCAAsC;AACtC,eAAO,MAAM,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAejD,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAAE,WAAW,CAAC,MAAM,CAW9C,CAAC;AAOH,wBAAgB,mBAAmB,IAAI,WAAW,CAAC,MAAM,CAAC,CAYzD;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,IAAI,oBAAoB;IACjC,OAAO,CAAC,SAAS,CAAsB;IAEvC;;;OAGG;gBACS,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC;IAI3C,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}

package/dist/core/text-processing/filters/StopWordFilter.js

@@ -0,0 +1,76 @@
+/**
+ * @fileoverview Removes tokens that match a configurable stop word list.
+ * @module agentos/core/text-processing/filters/StopWordFilter
+ */
+/** ~120 common English stop words. */
+export const ENGLISH_STOP_WORDS = new Set([
+    'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and',
+    'any', 'are', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below',
+    'between', 'both', 'but', 'by', 'can', 'could', 'did', 'do', 'does', 'doing',
+    'down', 'during', 'each', 'few', 'for', 'from', 'further', 'get', 'got', 'had',
+    'has', 'have', 'having', 'he', 'her', 'here', 'hers', 'herself', 'him',
+    'himself', 'his', 'how', 'if', 'in', 'into', 'is', 'it', 'its', 'itself',
+    'just', 'me', 'might', 'more', 'most', 'must', 'my', 'myself', 'no', 'nor',
+    'not', 'now', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'our', 'ours',
+    'ourselves', 'out', 'over', 'own', 'same', 'she', 'should', 'so', 'some',
+    'such', 'than', 'that', 'the', 'their', 'theirs', 'them', 'themselves', 'then',
+    'there', 'these', 'they', 'this', 'those', 'through', 'to', 'too', 'under',
+    'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which',
+    'while', 'who', 'whom', 'why', 'will', 'with', 'would', 'you', 'your', 'yours',
+    'yourself', 'yourselves',
+]);
+/**
+ * Stop words for code search. These are noise in code but NOT programming keywords.
+ * Note: `class`, `function`, `import`, `const`, `let`, `var`, `return`, `this`
+ * are deliberately NOT in this list — they're meaningful in code search.
+ */
+export const CODE_STOP_WORDS = new Set([
+    'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+    'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
+    'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
+    'before', 'after', 'above', 'below', 'between', 'out', 'off', 'over',
+    'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when',
+    'where', 'why', 'how', 'all', 'each', 'every', 'both', 'few', 'more',
+    'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own',
+    'same', 'so', 'than', 'too', 'very', 'just', 'because', 'but', 'and',
+    'or', 'if', 'while', 'about', 'up', 'out', 'also', 'it', 'its',
+]);
+/**
+ * Extended stop word list from the `natural` NLP library (170 words).
+ * Loaded lazily — falls back to ENGLISH_STOP_WORDS if natural is unavailable.
+ */
+let _naturalStopWords = null;
+export function getNaturalStopWords() {
+    if (_naturalStopWords)
+        return _naturalStopWords;
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const natural = require('natural');
+        if (natural.stopwords && Array.isArray(natural.stopwords)) {
+            _naturalStopWords = new Set(natural.stopwords);
+            return _naturalStopWords;
+        }
+    }
+    catch { /* natural not installed */ }
+    _naturalStopWords = ENGLISH_STOP_WORDS;
+    return _naturalStopWords;
+}
+/**
+ * Filters tokens whose `.text` appears in the provided stop word set.
+ * Case-sensitive — apply after LowercaseNormalizer for case-insensitive filtering.
+ */
+export class StopWordFilter {
+    /**
+     * @param stopWords — stop word set to filter against. Defaults to `natural`'s
+     * 170-word list when available, falls back to the built-in 120-word ENGLISH_STOP_WORDS.
+     */
+    constructor(stopWords) {
+        this.name = 'StopWordFilter';
+        this.stopWords = stopWords ?? getNaturalStopWords();
+    }
+    process(tokens) {
+        return tokens.filter(t => !this.stopWords.has(t.text));
+    }
+}
+//# sourceMappingURL=StopWordFilter.js.map

package/dist/core/text-processing/filters/StopWordFilter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StopWordFilter.js","sourceRoot":"","sources":["../../../../src/core/text-processing/filters/StopWordFilter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,sCAAsC;AACtC,MAAM,CAAC,MAAM,kBAAkB,GAAwB,IAAI,GAAG,CAAC;IAC7D,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IAC5E,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO;IAC7E,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IAC5E,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAC9E,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK;IACtE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ;IACxE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK;IAC1E,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM;IAC7E,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM;IACxE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM;IAC9E,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO;IAC1E,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAC5E,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC9E,UAAU,EAAE,YAAY;CACzB,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC;IAC1D,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACnE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IACjE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ;IACnE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;IACpE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACpE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IACpE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;IAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK;IACpE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK;CAC/D,CAAC,CAAC;AAEH;;;GAGG;AACH,IAAI,iBAAiB,GAA+B,IAAI,CAAC;AACzD,MAAM,UAAU,mBAAmB;IACjC,IAAI,iBAAiB;QAAE,OAAO,iBAAiB,CAAC;IAChD,IAAI,CAAC;QACH,8DAA8D;QAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACnC,IAAI,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1D,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,SAAqB,CAAC,CAAC;YAC3D,OAAO,iBAAiB,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,2BAA2B,CAAC,CAAC;IACvC,iBAAiB,GAAG,kBAAkB,CAAC;IACvC,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,cAAc;IAIzB;;;OAGG;IACH,YAAY,SAA+B;QAPlC,SAAI,GAAG,gBAAgB,CAAC;QAQ/B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,mBAAmB,EAAE,CAAC;IACtD,CAAC;IAED,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACzD,CAAC;CACF"}

package/dist/core/text-processing/index.d.ts

@@ -0,0 +1,20 @@
+/**
+ * @fileoverview Pluggable text processing pipeline for AgentOS.
+ * Feeds into BM25 keyword search, FTS5, and embedding generation.
+ *
+ * @module agentos/core/text-processing
+ */
+export { TextProcessingPipeline } from './TextProcessingPipeline';
+export type { ITextProcessor, ITokenizer } from './ITextProcessor';
+export type { Token } from './types';
+export { StandardTokenizer } from './tokenizers/StandardTokenizer';
+export { CodeTokenizer } from './tokenizers/CodeTokenizer';
+export { LowercaseNormalizer } from './normalizers/LowercaseNormalizer';
+export { AccentStripper } from './normalizers/AccentStripper';
+export { StopWordFilter, ENGLISH_STOP_WORDS, CODE_STOP_WORDS, getNaturalStopWords } from './filters/StopWordFilter';
+export { PorterStemmer } from './stemmers/PorterStemmer';
+export { NoOpStemmer } from './stemmers/NoOpStemmer';
+export { WordNetLemmatizer } from './lemmatizers/WordNetLemmatizer';
+export { NoOpLemmatizer } from './lemmatizers/NoOpLemmatizer';
+export { createProsePipeline, createCodePipeline, createRagPipeline } from './presets';
+//# sourceMappingURL=index.d.ts.map

package/dist/core/text-processing/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnE,YAAY,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAGrC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAG9D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAGpH,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAGrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAG9D,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC"}

package/dist/core/text-processing/index.js

@@ -0,0 +1,24 @@
+/**
+ * @fileoverview Pluggable text processing pipeline for AgentOS.
+ * Feeds into BM25 keyword search, FTS5, and embedding generation.
+ *
+ * @module agentos/core/text-processing
+ */
+export { TextProcessingPipeline } from './TextProcessingPipeline.js';
+// Tokenizers
+export { StandardTokenizer } from './tokenizers/StandardTokenizer.js';
+export { CodeTokenizer } from './tokenizers/CodeTokenizer.js';
+// Normalizers
+export { LowercaseNormalizer } from './normalizers/LowercaseNormalizer.js';
+export { AccentStripper } from './normalizers/AccentStripper.js';
+// Filters
+export { StopWordFilter, ENGLISH_STOP_WORDS, CODE_STOP_WORDS, getNaturalStopWords } from './filters/StopWordFilter.js';
+// Stemmers
+export { PorterStemmer } from './stemmers/PorterStemmer.js';
+export { NoOpStemmer } from './stemmers/NoOpStemmer.js';
+// Lemmatizers
+export { WordNetLemmatizer } from './lemmatizers/WordNetLemmatizer.js';
+export { NoOpLemmatizer } from './lemmatizers/NoOpLemmatizer.js';
+// Presets
+export { createProsePipeline, createCodePipeline, createRagPipeline } from './presets.js';
+//# sourceMappingURL=index.js.map

package/dist/core/text-processing/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/text-processing/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAIlE,aAAa;AACb,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,cAAc;AACd,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAEpH,WAAW;AACX,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,cAAc;AACd,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,UAAU;AACV,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC"}

package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @fileoverview Pass-through lemmatizer that does nothing.
+ * @module agentos/core/text-processing/lemmatizers/NoOpLemmatizer
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+export declare class NoOpLemmatizer implements ITextProcessor {
+    readonly name = "NoOpLemmatizer";
+    process(tokens: Token[]): Token[];
+}
+//# sourceMappingURL=NoOpLemmatizer.d.ts.map

package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"NoOpLemmatizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/NoOpLemmatizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}

package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.js

@@ -0,0 +1,13 @@
+/**
+ * @fileoverview Pass-through lemmatizer that does nothing.
+ * @module agentos/core/text-processing/lemmatizers/NoOpLemmatizer
+ */
+export class NoOpLemmatizer {
+    constructor() {
+        this.name = 'NoOpLemmatizer';
+    }
+    process(tokens) {
+        return tokens;
+    }
+}
+//# sourceMappingURL=NoOpLemmatizer.js.map

package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NoOpLemmatizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/NoOpLemmatizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,OAAO,cAAc;IAA3B;QACW,SAAI,GAAG,gBAAgB,CAAC;IAKnC,CAAC;IAHC,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @fileoverview WordNet lemmatizer wrapping the `natural` package.
+ * Falls back to no-op if `natural` is not installed.
+ *
+ * Lemmatization produces the dictionary form of a word:
+ * `ran` → `run`, `better` → `good`, `mice` → `mouse`.
+ *
+ * @module agentos/core/text-processing/lemmatizers/WordNetLemmatizer
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+/**
+ * WordNet-based lemmatizer. Reduces words to their dictionary (lemma) form.
+ *
+ * Sets `token.lemma` and updates `token.text` to the lemmatized form.
+ * Falls back to Lancaster stemming if full WordNet lookup is unavailable.
+ */
+export declare class WordNetLemmatizer implements ITextProcessor {
+    readonly name = "WordNetLemmatizer";
+    private initialized;
+    private ensureLoaded;
+    process(tokens: Token[]): Token[];
+    initialize(): Promise<void>;
+}
+//# sourceMappingURL=WordNetLemmatizer.d.ts.map

package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WordNetLemmatizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/WordNetLemmatizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AA0BxD;;;;;GAKG;AACH,qBAAa,iBAAkB,YAAW,cAAc;IACtD,QAAQ,CAAC,IAAI,uBAAuB;IAEpC,OAAO,CAAC,WAAW,CAAS;YAEd,YAAY;IAO1B,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;IAS3B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;CAGlC"}

package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.js

@@ -0,0 +1,64 @@
+/**
+ * @fileoverview WordNet lemmatizer wrapping the `natural` package.
+ * Falls back to no-op if `natural` is not installed.
+ *
+ * Lemmatization produces the dictionary form of a word:
+ * `ran` → `run`, `better` → `good`, `mice` → `mouse`.
+ *
+ * @module agentos/core/text-processing/lemmatizers/WordNetLemmatizer
+ */
+/** Lazy-loaded lemmatize function from the `natural` package. */
+let lemmatizeFn = null;
+let loadAttempted = false;
+async function loadLemmatizer() {
+    if (loadAttempted)
+        return;
+    loadAttempted = true;
+    try {
+        const natural = await import('natural');
+        const wordnet = new natural.WordNet();
+        lemmatizeFn = (word) => {
+            /* WordNet lookup is async in natural, but we need sync for the pipeline.
+               Use the synchronous stemmer-based lemmatizer as a practical fallback. */
+            try {
+                return natural.LancasterStemmer.stem(word);
+            }
+            catch {
+                return word;
+            }
+        };
+    }
+    catch {
+        lemmatizeFn = null;
+    }
+}
+/**
+ * WordNet-based lemmatizer. Reduces words to their dictionary (lemma) form.
+ *
+ * Sets `token.lemma` and updates `token.text` to the lemmatized form.
+ * Falls back to Lancaster stemming if full WordNet lookup is unavailable.
+ */
+export class WordNetLemmatizer {
+    constructor() {
+        this.name = 'WordNetLemmatizer';
+        this.initialized = false;
+    }
+    async ensureLoaded() {
+        if (!this.initialized) {
+            await loadLemmatizer();
+            this.initialized = true;
+        }
+    }
+    process(tokens) {
+        if (!lemmatizeFn)
+            return tokens;
+        return tokens.map(t => {
+            const lemma = lemmatizeFn(t.text);
+            return { ...t, text: lemma, lemma };
+        });
+    }
+    async initialize() {
+        await this.ensureLoaded();
+    }
+}
+//# sourceMappingURL=WordNetLemmatizer.js.map

package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WordNetLemmatizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/WordNetLemmatizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,iEAAiE;AACjE,IAAI,WAAW,GAAsC,IAAI,CAAC;AAC1D,IAAI,aAAa,GAAG,KAAK,CAAC;AAE1B,KAAK,UAAU,cAAc;IAC3B,IAAI,aAAa;QAAE,OAAO;IAC1B,aAAa,GAAG,IAAI,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACtC,WAAW,GAAG,CAAC,IAAY,EAAE,EAAE;YAC7B;uFAC2E;YAC3E,IAAI,CAAC;gBACH,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,iBAAiB;IAA9B;QACW,SAAI,GAAG,mBAAmB,CAAC;QAE5B,gBAAW,GAAG,KAAK,CAAC;IAqB9B,CAAC;IAnBS,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAe;QACrB,IAAI,CAAC,WAAW;YAAE,OAAO,MAAM,CAAC;QAEhC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACpB,MAAM,KAAK,GAAG,WAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACnC,OAAO,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC5B,CAAC;CACF"}

package/dist/core/text-processing/normalizers/AccentStripper.d.ts

@@ -0,0 +1,17 @@
+/**
+ * @fileoverview Strips diacritics/accents from token text.
+ * `café` → `cafe`, `naïve` → `naive`.
+ *
+ * @module agentos/core/text-processing/normalizers/AccentStripper
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+/**
+ * Removes combining diacritical marks after Unicode NFD decomposition.
+ * This makes accent-insensitive search possible.
+ */
+export declare class AccentStripper implements ITextProcessor {
+    readonly name = "AccentStripper";
+    process(tokens: Token[]): Token[];
+}
+//# sourceMappingURL=AccentStripper.d.ts.map

package/dist/core/text-processing/normalizers/AccentStripper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AccentStripper.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/AccentStripper.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;;GAGG;AACH,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAMlC"}

package/dist/core/text-processing/normalizers/AccentStripper.js

@@ -0,0 +1,22 @@
+/**
+ * @fileoverview Strips diacritics/accents from token text.
+ * `café` → `cafe`, `naïve` → `naive`.
+ *
+ * @module agentos/core/text-processing/normalizers/AccentStripper
+ */
+/**
+ * Removes combining diacritical marks after Unicode NFD decomposition.
+ * This makes accent-insensitive search possible.
+ */
+export class AccentStripper {
+    constructor() {
+        this.name = 'AccentStripper';
+    }
+    process(tokens) {
+        return tokens.map(t => ({
+            ...t,
+            text: t.text.normalize('NFD').replace(/[\u0300-\u036f]/g, ''),
+        }));
+    }
+}
+//# sourceMappingURL=AccentStripper.js.map

package/dist/core/text-processing/normalizers/AccentStripper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AccentStripper.js","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/AccentStripper.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH;;;GAGG;AACH,MAAM,OAAO,cAAc;IAA3B;QACW,SAAI,GAAG,gBAAgB,CAAC;IAQnC,CAAC;IANC,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACtB,GAAG,CAAC;YACJ,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC9D,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}

package/dist/core/text-processing/normalizers/LowercaseNormalizer.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @fileoverview Lowercases all token text. Preserves original.
+ * @module agentos/core/text-processing/normalizers/LowercaseNormalizer
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+export declare class LowercaseNormalizer implements ITextProcessor {
+    readonly name = "LowercaseNormalizer";
+    process(tokens: Token[]): Token[];
+}
+//# sourceMappingURL=LowercaseNormalizer.d.ts.map

package/dist/core/text-processing/normalizers/LowercaseNormalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LowercaseNormalizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/LowercaseNormalizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,yBAAyB;IAEtC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}

package/dist/core/text-processing/normalizers/LowercaseNormalizer.js

@@ -0,0 +1,13 @@
+/**
+ * @fileoverview Lowercases all token text. Preserves original.
+ * @module agentos/core/text-processing/normalizers/LowercaseNormalizer
+ */
+export class LowercaseNormalizer {
+    constructor() {
+        this.name = 'LowercaseNormalizer';
+    }
+    process(tokens) {
+        return tokens.map(t => ({ ...t, text: t.text.toLowerCase() }));
+    }
+}
+//# sourceMappingURL=LowercaseNormalizer.js.map

package/dist/core/text-processing/normalizers/LowercaseNormalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LowercaseNormalizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/LowercaseNormalizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,OAAO,mBAAmB;IAAhC;QACW,SAAI,GAAG,qBAAqB,CAAC;IAKxC,CAAC;IAHC,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;CACF"}

package/dist/core/text-processing/presets.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @fileoverview Pre-built pipeline configurations for common use cases.
+ * @module agentos/core/text-processing/presets
+ */
+import { TextProcessingPipeline } from './TextProcessingPipeline';
+/**
+ * Pipeline for English prose text.
+ * Standard tokenizer → lowercase → strip accents → remove stop words → Porter stem.
+ */
+export declare function createProsePipeline(): TextProcessingPipeline;
+/**
+ * Pipeline for source code and technical identifiers.
+ * Code tokenizer (camelCase/snake_case split) → lowercase → code stop words → no stemming.
+ */
+export declare function createCodePipeline(): TextProcessingPipeline;
+/**
+ * Default pipeline for RAG / hybrid search.
+ * Standard tokenizer → lowercase → remove stop words → Porter stem.
+ * Good balance of recall and precision for mixed-content corpora.
+ */
+export declare function createRagPipeline(): TextProcessingPipeline;
+//# sourceMappingURL=presets.d.ts.map

package/dist/core/text-processing/presets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"presets.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/presets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AASlE;;;GAGG;AACH,wBAAgB,mBAAmB,IAAI,sBAAsB,CAM5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,sBAAsB,CAK3D;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,IAAI,sBAAsB,CAK1D"}

package/dist/core/text-processing/presets.js

@@ -0,0 +1,45 @@
+/**
+ * @fileoverview Pre-built pipeline configurations for common use cases.
+ * @module agentos/core/text-processing/presets
+ */
+import { TextProcessingPipeline } from './TextProcessingPipeline.js';
+import { StandardTokenizer } from './tokenizers/StandardTokenizer.js';
+import { CodeTokenizer } from './tokenizers/CodeTokenizer.js';
+import { LowercaseNormalizer } from './normalizers/LowercaseNormalizer.js';
+import { AccentStripper } from './normalizers/AccentStripper.js';
+import { StopWordFilter, CODE_STOP_WORDS } from './filters/StopWordFilter.js';
+import { PorterStemmer } from './stemmers/PorterStemmer.js';
+import { NoOpStemmer } from './stemmers/NoOpStemmer.js';
+/**
+ * Pipeline for English prose text.
+ * Standard tokenizer → lowercase → strip accents → remove stop words → Porter stem.
+ */
+export function createProsePipeline() {
+    return new TextProcessingPipeline(new StandardTokenizer())
+        .add(new LowercaseNormalizer())
+        .add(new AccentStripper())
+        .add(new StopWordFilter()) /* default: getNaturalStopWords() → 170 words when natural available */
+        .add(new PorterStemmer());
+}
+/**
+ * Pipeline for source code and technical identifiers.
+ * Code tokenizer (camelCase/snake_case split) → lowercase → code stop words → no stemming.
+ */
+export function createCodePipeline() {
+    return new TextProcessingPipeline(new CodeTokenizer())
+        .add(new LowercaseNormalizer())
+        .add(new StopWordFilter(CODE_STOP_WORDS))
+        .add(new NoOpStemmer());
+}
+/**
+ * Default pipeline for RAG / hybrid search.
+ * Standard tokenizer → lowercase → remove stop words → Porter stem.
+ * Good balance of recall and precision for mixed-content corpora.
+ */
+export function createRagPipeline() {
+    return new TextProcessingPipeline(new StandardTokenizer())
+        .add(new LowercaseNormalizer())
+        .add(new StopWordFilter()) /* default: getNaturalStopWords() → 170 words when natural available */
+        .add(new PorterStemmer());
+}
+//# sourceMappingURL=presets.js.map

package/dist/core/text-processing/presets.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"presets.js","sourceRoot":"","sources":["../../../src/core/text-processing/presets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO,IAAI,sBAAsB,CAAC,IAAI,iBAAiB,EAAE,CAAC;SACvD,GAAG,CAAC,IAAI,mBAAmB,EAAE,CAAC;SAC9B,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;SACzB,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC,CAAC,uEAAuE;SACjG,GAAG,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,IAAI,sBAAsB,CAAC,IAAI,aAAa,EAAE,CAAC;SACnD,GAAG,CAAC,IAAI,mBAAmB,EAAE,CAAC;SAC9B,GAAG,CAAC,IAAI,cAAc,CAAC,eAAe,CAAC,CAAC;SACxC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO,IAAI,sBAAsB,CAAC,IAAI,iBAAiB,EAAE,CAAC;SACvD,GAAG,CAAC,IAAI,mBAAmB,EAAE,CAAC;SAC9B,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC,CAAC,uEAAuE;SACjG,GAAG,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAC9B,CAAC"}

package/dist/core/text-processing/stemmers/NoOpStemmer.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @fileoverview Pass-through stemmer that does nothing.
+ * Use for code identifiers where stemming would be harmful
+ * (e.g. `kubernetes` → `kubernet` is wrong).
+ *
+ * @module agentos/core/text-processing/stemmers/NoOpStemmer
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+export declare class NoOpStemmer implements ITextProcessor {
+    readonly name = "NoOpStemmer";
+    process(tokens: Token[]): Token[];
+}
+//# sourceMappingURL=NoOpStemmer.d.ts.map

package/dist/core/text-processing/stemmers/NoOpStemmer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"NoOpStemmer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/stemmers/NoOpStemmer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,qBAAa,WAAY,YAAW,cAAc;IAChD,QAAQ,CAAC,IAAI,iBAAiB;IAE9B,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}

package/dist/core/text-processing/stemmers/NoOpStemmer.js

@@ -0,0 +1,16 @@
+/**
+ * @fileoverview Pass-through stemmer that does nothing.
+ * Use for code identifiers where stemming would be harmful
+ * (e.g. `kubernetes` → `kubernet` is wrong).
+ *
+ * @module agentos/core/text-processing/stemmers/NoOpStemmer
+ */
+export class NoOpStemmer {
+    constructor() {
+        this.name = 'NoOpStemmer';
+    }
+    process(tokens) {
+        return tokens;
+    }
+}
+//# sourceMappingURL=NoOpStemmer.js.map