@framers/agentos 0.1.125 → 0.1.126

package/dist/core/text-processing/normalizers/LowercaseNormalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LowercaseNormalizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/LowercaseNormalizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,OAAO,mBAAmB;IAAhC;QACW,SAAI,GAAG,qBAAqB,CAAC;IAKxC,CAAC;IAHC,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;CACF"}

package/dist/core/text-processing/presets.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @fileoverview Pre-built pipeline configurations for common use cases.
+ * @module agentos/core/text-processing/presets
+ */
+import { TextProcessingPipeline } from './TextProcessingPipeline';
+/**
+ * Pipeline for English prose text.
+ * Standard tokenizer → lowercase → strip accents → remove stop words → Porter stem.
+ */
+export declare function createProsePipeline(): TextProcessingPipeline;
+/**
+ * Pipeline for source code and technical identifiers.
+ * Code tokenizer (camelCase/snake_case split) → lowercase → code stop words → no stemming.
+ */
+export declare function createCodePipeline(): TextProcessingPipeline;
+/**
+ * Default pipeline for RAG / hybrid search.
+ * Standard tokenizer → lowercase → remove stop words → Porter stem.
+ * Good balance of recall and precision for mixed-content corpora.
+ */
+export declare function createRagPipeline(): TextProcessingPipeline;
+//# sourceMappingURL=presets.d.ts.map

package/dist/core/text-processing/presets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"presets.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/presets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AASlE;;;GAGG;AACH,wBAAgB,mBAAmB,IAAI,sBAAsB,CAM5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,sBAAsB,CAK3D;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,IAAI,sBAAsB,CAK1D"}

package/dist/core/text-processing/presets.js

@@ -0,0 +1,45 @@
+/**
+ * @fileoverview Pre-built pipeline configurations for common use cases.
+ * @module agentos/core/text-processing/presets
+ */
+import { TextProcessingPipeline } from './TextProcessingPipeline.js';
+import { StandardTokenizer } from './tokenizers/StandardTokenizer.js';
+import { CodeTokenizer } from './tokenizers/CodeTokenizer.js';
+import { LowercaseNormalizer } from './normalizers/LowercaseNormalizer.js';
+import { AccentStripper } from './normalizers/AccentStripper.js';
+import { StopWordFilter, CODE_STOP_WORDS } from './filters/StopWordFilter.js';
+import { PorterStemmer } from './stemmers/PorterStemmer.js';
+import { NoOpStemmer } from './stemmers/NoOpStemmer.js';
+/**
+ * Pipeline for English prose text.
+ * Standard tokenizer → lowercase → strip accents → remove stop words → Porter stem.
+ */
+export function createProsePipeline() {
+    return new TextProcessingPipeline(new StandardTokenizer())
+        .add(new LowercaseNormalizer())
+        .add(new AccentStripper())
+        .add(new StopWordFilter()) /* default: getNaturalStopWords() → 170 words when natural available */
+        .add(new PorterStemmer());
+}
+/**
+ * Pipeline for source code and technical identifiers.
+ * Code tokenizer (camelCase/snake_case split) → lowercase → code stop words → no stemming.
+ */
+export function createCodePipeline() {
+    return new TextProcessingPipeline(new CodeTokenizer())
+        .add(new LowercaseNormalizer())
+        .add(new StopWordFilter(CODE_STOP_WORDS))
+        .add(new NoOpStemmer());
+}
+/**
+ * Default pipeline for RAG / hybrid search.
+ * Standard tokenizer → lowercase → remove stop words → Porter stem.
+ * Good balance of recall and precision for mixed-content corpora.
+ */
+export function createRagPipeline() {
+    return new TextProcessingPipeline(new StandardTokenizer())
+        .add(new LowercaseNormalizer())
+        .add(new StopWordFilter()) /* default: getNaturalStopWords() → 170 words when natural available */
+        .add(new PorterStemmer());
+}
+//# sourceMappingURL=presets.js.map

package/dist/core/text-processing/presets.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"presets.js","sourceRoot":"","sources":["../../../src/core/text-processing/presets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO,IAAI,sBAAsB,CAAC,IAAI,iBAAiB,EAAE,CAAC;SACvD,GAAG,CAAC,IAAI,mBAAmB,EAAE,CAAC;SAC9B,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;SACzB,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC,CAAC,uEAAuE;SACjG,GAAG,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,IAAI,sBAAsB,CAAC,IAAI,aAAa,EAAE,CAAC;SACnD,GAAG,CAAC,IAAI,mBAAmB,EAAE,CAAC;SAC9B,GAAG,CAAC,IAAI,cAAc,CAAC,eAAe,CAAC,CAAC;SACxC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO,IAAI,sBAAsB,CAAC,IAAI,iBAAiB,EAAE,CAAC;SACvD,GAAG,CAAC,IAAI,mBAAmB,EAAE,CAAC;SAC9B,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC,CAAC,uEAAuE;SACjG,GAAG,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAC9B,CAAC"}

package/dist/core/text-processing/stemmers/NoOpStemmer.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @fileoverview Pass-through stemmer that does nothing.
+ * Use for code identifiers where stemming would be harmful
+ * (e.g. `kubernetes` → `kubernet` is wrong).
+ *
+ * @module agentos/core/text-processing/stemmers/NoOpStemmer
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+export declare class NoOpStemmer implements ITextProcessor {
+    readonly name = "NoOpStemmer";
+    process(tokens: Token[]): Token[];
+}
+//# sourceMappingURL=NoOpStemmer.d.ts.map

package/dist/core/text-processing/stemmers/NoOpStemmer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"NoOpStemmer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/stemmers/NoOpStemmer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,qBAAa,WAAY,YAAW,cAAc;IAChD,QAAQ,CAAC,IAAI,iBAAiB;IAE9B,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}

package/dist/core/text-processing/stemmers/NoOpStemmer.js

@@ -0,0 +1,16 @@
+/**
+ * @fileoverview Pass-through stemmer that does nothing.
+ * Use for code identifiers where stemming would be harmful
+ * (e.g. `kubernetes` → `kubernet` is wrong).
+ *
+ * @module agentos/core/text-processing/stemmers/NoOpStemmer
+ */
+export class NoOpStemmer {
+    constructor() {
+        this.name = 'NoOpStemmer';
+    }
+    process(tokens) {
+        return tokens;
+    }
+}
+//# sourceMappingURL=NoOpStemmer.js.map

package/dist/core/text-processing/stemmers/NoOpStemmer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NoOpStemmer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/stemmers/NoOpStemmer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,MAAM,OAAO,WAAW;IAAxB;QACW,SAAI,GAAG,aAAa,CAAC;IAKhC,CAAC;IAHC,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/core/text-processing/stemmers/PorterStemmer.d.ts

@@ -0,0 +1,31 @@
+/**
+ * @fileoverview Porter stemmer wrapping the `natural` package.
+ * Falls back to no-op if `natural` is not installed.
+ *
+ * @module agentos/core/text-processing/stemmers/PorterStemmer
+ */
+import type { Token } from '../types';
+import type { ITextProcessor } from '../ITextProcessor';
+/**
+ * Porter stemmer — reduces words to their morphological root.
+ * `running` → `run`, `foxes` → `fox`, `connected` → `connect`.
+ *
+ * Uses the `natural` npm package (already in agentos dependencies).
+ * Falls back to no-op if `natural` can't be imported.
+ *
+ * Sets `token.stem` with the stemmed form. Also updates `token.text`
+ * so downstream processors work with stemmed tokens.
+ */
+export declare class PorterStemmer implements ITextProcessor {
+    readonly name = "PorterStemmer";
+    private initialized;
+    private ensureLoaded;
+    process(tokens: Token[]): Token[];
+    /**
+     * Async initialization — call once before first use to load `natural`.
+     * The pipeline calls this automatically, but you can call it early
+     * to avoid the lazy-load delay on first process() call.
+     */
+    initialize(): Promise<void>;
+}
+//# sourceMappingURL=PorterStemmer.d.ts.map

package/dist/core/text-processing/stemmers/PorterStemmer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PorterStemmer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/stemmers/PorterStemmer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAkBxD;;;;;;;;;GASG;AACH,qBAAa,aAAc,YAAW,cAAc;IAClD,QAAQ,CAAC,IAAI,mBAAmB;IAEhC,OAAO,CAAC,WAAW,CAAS;YAEd,YAAY;IAO1B,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;IAUjC;;;;OAIG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;CAGlC"}

package/dist/core/text-processing/stemmers/PorterStemmer.js

@@ -0,0 +1,62 @@
+/**
+ * @fileoverview Porter stemmer wrapping the `natural` package.
+ * Falls back to no-op if `natural` is not installed.
+ *
+ * @module agentos/core/text-processing/stemmers/PorterStemmer
+ */
+/** Lazy-loaded stem function from the `natural` package. */
+let stemFn = null;
+let loadAttempted = false;
+async function loadStemmer() {
+    if (loadAttempted)
+        return;
+    loadAttempted = true;
+    try {
+        const natural = await import('natural');
+        stemFn = (word) => natural.PorterStemmer.stem(word);
+    }
+    catch {
+        /* natural not installed — stemmer will be a no-op */
+        stemFn = null;
+    }
+}
+/**
+ * Porter stemmer — reduces words to their morphological root.
+ * `running` → `run`, `foxes` → `fox`, `connected` → `connect`.
+ *
+ * Uses the `natural` npm package (already in agentos dependencies).
+ * Falls back to no-op if `natural` can't be imported.
+ *
+ * Sets `token.stem` with the stemmed form. Also updates `token.text`
+ * so downstream processors work with stemmed tokens.
+ */
+export class PorterStemmer {
+    constructor() {
+        this.name = 'PorterStemmer';
+        this.initialized = false;
+    }
+    async ensureLoaded() {
+        if (!this.initialized) {
+            await loadStemmer();
+            this.initialized = true;
+        }
+    }
+    process(tokens) {
+        /* Synchronous path if already loaded */
+        if (!stemFn)
+            return tokens;
+        return tokens.map(t => {
+            const stemmed = stemFn(t.text);
+            return { ...t, text: stemmed, stem: stemmed };
+        });
+    }
+    /**
+     * Async initialization — call once before first use to load `natural`.
+     * The pipeline calls this automatically, but you can call it early
+     * to avoid the lazy-load delay on first process() call.
+     */
+    async initialize() {
+        await this.ensureLoaded();
+    }
+}
+//# sourceMappingURL=PorterStemmer.js.map

package/dist/core/text-processing/stemmers/PorterStemmer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PorterStemmer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/stemmers/PorterStemmer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,4DAA4D;AAC5D,IAAI,MAAM,GAAsC,IAAI,CAAC;AACrD,IAAI,aAAa,GAAG,KAAK,CAAC;AAE1B,KAAK,UAAU,WAAW;IACxB,IAAI,aAAa;QAAE,OAAO;IAC1B,aAAa,GAAG,IAAI,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,qDAAqD;QACrD,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,aAAa;IAA1B;QACW,SAAI,GAAG,eAAe,CAAC;QAExB,gBAAW,GAAG,KAAK,CAAC;IA2B9B,CAAC;IAzBS,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAe;QACrB,wCAAwC;QACxC,IAAI,CAAC,MAAM;YAAE,OAAO,MAAM,CAAC;QAE3B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACpB,MAAM,OAAO,GAAG,MAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAChC,OAAO,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC5B,CAAC;CACF"}

package/dist/core/text-processing/tokenizers/CodeTokenizer.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @fileoverview Code-aware tokenizer that splits camelCase, snake_case,
+ * SCREAMING_SNAKE, and dot-separated identifiers into individual words.
+ *
+ * @module agentos/core/text-processing/tokenizers/CodeTokenizer
+ */
+import type { Token } from '../types';
+import type { ITokenizer } from '../ITextProcessor';
+/**
+ * Code-aware tokenizer.
+ *
+ * Splits identifiers that programmers write:
+ * - `getUserName` → `get`, `user`, `name`
+ * - `get_user_name` → `get`, `user`, `name`
+ * - `MAX_RETRY_COUNT` → `max`, `retry`, `count`
+ * - `XMLParser` → `xml`, `parser`
+ * - `path.to.module` → `path`, `to`, `module`
+ */
+export declare class CodeTokenizer implements ITokenizer {
+    readonly name = "CodeTokenizer";
+    private minLength;
+    constructor(minLength?: number);
+    tokenize(text: string): Token[];
+}
+//# sourceMappingURL=CodeTokenizer.d.ts.map

package/dist/core/text-processing/tokenizers/CodeTokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CodeTokenizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/tokenizers/CodeTokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAWpD;;;;;;;;;GASG;AACH,qBAAa,aAAc,YAAW,UAAU;IAC9C,QAAQ,CAAC,IAAI,mBAAmB;IAEhC,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAU;IAIjC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE;CAgDhC"}

package/dist/core/text-processing/tokenizers/CodeTokenizer.js

@@ -0,0 +1,75 @@
+/**
+ * @fileoverview Code-aware tokenizer that splits camelCase, snake_case,
+ * SCREAMING_SNAKE, and dot-separated identifiers into individual words.
+ *
+ * @module agentos/core/text-processing/tokenizers/CodeTokenizer
+ */
+/** Matches word-like sequences including dots for qualified names. */
+const CODE_WORD_REGEX = /[\p{L}\p{N}_.]+/gu;
+/** Split camelCase: insert boundary before uppercase letters that follow lowercase. */
+const CAMEL_SPLIT = /([a-z\d])([A-Z])/g;
+/** Split ALLCAPS followed by lowercase: XMLParser → XML + Parser. */
+const CAPS_SPLIT = /([A-Z]+)([A-Z][a-z])/g;
+/**
+ * Code-aware tokenizer.
+ *
+ * Splits identifiers that programmers write:
+ * - `getUserName` → `get`, `user`, `name`
+ * - `get_user_name` → `get`, `user`, `name`
+ * - `MAX_RETRY_COUNT` → `max`, `retry`, `count`
+ * - `XMLParser` → `xml`, `parser`
+ * - `path.to.module` → `path`, `to`, `module`
+ */
+export class CodeTokenizer {
+    constructor(minLength = 2) {
+        this.name = 'CodeTokenizer';
+        this.minLength = minLength;
+    }
+    tokenize(text) {
+        const tokens = [];
+        let match;
+        CODE_WORD_REGEX.lastIndex = 0;
+        while ((match = CODE_WORD_REGEX.exec(text)) !== null) {
+            const word = match[0];
+            const position = match.index;
+            /* Split on dots first (path.to.module) */
+            const dotParts = word.split('.');
+            let offset = 0;
+            for (const part of dotParts) {
+                if (part.length === 0) {
+                    offset += 1;
+                    continue;
+                }
+                /* Split on underscores (snake_case) */
+                const underscoreParts = part.split('_');
+                let subOffset = 0;
+                for (const sub of underscoreParts) {
+                    if (sub.length === 0) {
+                        subOffset += 1;
+                        continue;
+                    }
+                    /* Split camelCase */
+                    const camelSplit = sub
+                        .replace(CAMEL_SPLIT, '$1\0$2')
+                        .replace(CAPS_SPLIT, '$1\0$2')
+                        .split('\0');
+                    let camelOffset = 0;
+                    for (const fragment of camelSplit) {
+                        if (fragment.length >= this.minLength) {
+                            tokens.push({
+                                text: fragment,
+                                original: word,
+                                position: position + offset + subOffset + camelOffset,
+                            });
+                        }
+                        camelOffset += fragment.length;
+                    }
+                    subOffset += sub.length + 1; /* +1 for the underscore */
+                }
+                offset += part.length + 1; /* +1 for the dot */
+            }
+        }
+        return tokens;
+    }
+}
+//# sourceMappingURL=CodeTokenizer.js.map

package/dist/core/text-processing/tokenizers/CodeTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CodeTokenizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/tokenizers/CodeTokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,sEAAsE;AACtE,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAE5C,uFAAuF;AACvF,MAAM,WAAW,GAAG,mBAAmB,CAAC;AAExC,qEAAqE;AACrE,MAAM,UAAU,GAAG,uBAAuB,CAAC;AAE3C;;;;;;;;;GASG;AACH,MAAM,OAAO,aAAa;IAKxB,YAAY,YAAoB,CAAC;QAJxB,SAAI,GAAG,eAAe,CAAC;QAK9B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,KAA6B,CAAC;QAElC,eAAe,CAAC,SAAS,GAAG,CAAC,CAAC;QAC9B,OAAO,CAAC,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;YAE7B,0CAA0C;YAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAEjC,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAAC,MAAM,IAAI,CAAC,CAAC;oBAAC,SAAS;gBAAC,CAAC;gBAEjD,uCAAuC;gBACvC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAExC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;oBAClC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAAC,SAAS,IAAI,CAAC,CAAC;wBAAC,SAAS;oBAAC,CAAC;oBAEnD,qBAAqB;oBACrB,MAAM,UAAU,GAAG,GAAG;yBACnB,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC;yBAC9B,OAAO,CAAC,UAAU,EAAE,QAAQ,CAAC;yBAC7B,KAAK,CAAC,IAAI,CAAC,CAAC;oBAEf,IAAI,WAAW,GAAG,CAAC,CAAC;oBACpB,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;wBAClC,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;4BACtC,MAAM,CAAC,IAAI,CAAC;gCACV,IAAI,EAAE,QAAQ;gCACd,QAAQ,EAAE,IAAI;gCACd,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,WAAW;6BACtD,CAAC,CAAC;wBACL,CAAC;wBACD,WAAW,IAAI,QAAQ,CAAC,MAAM,CAAC;oBACjC,CAAC;oBACD,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,2BAA2B;gBAC1D,CAAC;gBACD,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,oBAAoB;YACjD,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/core/text-processing/tokenizers/StandardTokenizer.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @fileoverview Unicode-aware word tokenizer.
+ * Splits on whitespace, punctuation, and special characters.
+ * Produces tokens with position tracking.
+ *
+ * @module agentos/core/text-processing/tokenizers/StandardTokenizer
+ */
+import type { Token } from '../types';
+import type { ITokenizer } from '../ITextProcessor';
+/**
+ * Standard tokenizer that splits text on Unicode word boundaries.
+ * Handles punctuation, whitespace, hyphens, and special characters.
+ * Preserves position offsets for each token.
+ */
+export declare class StandardTokenizer implements ITokenizer {
+    readonly name = "StandardTokenizer";
+    /** Minimum token length to emit (default 2). */
+    private minLength;
+    constructor(minLength?: number);
+    tokenize(text: string): Token[];
+}
+//# sourceMappingURL=StandardTokenizer.d.ts.map

package/dist/core/text-processing/tokenizers/StandardTokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StandardTokenizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/tokenizers/StandardTokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKpD;;;;GAIG;AACH,qBAAa,iBAAkB,YAAW,UAAU;IAClD,QAAQ,CAAC,IAAI,uBAAuB;IAEpC,gDAAgD;IAChD,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAU;IAIjC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE;CAkBhC"}

package/dist/core/text-processing/tokenizers/StandardTokenizer.js

@@ -0,0 +1,37 @@
+/**
+ * @fileoverview Unicode-aware word tokenizer.
+ * Splits on whitespace, punctuation, and special characters.
+ * Produces tokens with position tracking.
+ *
+ * @module agentos/core/text-processing/tokenizers/StandardTokenizer
+ */
+/** Matches word-like sequences: letters, digits, underscores. */
+const WORD_REGEX = /[\p{L}\p{N}_]+/gu;
+/**
+ * Standard tokenizer that splits text on Unicode word boundaries.
+ * Handles punctuation, whitespace, hyphens, and special characters.
+ * Preserves position offsets for each token.
+ */
+export class StandardTokenizer {
+    constructor(minLength = 2) {
+        this.name = 'StandardTokenizer';
+        this.minLength = minLength;
+    }
+    tokenize(text) {
+        const tokens = [];
+        let match;
+        WORD_REGEX.lastIndex = 0;
+        while ((match = WORD_REGEX.exec(text)) !== null) {
+            const word = match[0];
+            if (word.length >= this.minLength) {
+                tokens.push({
+                    text: word,
+                    original: word,
+                    position: match.index,
+                });
+            }
+        }
+        return tokens;
+    }
+}
+//# sourceMappingURL=StandardTokenizer.js.map

package/dist/core/text-processing/tokenizers/StandardTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StandardTokenizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/tokenizers/StandardTokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,iEAAiE;AACjE,MAAM,UAAU,GAAG,kBAAkB,CAAC;AAEtC;;;;GAIG;AACH,MAAM,OAAO,iBAAiB;IAM5B,YAAY,YAAoB,CAAC;QALxB,SAAI,GAAG,mBAAmB,CAAC;QAMlC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,KAA6B,CAAC;QAElC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC;QACzB,OAAO,CAAC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,IAAI;oBACd,QAAQ,EAAE,KAAK,CAAC,KAAK;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/core/text-processing/types.d.ts

@@ -0,0 +1,18 @@
+/**
+ * @fileoverview Core types for the text processing pipeline.
+ * @module agentos/core/text-processing/types
+ */
+/** A single processed token with position and optional linguistic annotations. */
+export interface Token {
+    /** The processed token text (after normalization, stemming, etc.). */
+    text: string;
+    /** The original text before any processing. */
+    original: string;
+    /** Character offset in the source text. */
+    position: number;
+    /** Stemmed form (set by stemmer processors). */
+    stem?: string;
+    /** Lemmatized form (set by lemmatizer processors). */
+    lemma?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/core/text-processing/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,kFAAkF;AAClF,MAAM,WAAW,KAAK;IACpB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}

package/dist/core/text-processing/types.js

@@ -0,0 +1,6 @@
+/**
+ * @fileoverview Core types for the text processing pipeline.
+ * @module agentos/core/text-processing/types
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/text-processing/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/text-processing/types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/dist/core/vector-search/HnswIndexSidecar.d.ts

@@ -0,0 +1,91 @@
+/**
+ * @fileoverview Shared HNSW index sidecar that sits alongside a SQLite database.
+ * Provides O(log n) approximate nearest neighbor search via hnswlib-node,
+ * with automatic activation at a configurable document count threshold
+ * and graceful fallback when the native addon is unavailable.
+ *
+ * Used by both the RAG system (SqlVectorStore) and the Memory system (SqliteBrain).
+ *
+ * @module agentos/core/vector-search/HnswIndexSidecar
+ */
+import type { HnswSidecarConfig, HnswSidecarStats, HnswSearchResult } from './types';
+/**
+ * HNSW index sidecar — manages an hnswlib-node index file alongside
+ * a primary data store (SQLite, etc.).
+ *
+ * The primary store remains the source of truth. The HNSW index is
+ * rebuildable from it at any time. This sidecar handles:
+ * - Dynamic import of hnswlib-node (graceful if missing)
+ * - Auto-activation at document count threshold
+ * - Auto-resize when capacity is reached
+ * - Persistence to disk (.hnsw + .hnsw.map.json)
+ * - Dirty tracking with explicit save()
+ */
+export declare class HnswIndexSidecar {
+    private config;
+    private index;
+    private hnswlib;
+    /** Maps HNSW integer labels → document string IDs. */
+    private labelToId;
+    /** Maps document string IDs → HNSW integer labels. */
+    private idToLabel;
+    /** Next label to assign. */
+    private nextLabel;
+    /** Current index capacity. */
+    private capacity;
+    /** Whether index has unsaved changes. */
+    private dirty;
+    /** Whether hnswlib-node was successfully imported. */
+    private hnswAvailable;
+    /**
+     * Initialize the sidecar. Attempts to dynamically import hnswlib-node.
+     * If the import fails, the sidecar stays inactive (brute-force fallback).
+     * If an existing index file is found, it's loaded from disk.
+     */
+    initialize(config: HnswSidecarConfig): Promise<void>;
+    /** Persist and release the index. */
+    shutdown(): Promise<void>;
+    /** True when the index is loaded AND has vectors (above threshold or loaded from disk). */
+    isActive(): boolean;
+    /** True when hnswlib-node was successfully imported. */
+    isAvailable(): boolean;
+    /** Get statistics about the sidecar. */
+    getStats(): HnswSidecarStats;
+    /**
+     * Add a single vector. Auto-resizes if capacity is reached.
+     * Does nothing if hnswlib is unavailable or the item is already indexed.
+     */
+    add(id: string, embedding: number[]): Promise<void>;
+    /**
+     * Add multiple vectors at once. More efficient than calling add() in a loop.
+     */
+    addBatch(items: Array<{
+        id: string;
+        embedding: number[];
+    }>): Promise<void>;
+    /**
+     * Soft-delete a vector by marking its label as deleted in the HNSW graph.
+     */
+    remove(id: string): Promise<void>;
+    /**
+     * Search for the top-K nearest neighbors.
+     * Returns empty if the sidecar is inactive.
+     */
+    search(query: number[], topK: number): Promise<HnswSearchResult[]>;
+    /**
+     * Full rebuild of the HNSW index from source-of-truth data.
+     * Called when the activation threshold is crossed or on manual rebuild.
+     */
+    rebuildFromData(items: Array<{
+        id: string;
+        embedding: number[];
+    }>): Promise<void>;
+    /** Save the HNSW index and label map to disk. */
+    save(): Promise<void>;
+    /**
+     * Load an existing HNSW index from disk.
+     * Returns true if loaded successfully, false if no index exists.
+     */
+    load(): Promise<boolean>;
+}
+//# sourceMappingURL=HnswIndexSidecar.d.ts.map

package/dist/core/vector-search/HnswIndexSidecar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HnswIndexSidecar.d.ts","sourceRoot":"","sources":["../../../src/core/vector-search/HnswIndexSidecar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EACV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAkBjB;;;;;;;;;;;GAWG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAa;IAE5B,sDAAsD;IACtD,OAAO,CAAC,SAAS,CAAkC;IACnD,sDAAsD;IACtD,OAAO,CAAC,SAAS,CAAkC;IACnD,4BAA4B;IAC5B,OAAO,CAAC,SAAS,CAAa;IAC9B,8BAA8B;IAC9B,OAAO,CAAC,QAAQ,CAAa;IAC7B,yCAAyC;IACzC,OAAO,CAAC,KAAK,CAAkB;IAC/B,sDAAsD;IACtD,OAAO,CAAC,aAAa,CAAkB;IAIvC;;;;OAIG;IACG,UAAU,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB1D,qCAAqC;IAC/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAW/B,2FAA2F;IAC3F,QAAQ,IAAI,OAAO;IAInB,wDAAwD;IACxD,WAAW,IAAI,OAAO;IAItB,wCAAwC;IACxC,QAAQ,IAAI,gBAAgB;IAY5B;;;OAGG;IACG,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBzD;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhF;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWvC;;;OAGG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAkCxE;;;OAGG;IACG,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BvF,iDAAiD;IAC3C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwB3B;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC;CAkC/B"}