@yamo/memory-mesh 2.3.2 → 3.0.1

package/lib/scrubber/scrubber.ts

@@ -0,0 +1,109 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Main Orchestrator
+ * @module smora/scrubber/scrubber
+ */
+import { StructuralCleaner } from "./stages/structural-cleaner.js";
+import { SemanticFilter } from "./stages/semantic-filter.js";
+import { Normalizer } from "./stages/normalizer.js";
+import { Chunker } from "./stages/chunker.js";
+import { MetadataAnnotator } from "./stages/metadata-annotator.js";
+import { Validator } from "./stages/validator.js";
+import { ScrubberTelemetry, } from "./telemetry.js";
+// import { ScrubberError } from './errors/scrubber-error'; // Assuming this exists or I should check
+import { defaultScrubberConfig } from "./config/defaults.js";
+export class Scrubber {
+    config;
+    stages; // Using any for stages as they are not yet converted
+    telemetry;
+    constructor(config = {}) {
+        this.config = { ...defaultScrubberConfig, ...config };
+        this.stages = this._initializeStages();
+        this.telemetry = new ScrubberTelemetry();
+    }
+    /**
+     * Main entry point - process a raw document
+     * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+     * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+     */
+    async process(document) {
+        const startTime = Date.now();
+        const result = {
+            chunks: [],
+            metadata: {
+                source: document.source,
+                type: document.type,
+                processingTimestamp: new Date().toISOString(),
+            },
+            telemetry: {},
+        };
+        try {
+            // If disabled, return empty chunks
+            if (!this.config.enabled) {
+                result.success = true;
+                result.telemetry.totalDuration = Date.now() - startTime;
+                return result;
+            }
+            // Stage 1: Structural Cleaning
+            const cleaned = await this._executeStage("structural", () => this.stages.structural.clean(document.content));
+            result.telemetry.structural = this.telemetry.getStageStats("structural");
+            // Stage 2: Semantic Filtering
+            const filtered = await this._executeStage("semantic", () => this.stages.semantic.filter(cleaned));
+            result.telemetry.semantic = this.telemetry.getStageStats("semantic");
+            // Stage 3: Normalization
+            const normalized = await this._executeStage("normalization", () => this.stages.normalizer.normalize(filtered));
+            result.telemetry.normalization =
+                this.telemetry.getStageStats("normalization");
+            // Stage 4: Chunking
+            const chunks = await this._executeStage("chunking", () => this.stages.chunker.chunk(normalized));
+            result.telemetry.chunking = this.telemetry.getStageStats("chunking");
+            // Stage 5: Metadata Annotation
+            const annotated = await this._executeStage("metadata", () => this.stages.metadata.annotate(chunks, document));
+            result.telemetry.metadata = this.telemetry.getStageStats("metadata");
+            // Stage 6: Validation
+            result.chunks = await this._executeStage("validation", () => this.stages.validator.validate(annotated));
+            result.telemetry.validation = this.telemetry.getStageStats("validation");
+            result.telemetry.totalDuration = Date.now() - startTime;
+            result.success = true;
+            return result;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            result.success = false;
+            result.error = message;
+            result.telemetry.totalDuration = Date.now() - startTime;
+            return result;
+        }
+    }
+    async _executeStage(stageName, stageFn) {
+        const startTime = Date.now();
+        try {
+            const result = await stageFn();
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, true);
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, false);
+            throw error;
+        }
+    }
+    _initializeStages() {
+        return {
+            structural: new StructuralCleaner(this.config.structural),
+            semantic: new SemanticFilter(this.config.semantic),
+            normalizer: new Normalizer(this.config.normalization),
+            chunker: new Chunker(this.config.chunking),
+            metadata: new MetadataAnnotator(this.config.metadata),
+            validator: new Validator(this.config.validation),
+        };
+    }
+    getMetrics() {
+        return this.telemetry.getSummary();
+    }
+    healthCheck() {
+        return Promise.resolve({ status: "healthy" });
+    }
+}
+export default Scrubber;

package/lib/scrubber/stages/chunker.d.ts

@@ -0,0 +1,25 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 4: Chunking
+ * @module smora/scrubber/stages/chunker
+ */
+export declare class Chunker {
+    constructor(config: any);
+    /**
+     * Split content into chunks
+     * @param {string} content - Normalized content
+     * @returns {Promise<Array>} - Array of chunks with metadata
+     */
+    chunk(content: any): Promise<{
+        index: number;
+        text: any;
+        metadata: {
+            tokens: any;
+            heading: any;
+            position: number;
+        };
+    }[]>;
+    _isHeading(line: any): boolean;
+    _shouldStartNewChunk(currentChunk: any, para: any, paraTokens: any, isHeading: any): boolean;
+    _extractInitialHeading(content: any): any;
+    _extractHeadingText(headingLine: any): any;
+}

package/lib/scrubber/stages/chunker.js

@@ -1,103 +1,86 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Stage 4: Chunking
  * @module smora/scrubber/stages/chunker
  */
-
 import { TokenCounter } from '../utils/token-counter.js';
-import { ChunkingError, ScrubberError } from '../errors/scrubber-error.js';
-
+import { ScrubberError } from '../errors/scrubber-error.js';
 export class Chunker {
-  constructor(config) {
-    this.config = config;
-    this.tokenCounter = new TokenCounter();
-  }
-
-  /**
-   * Split content into chunks
-   * @param {string} content - Normalized content
-   * @returns {Promise<Array>} - Array of chunks with metadata
-   */
-  async chunk(content) {
-    try {
-      const chunks = [];
-      const paragraphs = content.split(/\n\n+/);
-
-      let currentChunk = {
-        text: '',
-        tokens: 0,
-        heading: this._extractInitialHeading(content)
-      };
-
-      for (const para of paragraphs) {
-        const isHeading = this._isHeading(para);
-        const paraTokens = this.tokenCounter.count(para);
-
-        if (this._shouldStartNewChunk(currentChunk, para, paraTokens, isHeading)) {
-          if (currentChunk.tokens >= this.config.minTokens) {
-            chunks.push({ ...currentChunk });
-          }
-          currentChunk = {
-            text: '',
-            tokens: 0,
-            heading: isHeading ? this._extractHeadingText(para) : currentChunk.heading
-          };
+    constructor(config) {
+        this.config = config;
+        this.tokenCounter = new TokenCounter();
+    }
+    /**
+     * Split content into chunks
+     * @param {string} content - Normalized content
+     * @returns {Promise<Array>} - Array of chunks with metadata
+     */
+    async chunk(content) {
+        try {
+            const chunks = [];
+            const paragraphs = content.split(/\n\n+/);
+            let currentChunk = {
+                text: '',
+                tokens: 0,
+                heading: this._extractInitialHeading(content)
+            };
+            for (const para of paragraphs) {
+                const isHeading = this._isHeading(para);
+                const paraTokens = this.tokenCounter.count(para);
+                if (this._shouldStartNewChunk(currentChunk, para, paraTokens, isHeading)) {
+                    if (currentChunk.tokens >= this.config.minTokens) {
+                        chunks.push({ ...currentChunk });
+                    }
+                    currentChunk = {
+                        text: '',
+                        tokens: 0,
+                        heading: isHeading ? this._extractHeadingText(para) : currentChunk.heading
+                    };
+                }
+                currentChunk.text += (currentChunk.text ? '\n\n' : '') + para;
+                currentChunk.tokens += paraTokens;
+                if (currentChunk.tokens > this.config.hardMaxTokens) {
+                    chunks.push({ ...currentChunk });
+                    currentChunk = { text: '', tokens: 0, heading: null };
+                }
+            }
+            if (currentChunk.tokens >= this.config.minTokens) {
+                chunks.push(currentChunk);
+            }
+            return chunks.map((chunk, index) => ({
+                index,
+                text: chunk.text.trim(),
+                metadata: {
+                    tokens: chunk.tokens,
+                    heading: chunk.heading,
+                    position: index
+                }
+            }));
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new ScrubberError(`Failed to chunk content: ${message}`, { stage: 'chunker', originalError: error });
         }
-
-        currentChunk.text += (currentChunk.text ? '\n\n' : '') + para;
-        currentChunk.tokens += paraTokens;
-
-        if (currentChunk.tokens > this.config.hardMaxTokens) {
-          chunks.push({ ...currentChunk });
-          currentChunk = { text: '', tokens: 0, heading: null };
+    }
+    _isHeading(line) {
+        return /^#{1,6}\s/.test(line);
+    }
+    _shouldStartNewChunk(currentChunk, para, paraTokens, isHeading) {
+        if (this.config.splitOnHeadings && isHeading && currentChunk.tokens > 0) {
+            return true;
         }
-      }
-
-      if (currentChunk.tokens >= this.config.minTokens) {
-        chunks.push(currentChunk);
-      }
-
-      return chunks.map((chunk, index) => ({
-        index,
-        text: chunk.text.trim(),
-        metadata: {
-          tokens: chunk.tokens,
-          heading: chunk.heading,
-          position: index
+        const wouldExceed = (currentChunk.tokens + paraTokens) > this.config.maxTokens;
+        if (wouldExceed && currentChunk.tokens > 0) {
+            return true;
         }
-      }));
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      throw new ScrubberError(
-        `Failed to chunk content: ${message}`,
-        { stage: 'chunker', originalError: error }
-      );
+        return false;
     }
-  }
-
-  _isHeading(line) {
-    return /^#{1,6}\s/.test(line);
-  }
-
-  _shouldStartNewChunk(currentChunk, para, paraTokens, isHeading) {
-    if (this.config.splitOnHeadings && isHeading && currentChunk.tokens > 0) {
-      return true;
+    _extractInitialHeading(content) {
+        const match = content.match(/^#{1,6}\s+(.+)$/m);
+        return match ? match[1] : null;
     }
-
-    const wouldExceed = (currentChunk.tokens + paraTokens) > this.config.maxTokens;
-    if (wouldExceed && currentChunk.tokens > 0) {
-      return true;
+    _extractHeadingText(headingLine) {
+        const match = headingLine.match(/^#{1,6}\s+(.+)$/);
+        return match ? match[1] : null;
     }
-
-    return false;
-  }
-
-  _extractInitialHeading(content) {
-    const match = content.match(/^#{1,6}\s+(.+)$/m);
-    return match ? match[1] : null;
-  }
-
-  _extractHeadingText(headingLine) {
-    const match = headingLine.match(/^#{1,6}\s+(.+)$/);
-    return match ? match[1] : null;
-  }
-}
+}

package/lib/scrubber/stages/chunker.ts

@@ -0,0 +1,104 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 4: Chunking
+ * @module smora/scrubber/stages/chunker
+ */
+
+import { TokenCounter } from '../utils/token-counter.js';
+import { ChunkingError, ScrubberError } from '../errors/scrubber-error.js';
+
+export class Chunker {
+  constructor(config) {
+    this.config = config;
+    this.tokenCounter = new TokenCounter();
+  }
+
+  /**
+   * Split content into chunks
+   * @param {string} content - Normalized content
+   * @returns {Promise<Array>} - Array of chunks with metadata
+   */
+  async chunk(content) {
+    try {
+      const chunks = [];
+      const paragraphs = content.split(/\n\n+/);
+
+      let currentChunk = {
+        text: '',
+        tokens: 0,
+        heading: this._extractInitialHeading(content)
+      };
+
+      for (const para of paragraphs) {
+        const isHeading = this._isHeading(para);
+        const paraTokens = this.tokenCounter.count(para);
+
+        if (this._shouldStartNewChunk(currentChunk, para, paraTokens, isHeading)) {
+          if (currentChunk.tokens >= this.config.minTokens) {
+            chunks.push({ ...currentChunk });
+          }
+          currentChunk = {
+            text: '',
+            tokens: 0,
+            heading: isHeading ? this._extractHeadingText(para) : currentChunk.heading
+          };
+        }
+
+        currentChunk.text += (currentChunk.text ? '\n\n' : '') + para;
+        currentChunk.tokens += paraTokens;
+
+        if (currentChunk.tokens > this.config.hardMaxTokens) {
+          chunks.push({ ...currentChunk });
+          currentChunk = { text: '', tokens: 0, heading: null };
+        }
+      }
+
+      if (currentChunk.tokens >= this.config.minTokens) {
+        chunks.push(currentChunk);
+      }
+
+      return chunks.map((chunk, index) => ({
+        index,
+        text: chunk.text.trim(),
+        metadata: {
+          tokens: chunk.tokens,
+          heading: chunk.heading,
+          position: index
+        }
+      }));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ScrubberError(
+        `Failed to chunk content: ${message}`,
+        { stage: 'chunker', originalError: error }
+      );
+    }
+  }
+
+  _isHeading(line) {
+    return /^#{1,6}\s/.test(line);
+  }
+
+  _shouldStartNewChunk(currentChunk, para, paraTokens, isHeading) {
+    if (this.config.splitOnHeadings && isHeading && currentChunk.tokens > 0) {
+      return true;
+    }
+
+    const wouldExceed = (currentChunk.tokens + paraTokens) > this.config.maxTokens;
+    if (wouldExceed && currentChunk.tokens > 0) {
+      return true;
+    }
+
+    return false;
+  }
+
+  _extractInitialHeading(content) {
+    const match = content.match(/^#{1,6}\s+(.+)$/m);
+    return match ? match[1] : null;
+  }
+
+  _extractHeadingText(headingLine) {
+    const match = headingLine.match(/^#{1,6}\s+(.+)$/);
+    return match ? match[1] : null;
+  }
+}

package/lib/scrubber/stages/metadata-annotator.d.ts

@@ -0,0 +1,17 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 5: Metadata Annotation
+ * @module smora/scrubber/stages/metadata-annotator
+ */
+export declare class MetadataAnnotator {
+    constructor(config: any);
+    /**
+     * Add metadata to chunks
+     * @param {Array} chunks - Array of chunks
+     * @param {Object} document - Original document metadata
+     * @returns {Promise<Array>} - Annotated chunks
+     */
+    annotate(chunks: any, document: any): Promise<any>;
+    _extractSection(chunk: any): any;
+    _buildHeadingPath(chunk: any, currentPath: any): any[];
+    _isSubHeading(heading1: any, heading2: any): boolean;
+}

package/lib/scrubber/stages/metadata-annotator.js

@@ -1,74 +1,64 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Stage 5: Metadata Annotation
  * @module smora/scrubber/stages/metadata-annotator
  */
-
 import { HashUtil } from '../utils/hash.js';
-
 export class MetadataAnnotator {
-  constructor(config) {
-    this.config = config;
-    this.hashUtil = new HashUtil();
-  }
-
-  /**
-   * Add metadata to chunks
-   * @param {Array} chunks - Array of chunks
-   * @param {Object} document - Original document metadata
-   * @returns {Promise<Array>} - Annotated chunks
-   */
-  async annotate(chunks, document) {
-    const headingPath = [];
-
-    return chunks.map((chunk, index) => {
-      const metadata = {
-        ...chunk.metadata,
-        source: this.config.addSource ? document.source : undefined,
-        doc_type: this.config.addSource ? document.type : undefined,
-        section: this.config.addSection ? this._extractSection(chunk) : undefined,
-        heading_path: this.config.addHeadingPath ?
-          this._buildHeadingPath(chunk, headingPath) :
-          undefined,
-        ingestion_timestamp: this.config.addTimestamp ?
-          new Date().toISOString() :
-          undefined,
-        hash: this.config.addHash ?
-          this.hashUtil.hash(chunk.text) :
-          undefined
-      };
-
-      return {
-        ...chunk,
-        metadata: Object.fromEntries(
-          Object.entries(metadata).filter(([_, v]) => v !== undefined)
-        )
-      };
-    });
-  }
-
-  _extractSection(chunk) {
-    if (chunk.metadata.heading) {
-      return chunk.metadata.heading;
+    constructor(config) {
+        this.config = config;
+        this.hashUtil = new HashUtil();
     }
-    return 'unnamed-section';
-  }
-
-  _buildHeadingPath(chunk, currentPath) {
-    const heading = chunk.metadata.heading;
-
-    if (heading && heading !== currentPath[currentPath.length - 1]) {
-      if (currentPath.length === 0 || this._isSubHeading(heading, currentPath[currentPath.length - 1])) {
-        currentPath.push(heading);
-      } else {
-        currentPath.length = 0;
-        currentPath.push(heading);
-      }
+    /**
+     * Add metadata to chunks
+     * @param {Array} chunks - Array of chunks
+     * @param {Object} document - Original document metadata
+     * @returns {Promise<Array>} - Annotated chunks
+     */
+    async annotate(chunks, document) {
+        const headingPath = [];
+        return chunks.map((chunk, index) => {
+            const metadata = {
+                ...chunk.metadata,
+                source: this.config.addSource ? document.source : undefined,
+                doc_type: this.config.addSource ? document.type : undefined,
+                section: this.config.addSection ? this._extractSection(chunk) : undefined,
+                heading_path: this.config.addHeadingPath ?
+                    this._buildHeadingPath(chunk, headingPath) :
+                    undefined,
+                ingestion_timestamp: this.config.addTimestamp ?
+                    new Date().toISOString() :
+                    undefined,
+                hash: this.config.addHash ?
+                    this.hashUtil.hash(chunk.text) :
+                    undefined
+            };
+            return {
+                ...chunk,
+                metadata: Object.fromEntries(Object.entries(metadata).filter(([_, v]) => v !== undefined))
+            };
+        });
+    }
+    _extractSection(chunk) {
+        if (chunk.metadata.heading) {
+            return chunk.metadata.heading;
+        }
+        return 'unnamed-section';
+    }
+    _buildHeadingPath(chunk, currentPath) {
+        const heading = chunk.metadata.heading;
+        if (heading && heading !== currentPath[currentPath.length - 1]) {
+            if (currentPath.length === 0 || this._isSubHeading(heading, currentPath[currentPath.length - 1])) {
+                currentPath.push(heading);
+            }
+            else {
+                currentPath.length = 0;
+                currentPath.push(heading);
+            }
+        }
+        return [...currentPath];
+    }
+    _isSubHeading(heading1, heading2) {
+        return heading1.length > heading2.length;
     }
-
-    return [...currentPath];
-  }
-
-  _isSubHeading(heading1, heading2) {
-    return heading1.length > heading2.length;
-  }
 }

package/lib/scrubber/stages/metadata-annotator.ts

@@ -0,0 +1,75 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 5: Metadata Annotation
+ * @module smora/scrubber/stages/metadata-annotator
+ */
+
+import { HashUtil } from '../utils/hash.js';
+
+export class MetadataAnnotator {
+  constructor(config) {
+    this.config = config;
+    this.hashUtil = new HashUtil();
+  }
+
+  /**
+   * Add metadata to chunks
+   * @param {Array} chunks - Array of chunks
+   * @param {Object} document - Original document metadata
+   * @returns {Promise<Array>} - Annotated chunks
+   */
+  async annotate(chunks, document) {
+    const headingPath = [];
+
+    return chunks.map((chunk, index) => {
+      const metadata = {
+        ...chunk.metadata,
+        source: this.config.addSource ? document.source : undefined,
+        doc_type: this.config.addSource ? document.type : undefined,
+        section: this.config.addSection ? this._extractSection(chunk) : undefined,
+        heading_path: this.config.addHeadingPath ?
+          this._buildHeadingPath(chunk, headingPath) :
+          undefined,
+        ingestion_timestamp: this.config.addTimestamp ?
+          new Date().toISOString() :
+          undefined,
+        hash: this.config.addHash ?
+          this.hashUtil.hash(chunk.text) :
+          undefined
+      };
+
+      return {
+        ...chunk,
+        metadata: Object.fromEntries(
+          Object.entries(metadata).filter(([_, v]) => v !== undefined)
+        )
+      };
+    });
+  }
+
+  _extractSection(chunk) {
+    if (chunk.metadata.heading) {
+      return chunk.metadata.heading;
+    }
+    return 'unnamed-section';
+  }
+
+  _buildHeadingPath(chunk, currentPath) {
+    const heading = chunk.metadata.heading;
+
+    if (heading && heading !== currentPath[currentPath.length - 1]) {
+      if (currentPath.length === 0 || this._isSubHeading(heading, currentPath[currentPath.length - 1])) {
+        currentPath.push(heading);
+      } else {
+        currentPath.length = 0;
+        currentPath.push(heading);
+      }
+    }
+
+    return [...currentPath];
+  }
+
+  _isSubHeading(heading1, heading2) {
+    return heading1.length > heading2.length;
+  }
+}

package/lib/scrubber/stages/normalizer.d.ts

@@ -0,0 +1,16 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 3: Normalization
+ * @module smora/scrubber/stages/normalizer
+ */
+export declare class Normalizer {
+    constructor(config: any);
+    /**
+     * Normalize content structure
+     * @param {string} content - Filtered content
+     * @returns {Promise<string>} - Normalized content
+     */
+    normalize(content: any): Promise<any>;
+    _normalizeHeadings(content: any): any;
+    _normalizeLists(content: any): any;
+    _normalizePunctuation(content: any): any;
+}