@yamo/memory-mesh 3.0.0 → 3.0.1

package/lib/scrubber/stages/metadata-annotator.ts

@@ -0,0 +1,75 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 5: Metadata Annotation
+ * @module smora/scrubber/stages/metadata-annotator
+ */
+
+import { HashUtil } from '../utils/hash.js';
+
+export class MetadataAnnotator {
+  constructor(config) {
+    this.config = config;
+    this.hashUtil = new HashUtil();
+  }
+
+  /**
+   * Add metadata to chunks
+   * @param {Array} chunks - Array of chunks
+   * @param {Object} document - Original document metadata
+   * @returns {Promise<Array>} - Annotated chunks
+   */
+  async annotate(chunks, document) {
+    const headingPath = [];
+
+    return chunks.map((chunk, index) => {
+      const metadata = {
+        ...chunk.metadata,
+        source: this.config.addSource ? document.source : undefined,
+        doc_type: this.config.addSource ? document.type : undefined,
+        section: this.config.addSection ? this._extractSection(chunk) : undefined,
+        heading_path: this.config.addHeadingPath ?
+          this._buildHeadingPath(chunk, headingPath) :
+          undefined,
+        ingestion_timestamp: this.config.addTimestamp ?
+          new Date().toISOString() :
+          undefined,
+        hash: this.config.addHash ?
+          this.hashUtil.hash(chunk.text) :
+          undefined
+      };
+
+      return {
+        ...chunk,
+        metadata: Object.fromEntries(
+          Object.entries(metadata).filter(([_, v]) => v !== undefined)
+        )
+      };
+    });
+  }
+
+  _extractSection(chunk) {
+    if (chunk.metadata.heading) {
+      return chunk.metadata.heading;
+    }
+    return 'unnamed-section';
+  }
+
+  _buildHeadingPath(chunk, currentPath) {
+    const heading = chunk.metadata.heading;
+
+    if (heading && heading !== currentPath[currentPath.length - 1]) {
+      if (currentPath.length === 0 || this._isSubHeading(heading, currentPath[currentPath.length - 1])) {
+        currentPath.push(heading);
+      } else {
+        currentPath.length = 0;
+        currentPath.push(heading);
+      }
+    }
+
+    return [...currentPath];
+  }
+
+  _isSubHeading(heading1, heading2) {
+    return heading1.length > heading2.length;
+  }
+}

package/lib/scrubber/stages/normalizer.d.ts

@@ -1,13 +1,16 @@
 /**
- * Type definitions for normalizer.js
+ * S-MORA Layer 0 Scrubber - Stage 3: Normalization
+ * @module smora/scrubber/stages/normalizer
  */
-
-export interface NormalizerConfig {
-  lowercase?: boolean;
-  [key: string]: any;
-}
-
-export class Normalizer {
-  constructor(config?: NormalizerConfig);
-  normalize(content: string): Promise<string>;
+export declare class Normalizer {
+    constructor(config: any);
+    /**
+     * Normalize content structure
+     * @param {string} content - Filtered content
+     * @returns {Promise<string>} - Normalized content
+     */
+    normalize(content: any): Promise<any>;
+    _normalizeHeadings(content: any): any;
+    _normalizeLists(content: any): any;
+    _normalizePunctuation(content: any): any;
 }

package/lib/scrubber/stages/normalizer.js

@@ -0,0 +1,51 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 3: Normalization
+ * @module smora/scrubber/stages/normalizer
+ */
+export class Normalizer {
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Normalize content structure
+     * @param {string} content - Filtered content
+     * @returns {Promise<string>} - Normalized content
+     */
+    async normalize(content) {
+        let normalized = content;
+        if (this.config.normalizeHeadings) {
+            normalized = this._normalizeHeadings(normalized);
+        }
+        if (this.config.normalizeLists) {
+            normalized = this._normalizeLists(normalized);
+        }
+        if (this.config.normalizePunctuation) {
+            normalized = this._normalizePunctuation(normalized);
+        }
+        return normalized;
+    }
+    _normalizeHeadings(content) {
+        let normalized = content.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+        normalized = normalized.replace(/^\s*(#{1,6})/gm, '$1');
+        normalized = normalized.replace(/#{7,}/g, '######');
+        return normalized;
+    }
+    _normalizeLists(content) {
+        let normalized = content.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+        normalized = normalized.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+            if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+                return `${ws}${num}. ${char}`;
+            }
+            return match;
+        });
+        return normalized;
+    }
+    _normalizePunctuation(content) {
+        // Remove quotes (both straight and curly)
+        let normalized = content.replace(/["'""''`]/g, '');
+        normalized = normalized.replace(/ +/g, ' ');
+        normalized = normalized.replace(/\.{4,}/g, '...');
+        return normalized;
+    }
+}

package/lib/scrubber/stages/normalizer.ts

@@ -0,0 +1,60 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 3: Normalization
+ * @module smora/scrubber/stages/normalizer
+ */
+
+export class Normalizer {
+  constructor(config) {
+    this.config = config;
+  }
+
+  /**
+   * Normalize content structure
+   * @param {string} content - Filtered content
+   * @returns {Promise<string>} - Normalized content
+   */
+  async normalize(content) {
+    let normalized = content;
+
+    if (this.config.normalizeHeadings) {
+      normalized = this._normalizeHeadings(normalized);
+    }
+
+    if (this.config.normalizeLists) {
+      normalized = this._normalizeLists(normalized);
+    }
+
+    if (this.config.normalizePunctuation) {
+      normalized = this._normalizePunctuation(normalized);
+    }
+
+    return normalized;
+  }
+
+  _normalizeHeadings(content) {
+    let normalized = content.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+    normalized = normalized.replace(/^\s*(#{1,6})/gm, '$1');
+    normalized = normalized.replace(/#{7,}/g, '######');
+    return normalized;
+  }
+
+  _normalizeLists(content) {
+    let normalized = content.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+    normalized = normalized.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+        return `${ws}${num}. ${char}`;
+      }
+      return match;
+    });
+    return normalized;
+  }
+
+  _normalizePunctuation(content) {
+    // Remove quotes (both straight and curly)
+    let normalized = content.replace(/["'""''`]/g, '');
+    normalized = normalized.replace(/ +/g, ' ');
+    normalized = normalized.replace(/\.{4,}/g, '...');
+    return normalized;
+  }
+}

package/lib/scrubber/stages/semantic-filter.d.ts

@@ -1,13 +1,16 @@
 /**
- * Type definitions for semantic-filter.js
+ * S-MORA Layer 0 Scrubber - Stage 2: Semantic Filtering
+ * @module smora/scrubber/stages/semantic-filter
  */
-
-export interface FilterConfig {
-  threshold?: number;
-  [key: string]: any;
-}
-
-export class SemanticFilter {
-  constructor(config?: FilterConfig);
-  filter(content: string): Promise<string>;
+export declare class SemanticFilter {
+    constructor(config: any);
+    /**
+     * Filter semantically empty content
+     * @param {string} content - Cleaned content
+     * @returns {Promise<string>} - Filtered content
+     */
+    filter(content: any): Promise<any>;
+    _isBoilerplate(paragraph: any): any;
+    _removeDuplicates(paragraphs: any): Promise<any>;
+    _hasSignal(paragraph: any): boolean;
 }

package/lib/scrubber/stages/semantic-filter.js

@@ -0,0 +1,51 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 2: Semantic Filtering
+ * @module smora/scrubber/stages/semantic-filter
+ */
+import { PatternMatcher } from '../utils/pattern-matcher.js';
+import { HashUtil } from '../utils/hash.js';
+export class SemanticFilter {
+    constructor(config) {
+        this.config = config;
+        this.patternMatcher = new PatternMatcher();
+        this.hashUtil = new HashUtil();
+    }
+    /**
+     * Filter semantically empty content
+     * @param {string} content - Cleaned content
+     * @returns {Promise<string>} - Filtered content
+     */
+    async filter(content) {
+        const paragraphs = content.split(/\n\n+/);
+        let filtered = paragraphs.filter(p => !this._isBoilerplate(p));
+        filtered = await this._removeDuplicates(filtered);
+        filtered = filtered.filter(p => this._hasSignal(p));
+        return filtered.join('\n\n');
+    }
+    _isBoilerplate(paragraph) {
+        return this.patternMatcher.isBoilerplate(paragraph);
+    }
+    async _removeDuplicates(paragraphs) {
+        if (!this.config.removeDuplicates)
+            return paragraphs;
+        const seen = new Set();
+        const unique = [];
+        for (const para of paragraphs) {
+            const hash = this.hashUtil.hash(para);
+            if (!seen.has(hash)) {
+                seen.add(hash);
+                unique.push(para);
+            }
+        }
+        return unique;
+    }
+    _hasSignal(paragraph) {
+        const text = paragraph.trim();
+        if (text.length < 10)
+            return false;
+        const signalChars = text.replace(/[^a-zA-Z0-9]/g, '').length;
+        const ratio = signalChars / text.length;
+        return ratio >= (this.config.minSignalRatio || 0.3);
+    }
+}

package/lib/scrubber/stages/semantic-filter.ts

@@ -0,0 +1,62 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 2: Semantic Filtering
+ * @module smora/scrubber/stages/semantic-filter
+ */
+
+import { PatternMatcher } from '../utils/pattern-matcher.js';
+import { HashUtil } from '../utils/hash.js';
+
+export class SemanticFilter {
+  constructor(config) {
+    this.config = config;
+    this.patternMatcher = new PatternMatcher();
+    this.hashUtil = new HashUtil();
+  }
+
+  /**
+   * Filter semantically empty content
+   * @param {string} content - Cleaned content
+   * @returns {Promise<string>} - Filtered content
+   */
+  async filter(content) {
+    const paragraphs = content.split(/\n\n+/);
+
+    let filtered = paragraphs.filter(p => !this._isBoilerplate(p));
+    filtered = await this._removeDuplicates(filtered);
+    filtered = filtered.filter(p => this._hasSignal(p));
+
+    return filtered.join('\n\n');
+  }
+
+  _isBoilerplate(paragraph) {
+    return this.patternMatcher.isBoilerplate(paragraph);
+  }
+
+  async _removeDuplicates(paragraphs) {
+    if (!this.config.removeDuplicates) return paragraphs;
+
+    const seen = new Set();
+    const unique = [];
+
+    for (const para of paragraphs) {
+      const hash = this.hashUtil.hash(para);
+      if (!seen.has(hash)) {
+        seen.add(hash);
+        unique.push(para);
+      }
+    }
+
+    return unique;
+  }
+
+  _hasSignal(paragraph) {
+    const text = paragraph.trim();
+    if (text.length < 10) return false;
+
+    const signalChars = text.replace(/[^a-zA-Z0-9]/g, '').length;
+    const ratio = signalChars / text.length;
+
+    return ratio >= (this.config.minSignalRatio || 0.3);
+  }
+}

package/lib/scrubber/stages/structural-cleaner.d.ts

@@ -1,13 +1,18 @@
 /**
- * Type definitions for structural-cleaner.js
+ * S-MORA Layer 0 Scrubber - Stage 1: Structural Cleaning
+ * @module smora/scrubber/stages/structural-cleaner
  */
-
-export interface CleanConfig {
-  preserveStructure?: boolean;
-  [key: string]: any;
-}
-
-export class StructuralCleaner {
-  constructor(config?: CleanConfig);
-  clean(content: string): Promise<string>;
+export declare class StructuralCleaner {
+    constructor(config: any);
+    /**
+     * Clean document structure
+     * @param {string} content - Raw document content
+     * @returns {Promise<string>} - Cleaned content
+     */
+    clean(content: any): Promise<any>;
+    _detectType(content: any): "html" | "markdown" | "text";
+    _cleanHTML(content: any): Promise<any>;
+    _cleanMarkdown(content: any): Promise<any>;
+    _collapseWhitespace(content: any): any;
+    _normalizeLineBreaks(content: any): any;
 }

package/lib/scrubber/stages/structural-cleaner.js

@@ -0,0 +1,73 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 1: Structural Cleaning
+ * @module smora/scrubber/stages/structural-cleaner
+ */
+import { HTMLParser } from '../utils/html-parser.js';
+import { ScrubberError } from '../errors/scrubber-error.js';
+export class StructuralCleaner {
+    constructor(config) {
+        this.config = config;
+        this.htmlParser = new HTMLParser();
+    }
+    /**
+     * Clean document structure
+     * @param {string} content - Raw document content
+     * @returns {Promise<string>} - Cleaned content
+     */
+    async clean(content) {
+        try {
+            const type = this._detectType(content);
+            let cleaned = content;
+            if (type === 'html') {
+                cleaned = await this._cleanHTML(cleaned);
+                // HTML may have markdown headings, normalize them
+                cleaned = await this._cleanMarkdown(cleaned);
+            }
+            else if (type === 'markdown') {
+                cleaned = await this._cleanMarkdown(cleaned);
+            }
+            cleaned = this._collapseWhitespace(cleaned);
+            cleaned = this._normalizeLineBreaks(cleaned);
+            return cleaned;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new ScrubberError(`Failed to clean content: ${message}`, { stage: 'structural-cleaner', originalError: error });
+        }
+    }
+    _detectType(content) {
+        if (content.trim().startsWith('<'))
+            return 'html';
+        if (/^#{1,6}\s/.test(content) || /^#{1,6}[A-Za-z]/.test(content))
+            return 'markdown';
+        return 'text';
+    }
+    async _cleanHTML(content) {
+        return this.htmlParser.parse(content);
+    }
+    async _cleanMarkdown(content) {
+        let cleaned = content;
+        // Add space after heading markers when missing
+        cleaned = cleaned.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+        // Add space after list markers when missing
+        cleaned = cleaned.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+        // Add space after numbered list markers when missing
+        cleaned = cleaned.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+            // Only if it looks like a numbered list (digit followed by non-dot, non-space)
+            if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+                return `${ws}${num}. ${char}`;
+            }
+            return match;
+        });
+        return cleaned;
+    }
+    _collapseWhitespace(content) {
+        let cleaned = content.replace(/[ \t]+/g, ' ');
+        cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+        return cleaned;
+    }
+    _normalizeLineBreaks(content) {
+        return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    }
+}

package/lib/scrubber/stages/structural-cleaner.ts

@@ -0,0 +1,83 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 1: Structural Cleaning
+ * @module smora/scrubber/stages/structural-cleaner
+ */
+
+import { HTMLParser } from '../utils/html-parser.js';
+import { StructuralCleaningError, ScrubberError } from '../errors/scrubber-error.js';
+
+export class StructuralCleaner {
+  constructor(config) {
+    this.config = config;
+    this.htmlParser = new HTMLParser();
+  }
+
+  /**
+   * Clean document structure
+   * @param {string} content - Raw document content
+   * @returns {Promise<string>} - Cleaned content
+   */
+  async clean(content) {
+    try {
+      const type = this._detectType(content);
+      let cleaned = content;
+
+      if (type === 'html') {
+        cleaned = await this._cleanHTML(cleaned);
+        // HTML may have markdown headings, normalize them
+        cleaned = await this._cleanMarkdown(cleaned);
+      } else if (type === 'markdown') {
+        cleaned = await this._cleanMarkdown(cleaned);
+      }
+
+      cleaned = this._collapseWhitespace(cleaned);
+      cleaned = this._normalizeLineBreaks(cleaned);
+
+      return cleaned;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ScrubberError(
+        `Failed to clean content: ${message}`,
+        { stage: 'structural-cleaner', originalError: error }
+      );
+    }
+  }
+
+  _detectType(content) {
+    if (content.trim().startsWith('<')) return 'html';
+    if (/^#{1,6}\s/.test(content) || /^#{1,6}[A-Za-z]/.test(content)) return 'markdown';
+    return 'text';
+  }
+
+  async _cleanHTML(content) {
+    return this.htmlParser.parse(content);
+  }
+
+  async _cleanMarkdown(content) {
+    let cleaned = content;
+    // Add space after heading markers when missing
+    cleaned = cleaned.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+    // Add space after list markers when missing
+    cleaned = cleaned.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+    // Add space after numbered list markers when missing
+    cleaned = cleaned.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+      // Only if it looks like a numbered list (digit followed by non-dot, non-space)
+      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+        return `${ws}${num}. ${char}`;
+      }
+      return match;
+    });
+    return cleaned;
+  }
+
+  _collapseWhitespace(content) {
+    let cleaned = content.replace(/[ \t]+/g, ' ');
+    cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+    return cleaned;
+  }
+
+  _normalizeLineBreaks(content) {
+    return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+  }
+}

package/lib/scrubber/stages/validator.d.ts

@@ -1,18 +1,17 @@
 /**
- * Type definitions for validator.js
+ * S-MORA Layer 0 Scrubber - Stage 6: Validation
+ * @module smora/scrubber/stages/validator
  */
-
-export interface ValidatorConfig {
-  strict?: boolean;
-  [key: string]: any;
-}
-
-export interface ValidationResult {
-  valid: boolean;
-  errors?: string[];
-}
-
-export class Validator {
-  constructor(config?: ValidatorConfig);
-  validate(content: string): Promise<ValidationResult>;
+export declare class Validator {
+    constructor(config: any);
+    /**
+     * Validate chunks
+     * @param {Array} chunks - Array of chunks
+     * @returns {Promise<Array>} - Validated chunks
+     */
+    validate(chunks: any): Promise<any[]>;
+    _validateChunk(chunk: any): {
+        valid: boolean;
+        errors: any[];
+    };
 }

package/lib/scrubber/stages/validator.js

@@ -0,0 +1,56 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 6: Validation
+ * @module smora/scrubber/stages/validator
+ */
+import { TokenCounter } from '../utils/token-counter.js';
+export class Validator {
+    constructor(config) {
+        this.config = config;
+        this.tokenCounter = new TokenCounter();
+    }
+    /**
+     * Validate chunks
+     * @param {Array} chunks - Array of chunks
+     * @returns {Promise<Array>} - Validated chunks
+     */
+    async validate(chunks) {
+        const valid = [];
+        const errors = [];
+        for (const chunk of chunks) {
+            const validation = this._validateChunk(chunk);
+            if (validation.valid) {
+                valid.push(chunk);
+            }
+            else {
+                errors.push({
+                    chunkIndex: chunk.index,
+                    errors: validation.errors
+                });
+            }
+        }
+        return valid;
+    }
+    _validateChunk(chunk) {
+        const errors = [];
+        if (this.config.rejectEmptyChunks && !chunk.text.trim()) {
+            errors.push('empty_chunk');
+        }
+        if (this.config.enforceMinLength) {
+            const tokens = this.tokenCounter.count(chunk.text);
+            if (tokens < this.config.minTokens) {
+                errors.push(`chunk_too_short: ${tokens} < ${this.config.minTokens}`);
+            }
+        }
+        if (this.config.enforceMaxLength) {
+            const tokens = this.tokenCounter.count(chunk.text);
+            if (tokens > this.config.hardMaxTokens) {
+                errors.push(`chunk_too_long: ${tokens} > ${this.config.hardMaxTokens}`);
+            }
+        }
+        return {
+            valid: errors.length === 0,
+            errors
+        };
+    }
+}