@yamo/memory-mesh 2.3.2 → 3.0.1

package/lib/scrubber/stages/normalizer.js

@@ -1,59 +1,51 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Stage 3: Normalization
  * @module smora/scrubber/stages/normalizer
  */
-
 export class Normalizer {
-  constructor(config) {
-    this.config = config;
-  }
-
-  /**
-   * Normalize content structure
-   * @param {string} content - Filtered content
-   * @returns {Promise<string>} - Normalized content
-   */
-  async normalize(content) {
-    let normalized = content;
-
-    if (this.config.normalizeHeadings) {
-      normalized = this._normalizeHeadings(normalized);
+    constructor(config) {
+        this.config = config;
     }
-
-    if (this.config.normalizeLists) {
-      normalized = this._normalizeLists(normalized);
+    /**
+     * Normalize content structure
+     * @param {string} content - Filtered content
+     * @returns {Promise<string>} - Normalized content
+     */
+    async normalize(content) {
+        let normalized = content;
+        if (this.config.normalizeHeadings) {
+            normalized = this._normalizeHeadings(normalized);
+        }
+        if (this.config.normalizeLists) {
+            normalized = this._normalizeLists(normalized);
+        }
+        if (this.config.normalizePunctuation) {
+            normalized = this._normalizePunctuation(normalized);
+        }
+        return normalized;
     }
-
-    if (this.config.normalizePunctuation) {
-      normalized = this._normalizePunctuation(normalized);
+    _normalizeHeadings(content) {
+        let normalized = content.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+        normalized = normalized.replace(/^\s*(#{1,6})/gm, '$1');
+        normalized = normalized.replace(/#{7,}/g, '######');
+        return normalized;
+    }
+    _normalizeLists(content) {
+        let normalized = content.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+        normalized = normalized.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+            if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+                return `${ws}${num}. ${char}`;
+            }
+            return match;
+        });
+        return normalized;
+    }
+    _normalizePunctuation(content) {
+        // Remove quotes (both straight and curly)
+        let normalized = content.replace(/["'""''`]/g, '');
+        normalized = normalized.replace(/ +/g, ' ');
+        normalized = normalized.replace(/\.{4,}/g, '...');
+        return normalized;
     }
-
-    return normalized;
-  }
-
-  _normalizeHeadings(content) {
-    let normalized = content.replace(/(#{1,6})([^\s#])/g, '$1 $2');
-    normalized = normalized.replace(/^\s*(#{1,6})/gm, '$1');
-    normalized = normalized.replace(/#{7,}/g, '######');
-    return normalized;
-  }
-
-  _normalizeLists(content) {
-    let normalized = content.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
-    normalized = normalized.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
-      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
-        return `${ws}${num}. ${char}`;
-      }
-      return match;
-    });
-    return normalized;
-  }
-
-  _normalizePunctuation(content) {
-    // Remove quotes (both straight and curly)
-    let normalized = content.replace(/["'""''`]/g, '');
-    normalized = normalized.replace(/ +/g, ' ');
-    normalized = normalized.replace(/\.{4,}/g, '...');
-    return normalized;
-  }
 }

package/lib/scrubber/stages/normalizer.ts

@@ -0,0 +1,60 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 3: Normalization
+ * @module smora/scrubber/stages/normalizer
+ */
+
+export class Normalizer {
+  constructor(config) {
+    this.config = config;
+  }
+
+  /**
+   * Normalize content structure
+   * @param {string} content - Filtered content
+   * @returns {Promise<string>} - Normalized content
+   */
+  async normalize(content) {
+    let normalized = content;
+
+    if (this.config.normalizeHeadings) {
+      normalized = this._normalizeHeadings(normalized);
+    }
+
+    if (this.config.normalizeLists) {
+      normalized = this._normalizeLists(normalized);
+    }
+
+    if (this.config.normalizePunctuation) {
+      normalized = this._normalizePunctuation(normalized);
+    }
+
+    return normalized;
+  }
+
+  _normalizeHeadings(content) {
+    let normalized = content.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+    normalized = normalized.replace(/^\s*(#{1,6})/gm, '$1');
+    normalized = normalized.replace(/#{7,}/g, '######');
+    return normalized;
+  }
+
+  _normalizeLists(content) {
+    let normalized = content.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+    normalized = normalized.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+        return `${ws}${num}. ${char}`;
+      }
+      return match;
+    });
+    return normalized;
+  }
+
+  _normalizePunctuation(content) {
+    // Remove quotes (both straight and curly)
+    let normalized = content.replace(/["'""''`]/g, '');
+    normalized = normalized.replace(/ +/g, ' ');
+    normalized = normalized.replace(/\.{4,}/g, '...');
+    return normalized;
+  }
+}

package/lib/scrubber/stages/semantic-filter.d.ts

@@ -0,0 +1,16 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 2: Semantic Filtering
+ * @module smora/scrubber/stages/semantic-filter
+ */
+export declare class SemanticFilter {
+    constructor(config: any);
+    /**
+     * Filter semantically empty content
+     * @param {string} content - Cleaned content
+     * @returns {Promise<string>} - Filtered content
+     */
+    filter(content: any): Promise<any>;
+    _isBoilerplate(paragraph: any): any;
+    _removeDuplicates(paragraphs: any): Promise<any>;
+    _hasSignal(paragraph: any): boolean;
+}

package/lib/scrubber/stages/semantic-filter.js

@@ -1,61 +1,51 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Stage 2: Semantic Filtering
  * @module smora/scrubber/stages/semantic-filter
  */
-
 import { PatternMatcher } from '../utils/pattern-matcher.js';
 import { HashUtil } from '../utils/hash.js';
-
 export class SemanticFilter {
-  constructor(config) {
-    this.config = config;
-    this.patternMatcher = new PatternMatcher();
-    this.hashUtil = new HashUtil();
-  }
-
-  /**
-   * Filter semantically empty content
-   * @param {string} content - Cleaned content
-   * @returns {Promise<string>} - Filtered content
-   */
-  async filter(content) {
-    const paragraphs = content.split(/\n\n+/);
-
-    let filtered = paragraphs.filter(p => !this._isBoilerplate(p));
-    filtered = await this._removeDuplicates(filtered);
-    filtered = filtered.filter(p => this._hasSignal(p));
-
-    return filtered.join('\n\n');
-  }
-
-  _isBoilerplate(paragraph) {
-    return this.patternMatcher.isBoilerplate(paragraph);
-  }
-
-  async _removeDuplicates(paragraphs) {
-    if (!this.config.removeDuplicates) return paragraphs;
-
-    const seen = new Set();
-    const unique = [];
-
-    for (const para of paragraphs) {
-      const hash = this.hashUtil.hash(para);
-      if (!seen.has(hash)) {
-        seen.add(hash);
-        unique.push(para);
-      }
+    constructor(config) {
+        this.config = config;
+        this.patternMatcher = new PatternMatcher();
+        this.hashUtil = new HashUtil();
+    }
+    /**
+     * Filter semantically empty content
+     * @param {string} content - Cleaned content
+     * @returns {Promise<string>} - Filtered content
+     */
+    async filter(content) {
+        const paragraphs = content.split(/\n\n+/);
+        let filtered = paragraphs.filter(p => !this._isBoilerplate(p));
+        filtered = await this._removeDuplicates(filtered);
+        filtered = filtered.filter(p => this._hasSignal(p));
+        return filtered.join('\n\n');
+    }
+    _isBoilerplate(paragraph) {
+        return this.patternMatcher.isBoilerplate(paragraph);
+    }
+    async _removeDuplicates(paragraphs) {
+        if (!this.config.removeDuplicates)
+            return paragraphs;
+        const seen = new Set();
+        const unique = [];
+        for (const para of paragraphs) {
+            const hash = this.hashUtil.hash(para);
+            if (!seen.has(hash)) {
+                seen.add(hash);
+                unique.push(para);
+            }
+        }
+        return unique;
+    }
+    _hasSignal(paragraph) {
+        const text = paragraph.trim();
+        if (text.length < 10)
+            return false;
+        const signalChars = text.replace(/[^a-zA-Z0-9]/g, '').length;
+        const ratio = signalChars / text.length;
+        return ratio >= (this.config.minSignalRatio || 0.3);
     }
-
-    return unique;
-  }
-
-  _hasSignal(paragraph) {
-    const text = paragraph.trim();
-    if (text.length < 10) return false;
-
-    const signalChars = text.replace(/[^a-zA-Z0-9]/g, '').length;
-    const ratio = signalChars / text.length;
-
-    return ratio >= (this.config.minSignalRatio || 0.3);
-  }
 }

package/lib/scrubber/stages/semantic-filter.ts

@@ -0,0 +1,62 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 2: Semantic Filtering
+ * @module smora/scrubber/stages/semantic-filter
+ */
+
+import { PatternMatcher } from '../utils/pattern-matcher.js';
+import { HashUtil } from '../utils/hash.js';
+
+export class SemanticFilter {
+  constructor(config) {
+    this.config = config;
+    this.patternMatcher = new PatternMatcher();
+    this.hashUtil = new HashUtil();
+  }
+
+  /**
+   * Filter semantically empty content
+   * @param {string} content - Cleaned content
+   * @returns {Promise<string>} - Filtered content
+   */
+  async filter(content) {
+    const paragraphs = content.split(/\n\n+/);
+
+    let filtered = paragraphs.filter(p => !this._isBoilerplate(p));
+    filtered = await this._removeDuplicates(filtered);
+    filtered = filtered.filter(p => this._hasSignal(p));
+
+    return filtered.join('\n\n');
+  }
+
+  _isBoilerplate(paragraph) {
+    return this.patternMatcher.isBoilerplate(paragraph);
+  }
+
+  async _removeDuplicates(paragraphs) {
+    if (!this.config.removeDuplicates) return paragraphs;
+
+    const seen = new Set();
+    const unique = [];
+
+    for (const para of paragraphs) {
+      const hash = this.hashUtil.hash(para);
+      if (!seen.has(hash)) {
+        seen.add(hash);
+        unique.push(para);
+      }
+    }
+
+    return unique;
+  }
+
+  _hasSignal(paragraph) {
+    const text = paragraph.trim();
+    if (text.length < 10) return false;
+
+    const signalChars = text.replace(/[^a-zA-Z0-9]/g, '').length;
+    const ratio = signalChars / text.length;
+
+    return ratio >= (this.config.minSignalRatio || 0.3);
+  }
+}

package/lib/scrubber/stages/structural-cleaner.d.ts

@@ -0,0 +1,18 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 1: Structural Cleaning
+ * @module smora/scrubber/stages/structural-cleaner
+ */
+export declare class StructuralCleaner {
+    constructor(config: any);
+    /**
+     * Clean document structure
+     * @param {string} content - Raw document content
+     * @returns {Promise<string>} - Cleaned content
+     */
+    clean(content: any): Promise<any>;
+    _detectType(content: any): "html" | "markdown" | "text";
+    _cleanHTML(content: any): Promise<any>;
+    _cleanMarkdown(content: any): Promise<any>;
+    _collapseWhitespace(content: any): any;
+    _normalizeLineBreaks(content: any): any;
+}

package/lib/scrubber/stages/structural-cleaner.js

@@ -1,82 +1,73 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Stage 1: Structural Cleaning
  * @module smora/scrubber/stages/structural-cleaner
  */
-
 import { HTMLParser } from '../utils/html-parser.js';
-import { StructuralCleaningError, ScrubberError } from '../errors/scrubber-error.js';
-
+import { ScrubberError } from '../errors/scrubber-error.js';
 export class StructuralCleaner {
-  constructor(config) {
-    this.config = config;
-    this.htmlParser = new HTMLParser();
-  }
-
-  /**
-   * Clean document structure
-   * @param {string} content - Raw document content
-   * @returns {Promise<string>} - Cleaned content
-   */
-  async clean(content) {
-    try {
-      const type = this._detectType(content);
-      let cleaned = content;
-
-      if (type === 'html') {
-        cleaned = await this._cleanHTML(cleaned);
-        // HTML may have markdown headings, normalize them
-        cleaned = await this._cleanMarkdown(cleaned);
-      } else if (type === 'markdown') {
-        cleaned = await this._cleanMarkdown(cleaned);
-      }
-
-      cleaned = this._collapseWhitespace(cleaned);
-      cleaned = this._normalizeLineBreaks(cleaned);
-
-      return cleaned;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      throw new ScrubberError(
-        `Failed to clean content: ${message}`,
-        { stage: 'structural-cleaner', originalError: error }
-      );
+    constructor(config) {
+        this.config = config;
+        this.htmlParser = new HTMLParser();
     }
-  }
-
-  _detectType(content) {
-    if (content.trim().startsWith('<')) return 'html';
-    if (/^#{1,6}\s/.test(content) || /^#{1,6}[A-Za-z]/.test(content)) return 'markdown';
-    return 'text';
-  }
-
-  async _cleanHTML(content) {
-    return this.htmlParser.parse(content);
-  }
-
-  async _cleanMarkdown(content) {
-    let cleaned = content;
-    // Add space after heading markers when missing
-    cleaned = cleaned.replace(/(#{1,6})([^\s#])/g, '$1 $2');
-    // Add space after list markers when missing
-    cleaned = cleaned.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
-    // Add space after numbered list markers when missing
-    cleaned = cleaned.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
-      // Only if it looks like a numbered list (digit followed by non-dot, non-space)
-      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
-        return `${ws}${num}. ${char}`;
-      }
-      return match;
-    });
-    return cleaned;
-  }
-
-  _collapseWhitespace(content) {
-    let cleaned = content.replace(/[ \t]+/g, ' ');
-    cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
-    return cleaned;
-  }
-
-  _normalizeLineBreaks(content) {
-    return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
-  }
-}
+    /**
+     * Clean document structure
+     * @param {string} content - Raw document content
+     * @returns {Promise<string>} - Cleaned content
+     */
+    async clean(content) {
+        try {
+            const type = this._detectType(content);
+            let cleaned = content;
+            if (type === 'html') {
+                cleaned = await this._cleanHTML(cleaned);
+                // HTML may have markdown headings, normalize them
+                cleaned = await this._cleanMarkdown(cleaned);
+            }
+            else if (type === 'markdown') {
+                cleaned = await this._cleanMarkdown(cleaned);
+            }
+            cleaned = this._collapseWhitespace(cleaned);
+            cleaned = this._normalizeLineBreaks(cleaned);
+            return cleaned;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new ScrubberError(`Failed to clean content: ${message}`, { stage: 'structural-cleaner', originalError: error });
+        }
+    }
+    _detectType(content) {
+        if (content.trim().startsWith('<'))
+            return 'html';
+        if (/^#{1,6}\s/.test(content) || /^#{1,6}[A-Za-z]/.test(content))
+            return 'markdown';
+        return 'text';
+    }
+    async _cleanHTML(content) {
+        return this.htmlParser.parse(content);
+    }
+    async _cleanMarkdown(content) {
+        let cleaned = content;
+        // Add space after heading markers when missing
+        cleaned = cleaned.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+        // Add space after list markers when missing
+        cleaned = cleaned.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+        // Add space after numbered list markers when missing
+        cleaned = cleaned.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+            // Only if it looks like a numbered list (digit followed by non-dot, non-space)
+            if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+                return `${ws}${num}. ${char}`;
+            }
+            return match;
+        });
+        return cleaned;
+    }
+    _collapseWhitespace(content) {
+        let cleaned = content.replace(/[ \t]+/g, ' ');
+        cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+        return cleaned;
+    }
+    _normalizeLineBreaks(content) {
+        return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    }
+}

package/lib/scrubber/stages/structural-cleaner.ts

@@ -0,0 +1,83 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 1: Structural Cleaning
+ * @module smora/scrubber/stages/structural-cleaner
+ */
+
+import { HTMLParser } from '../utils/html-parser.js';
+import { StructuralCleaningError, ScrubberError } from '../errors/scrubber-error.js';
+
+export class StructuralCleaner {
+  constructor(config) {
+    this.config = config;
+    this.htmlParser = new HTMLParser();
+  }
+
+  /**
+   * Clean document structure
+   * @param {string} content - Raw document content
+   * @returns {Promise<string>} - Cleaned content
+   */
+  async clean(content) {
+    try {
+      const type = this._detectType(content);
+      let cleaned = content;
+
+      if (type === 'html') {
+        cleaned = await this._cleanHTML(cleaned);
+        // HTML may have markdown headings, normalize them
+        cleaned = await this._cleanMarkdown(cleaned);
+      } else if (type === 'markdown') {
+        cleaned = await this._cleanMarkdown(cleaned);
+      }
+
+      cleaned = this._collapseWhitespace(cleaned);
+      cleaned = this._normalizeLineBreaks(cleaned);
+
+      return cleaned;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ScrubberError(
+        `Failed to clean content: ${message}`,
+        { stage: 'structural-cleaner', originalError: error }
+      );
+    }
+  }
+
+  _detectType(content) {
+    if (content.trim().startsWith('<')) return 'html';
+    if (/^#{1,6}\s/.test(content) || /^#{1,6}[A-Za-z]/.test(content)) return 'markdown';
+    return 'text';
+  }
+
+  async _cleanHTML(content) {
+    return this.htmlParser.parse(content);
+  }
+
+  async _cleanMarkdown(content) {
+    let cleaned = content;
+    // Add space after heading markers when missing
+    cleaned = cleaned.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+    // Add space after list markers when missing
+    cleaned = cleaned.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+    // Add space after numbered list markers when missing
+    cleaned = cleaned.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+      // Only if it looks like a numbered list (digit followed by non-dot, non-space)
+      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+        return `${ws}${num}. ${char}`;
+      }
+      return match;
+    });
+    return cleaned;
+  }
+
+  _collapseWhitespace(content) {
+    let cleaned = content.replace(/[ \t]+/g, ' ');
+    cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+    return cleaned;
+  }
+
+  _normalizeLineBreaks(content) {
+    return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+  }
+}

package/lib/scrubber/stages/validator.d.ts

@@ -0,0 +1,17 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 6: Validation
+ * @module smora/scrubber/stages/validator
+ */
+export declare class Validator {
+    constructor(config: any);
+    /**
+     * Validate chunks
+     * @param {Array} chunks - Array of chunks
+     * @returns {Promise<Array>} - Validated chunks
+     */
+    validate(chunks: any): Promise<any[]>;
+    _validateChunk(chunk: any): {
+        valid: boolean;
+        errors: any[];
+    };
+}