@yamo/memory-mesh 2.3.2 → 3.0.1

package/lib/scrubber/stages/validator.js

@@ -1,66 +1,56 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Stage 6: Validation
  * @module smora/scrubber/stages/validator
  */
-
 import { TokenCounter } from '../utils/token-counter.js';
-import { ValidationError } from '../errors/scrubber-error.js';
-
 export class Validator {
-  constructor(config) {
-    this.config = config;
-    this.tokenCounter = new TokenCounter();
-  }
-
-  /**
-   * Validate chunks
-   * @param {Array} chunks - Array of chunks
-   * @returns {Promise<Array>} - Validated chunks
-   */
-  async validate(chunks) {
-    const valid = [];
-    const errors = [];
-
-    for (const chunk of chunks) {
-      const validation = this._validateChunk(chunk);
-
-      if (validation.valid) {
-        valid.push(chunk);
-      } else {
-        errors.push({
-          chunkIndex: chunk.index,
-          errors: validation.errors
-        });
-      }
+    constructor(config) {
+        this.config = config;
+        this.tokenCounter = new TokenCounter();
     }
-
-    return valid;
-  }
-
-  _validateChunk(chunk) {
-    const errors = [];
-
-    if (this.config.rejectEmptyChunks && !chunk.text.trim()) {
-      errors.push('empty_chunk');
+    /**
+     * Validate chunks
+     * @param {Array} chunks - Array of chunks
+     * @returns {Promise<Array>} - Validated chunks
+     */
+    async validate(chunks) {
+        const valid = [];
+        const errors = [];
+        for (const chunk of chunks) {
+            const validation = this._validateChunk(chunk);
+            if (validation.valid) {
+                valid.push(chunk);
+            }
+            else {
+                errors.push({
+                    chunkIndex: chunk.index,
+                    errors: validation.errors
+                });
+            }
+        }
+        return valid;
     }
-
-    if (this.config.enforceMinLength) {
-      const tokens = this.tokenCounter.count(chunk.text);
-      if (tokens < this.config.minTokens) {
-        errors.push(`chunk_too_short: ${tokens} < ${this.config.minTokens}`);
-      }
+    _validateChunk(chunk) {
+        const errors = [];
+        if (this.config.rejectEmptyChunks && !chunk.text.trim()) {
+            errors.push('empty_chunk');
+        }
+        if (this.config.enforceMinLength) {
+            const tokens = this.tokenCounter.count(chunk.text);
+            if (tokens < this.config.minTokens) {
+                errors.push(`chunk_too_short: ${tokens} < ${this.config.minTokens}`);
+            }
+        }
+        if (this.config.enforceMaxLength) {
+            const tokens = this.tokenCounter.count(chunk.text);
+            if (tokens > this.config.hardMaxTokens) {
+                errors.push(`chunk_too_long: ${tokens} > ${this.config.hardMaxTokens}`);
+            }
+        }
+        return {
+            valid: errors.length === 0,
+            errors
+        };
     }
-
-    if (this.config.enforceMaxLength) {
-      const tokens = this.tokenCounter.count(chunk.text);
-      if (tokens > this.config.hardMaxTokens) {
-        errors.push(`chunk_too_long: ${tokens} > ${this.config.hardMaxTokens}`);
-      }
-    }
-
-    return {
-      valid: errors.length === 0,
-      errors
-    };
-  }
 }

package/lib/scrubber/stages/validator.ts

@@ -0,0 +1,67 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 6: Validation
+ * @module smora/scrubber/stages/validator
+ */
+
+import { TokenCounter } from '../utils/token-counter.js';
+import { ValidationError } from '../errors/scrubber-error.js';
+
+export class Validator {
+  constructor(config) {
+    this.config = config;
+    this.tokenCounter = new TokenCounter();
+  }
+
+  /**
+   * Validate chunks
+   * @param {Array} chunks - Array of chunks
+   * @returns {Promise<Array>} - Validated chunks
+   */
+  async validate(chunks) {
+    const valid = [];
+    const errors = [];
+
+    for (const chunk of chunks) {
+      const validation = this._validateChunk(chunk);
+
+      if (validation.valid) {
+        valid.push(chunk);
+      } else {
+        errors.push({
+          chunkIndex: chunk.index,
+          errors: validation.errors
+        });
+      }
+    }
+
+    return valid;
+  }
+
+  _validateChunk(chunk) {
+    const errors = [];
+
+    if (this.config.rejectEmptyChunks && !chunk.text.trim()) {
+      errors.push('empty_chunk');
+    }
+
+    if (this.config.enforceMinLength) {
+      const tokens = this.tokenCounter.count(chunk.text);
+      if (tokens < this.config.minTokens) {
+        errors.push(`chunk_too_short: ${tokens} < ${this.config.minTokens}`);
+      }
+    }
+
+    if (this.config.enforceMaxLength) {
+      const tokens = this.tokenCounter.count(chunk.text);
+      if (tokens > this.config.hardMaxTokens) {
+        errors.push(`chunk_too_long: ${tokens} > ${this.config.hardMaxTokens}`);
+      }
+    }
+
+    return {
+      valid: errors.length === 0,
+      errors
+    };
+  }
+}

package/lib/scrubber/telemetry.d.ts

@@ -0,0 +1,29 @@
+/**
+ * S-MORA Layer 0 Scrubber Telemetry Collection
+ * @module smora/scrubber/telemetry
+ */
+export declare class ScrubberTelemetry {
+    stats: any;
+    constructor();
+    recordStage(stage: any, duration: any, success?: boolean): void;
+    getStageStats(stage: any): {
+        count: any;
+        avgTime: number;
+        totalTime: any;
+        errors: any;
+    };
+    getSummary(): {
+        stages: any;
+        performance: {
+            structural: any;
+            semantic: any;
+            normalization: any;
+            chunking: any;
+            metadata: any;
+            validation: any;
+            total: unknown;
+        };
+    };
+    reset(): void;
+    assertPerformanceBudget(budget?: number): void;
+}

package/lib/scrubber/telemetry.js

@@ -1,66 +1,62 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber Telemetry Collection
  * @module smora/scrubber/telemetry
  */
-
 export class ScrubberTelemetry {
-  constructor() {
-    this.stats = {
-      structural: { count: 0, totalTime: 0, errors: 0 },
-      semantic: { count: 0, totalTime: 0, errors: 0 },
-      normalization: { count: 0, totalTime: 0, errors: 0 },
-      chunking: { count: 0, totalTime: 0, errors: 0 },
-      metadata: { count: 0, totalTime: 0, errors: 0 },
-      validation: { count: 0, totalTime: 0, errors: 0 }
-    };
-  }
-
-  recordStage(stage, duration, success = true) {
-    if (!this.stats[stage]) {
-      this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+    stats;
+    constructor() {
+        this.stats = {
+            structural: { count: 0, totalTime: 0, errors: 0 },
+            semantic: { count: 0, totalTime: 0, errors: 0 },
+            normalization: { count: 0, totalTime: 0, errors: 0 },
+            chunking: { count: 0, totalTime: 0, errors: 0 },
+            metadata: { count: 0, totalTime: 0, errors: 0 },
+            validation: { count: 0, totalTime: 0, errors: 0 },
+        };
     }
-    this.stats[stage].count++;
-    this.stats[stage].totalTime += duration;
-    if (!success) this.stats[stage].errors++;
-  }
-
-  getStageStats(stage) {
-    const stats = this.stats[stage];
-    return {
-      count: stats.count,
-      avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
-      totalTime: stats.totalTime,
-      errors: stats.errors
-    };
-  }
-
-  getSummary() {
-    return {
-      stages: this.stats,
-      performance: {
-        structural: this.stats.structural.totalTime,
-        semantic: this.stats.semantic.totalTime,
-        normalization: this.stats.normalization.totalTime,
-        chunking: this.stats.chunking.totalTime,
-        metadata: this.stats.metadata.totalTime,
-        validation: this.stats.validation.totalTime,
-        total: Object.values(this.stats).reduce((sum, s) => sum + s.totalTime, 0)
-      }
-    };
-  }
-
-  reset() {
-    Object.keys(this.stats).forEach(key => {
-      this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
-    });
-  }
-
-  assertPerformanceBudget(budget = 10) {
-    const summary = this.getSummary();
-    if (summary.performance.total > budget) {
-      throw new Error(
-        `Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`
-      );
+    recordStage(stage, duration, success = true) {
+        if (!this.stats[stage]) {
+            this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+        }
+        this.stats[stage].count++;
+        this.stats[stage].totalTime += duration;
+        if (!success) {
+            this.stats[stage].errors++;
+        }
+    }
+    getStageStats(stage) {
+        const stats = this.stats[stage] || { count: 0, totalTime: 0, errors: 0 };
+        return {
+            count: stats.count,
+            avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
+            totalTime: stats.totalTime,
+            errors: stats.errors,
+        };
+    }
+    getSummary() {
+        return {
+            stages: this.stats,
+            performance: {
+                structural: this.stats.structural.totalTime,
+                semantic: this.stats.semantic.totalTime,
+                normalization: this.stats.normalization.totalTime,
+                chunking: this.stats.chunking.totalTime,
+                metadata: this.stats.metadata.totalTime,
+                validation: this.stats.validation.totalTime,
+                total: Object.values(this.stats).reduce((sum, s) => sum + s.totalTime, 0),
+            },
+        };
+    }
+    reset() {
+        Object.keys(this.stats).forEach((key) => {
+            this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
+        });
+    }
+    assertPerformanceBudget(budget = 10) {
+        const summary = this.getSummary();
+        if (summary.performance.total > budget) {
+            throw new Error(`Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`);
+        }
     }
-  }
 }

package/lib/scrubber/telemetry.ts

@@ -0,0 +1,62 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber Telemetry Collection
+ * @module smora/scrubber/telemetry
+ */
+export class ScrubberTelemetry {
+    stats;
+    constructor() {
+        this.stats = {
+            structural: { count: 0, totalTime: 0, errors: 0 },
+            semantic: { count: 0, totalTime: 0, errors: 0 },
+            normalization: { count: 0, totalTime: 0, errors: 0 },
+            chunking: { count: 0, totalTime: 0, errors: 0 },
+            metadata: { count: 0, totalTime: 0, errors: 0 },
+            validation: { count: 0, totalTime: 0, errors: 0 },
+        };
+    }
+    recordStage(stage, duration, success = true) {
+        if (!this.stats[stage]) {
+            this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+        }
+        this.stats[stage].count++;
+        this.stats[stage].totalTime += duration;
+        if (!success) {
+            this.stats[stage].errors++;
+        }
+    }
+    getStageStats(stage) {
+        const stats = this.stats[stage] || { count: 0, totalTime: 0, errors: 0 };
+        return {
+            count: stats.count,
+            avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
+            totalTime: stats.totalTime,
+            errors: stats.errors,
+        };
+    }
+    getSummary() {
+        return {
+            stages: this.stats,
+            performance: {
+                structural: this.stats.structural.totalTime,
+                semantic: this.stats.semantic.totalTime,
+                normalization: this.stats.normalization.totalTime,
+                chunking: this.stats.chunking.totalTime,
+                metadata: this.stats.metadata.totalTime,
+                validation: this.stats.validation.totalTime,
+                total: Object.values(this.stats).reduce((sum, s) => sum + s.totalTime, 0),
+            },
+        };
+    }
+    reset() {
+        Object.keys(this.stats).forEach((key) => {
+            this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
+        });
+    }
+    assertPerformanceBudget(budget = 10) {
+        const summary = this.getSummary();
+        if (summary.performance.total > budget) {
+            throw new Error(`Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`);
+        }
+    }
+}

package/lib/scrubber/utils/hash.d.ts

@@ -0,0 +1,14 @@
+export declare class HashUtil {
+    /**
+     * Hash content for deduplication
+     * @param {string} content - Content to hash
+     * @returns {string} - SHA256 hash
+     */
+    hash(content: any): string;
+    /**
+     * Fast hash for caching (non-cryptographic)
+     * @param {string} content - Content to hash
+     * @returns {string} - Simple hash
+     */
+    fastHash(content: any): string;
+}

package/lib/scrubber/utils/hash.js

@@ -1,39 +1,37 @@
+// @ts-nocheck
 /**
  * Content Hashing Utilities
  * @module smora/scrubber/utils/hash
  */
 import crypto from 'crypto';
-
 export class HashUtil {
-  /**
-   * Hash content for deduplication
-   * @param {string} content - Content to hash
-   * @returns {string} - SHA256 hash
-   */
-  hash(content) {
-    const normalized = content
-      .toLowerCase()
-      .trim()
-      .replace(/\s+/g, ' ');
-
-    return crypto
-      .createHash('sha256')
-      .update(normalized)
-      .digest('hex');
-  }
-
-  /**
-   * Fast hash for caching (non-cryptographic)
-   * @param {string} content - Content to hash
-   * @returns {string} - Simple hash
-   */
-  fastHash(content) {
-    let hash = 0;
-    for (let i = 0; i < content.length; i++) {
-      const char = content.charCodeAt(i);
-      hash = ((hash << 5) - hash) + char;
-      hash = hash & hash;
+    /**
+     * Hash content for deduplication
+     * @param {string} content - Content to hash
+     * @returns {string} - SHA256 hash
+     */
+    hash(content) {
+        const normalized = content
+            .toLowerCase()
+            .trim()
+            .replace(/\s+/g, ' ');
+        return crypto
+            .createHash('sha256')
+            .update(normalized)
+            .digest('hex');
     }
-    return hash.toString(36);
-  }
-}
+    /**
+     * Fast hash for caching (non-cryptographic)
+     * @param {string} content - Content to hash
+     * @returns {string} - Simple hash
+     */
+    fastHash(content) {
+        let hash = 0;
+        for (let i = 0; i < content.length; i++) {
+            const char = content.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash;
+        }
+        return hash.toString(36);
+    }
+}

package/lib/scrubber/utils/hash.ts

@@ -0,0 +1,40 @@
+// @ts-nocheck
+/**
+ * Content Hashing Utilities
+ * @module smora/scrubber/utils/hash
+ */
+import crypto from 'crypto';
+
+export class HashUtil {
+  /**
+   * Hash content for deduplication
+   * @param {string} content - Content to hash
+   * @returns {string} - SHA256 hash
+   */
+  hash(content) {
+    const normalized = content
+      .toLowerCase()
+      .trim()
+      .replace(/\s+/g, ' ');
+
+    return crypto
+      .createHash('sha256')
+      .update(normalized)
+      .digest('hex');
+  }
+
+  /**
+   * Fast hash for caching (non-cryptographic)
+   * @param {string} content - Content to hash
+   * @returns {string} - Simple hash
+   */
+  fastHash(content) {
+    let hash = 0;
+    for (let i = 0; i < content.length; i++) {
+      const char = content.charCodeAt(i);
+      hash = ((hash << 5) - hash) + char;
+      hash = hash & hash;
+    }
+    return hash.toString(36);
+  }
+}

package/lib/scrubber/utils/html-parser.d.ts

@@ -0,0 +1,14 @@
+/**
+ * HTML Parsing Utilities
+ * @module smora/scrubber/utils/html-parser
+ */
+export declare class HTMLParser {
+    /**
+     * Extract text content from HTML
+     * @param {string} html - HTML content
+     * @returns {string} - Extracted text
+     */
+    parse(html: any): any;
+    _extractText(html: any): any;
+    _stripTags(html: any): any;
+}

package/lib/scrubber/utils/html-parser.js

@@ -1,45 +1,38 @@
+// @ts-nocheck
 /**
  * HTML Parsing Utilities
  * @module smora/scrubber/utils/html-parser
  */
-
 export class HTMLParser {
-  /**
-   * Extract text content from HTML
-   * @param {string} html - HTML content
-   * @returns {string} - Extracted text
-   */
-  parse(html) {
-    return this._extractText(html);
-  }
-
-  _extractText(html) {
-    // Remove scripts, styles, and comments
-    let text = html;
-    text = text.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
-    text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
-    text = text.replace(/<!--[\s\S]*?-->/g, '');
-
-    // Convert headings to markdown
-    text = text.replace(/<h([1-6])([^>]*)>(.*?)<\/h\1>/gi, (match, level, attrs, content) => {
-      const headingLevel = parseInt(level);
-      const hashes = '#'.repeat(headingLevel);
-      return `${hashes} ${this._stripTags(content)}\n\n`;
-    });
-
-    // Convert paragraphs
-    text = text.replace(/<p[^>]*>(.*?)<\/p>/gi, '$1\n\n');
-
-    // Convert lists
-    text = text.replace(/<li[^>]*>(.*?)<\/li>/gi, '- $1\n');
-
-    // Remove remaining tags
-    text = text.replace(/<[^>]+>/g, '');
-
-    return text;
-  }
-
-  _stripTags(html) {
-    return html.replace(/<[^>]+>/g, '');
-  }
+    /**
+     * Extract text content from HTML
+     * @param {string} html - HTML content
+     * @returns {string} - Extracted text
+     */
+    parse(html) {
+        return this._extractText(html);
+    }
+    _extractText(html) {
+        // Remove scripts, styles, and comments
+        let text = html;
+        text = text.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+        text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
+        text = text.replace(/<!--[\s\S]*?-->/g, '');
+        // Convert headings to markdown
+        text = text.replace(/<h([1-6])([^>]*)>(.*?)<\/h\1>/gi, (match, level, attrs, content) => {
+            const headingLevel = parseInt(level);
+            const hashes = '#'.repeat(headingLevel);
+            return `${hashes} ${this._stripTags(content)}\n\n`;
+        });
+        // Convert paragraphs
+        text = text.replace(/<p[^>]*>(.*?)<\/p>/gi, '$1\n\n');
+        // Convert lists
+        text = text.replace(/<li[^>]*>(.*?)<\/li>/gi, '- $1\n');
+        // Remove remaining tags
+        text = text.replace(/<[^>]+>/g, '');
+        return text;
+    }
+    _stripTags(html) {
+        return html.replace(/<[^>]+>/g, '');
+    }
 }

package/lib/scrubber/utils/html-parser.ts

@@ -0,0 +1,46 @@
+// @ts-nocheck
+/**
+ * HTML Parsing Utilities
+ * @module smora/scrubber/utils/html-parser
+ */
+
+export class HTMLParser {
+  /**
+   * Extract text content from HTML
+   * @param {string} html - HTML content
+   * @returns {string} - Extracted text
+   */
+  parse(html) {
+    return this._extractText(html);
+  }
+
+  _extractText(html) {
+    // Remove scripts, styles, and comments
+    let text = html;
+    text = text.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+    text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
+    text = text.replace(/<!--[\s\S]*?-->/g, '');
+
+    // Convert headings to markdown
+    text = text.replace(/<h([1-6])([^>]*)>(.*?)<\/h\1>/gi, (match, level, attrs, content) => {
+      const headingLevel = parseInt(level);
+      const hashes = '#'.repeat(headingLevel);
+      return `${hashes} ${this._stripTags(content)}\n\n`;
+    });
+
+    // Convert paragraphs
+    text = text.replace(/<p[^>]*>(.*?)<\/p>/gi, '$1\n\n');
+
+    // Convert lists
+    text = text.replace(/<li[^>]*>(.*?)<\/li>/gi, '- $1\n');
+
+    // Remove remaining tags
+    text = text.replace(/<[^>]+>/g, '');
+
+    return text;
+  }
+
+  _stripTags(html) {
+    return html.replace(/<[^>]+>/g, '');
+  }
+}

package/lib/scrubber/utils/pattern-matcher.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Boilerplate Pattern Matching Utilities
+ * @module smora/scrubber/utils/pattern-matcher
+ */
+export declare class PatternMatcher {
+    constructor();
+    _loadDefaultPatterns(): (string | RegExp)[];
+    getBoilerplatePatterns(): any;
+    addPattern(pattern: any): void;
+    removePattern(index: any): void;
+    isBoilerplate(text: any): any;
+}