@yamo/memory-mesh 3.0.0 → 3.0.1

package/lib/scrubber/stages/validator.ts

@@ -0,0 +1,67 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 6: Validation
+ * @module smora/scrubber/stages/validator
+ */
+
+import { TokenCounter } from '../utils/token-counter.js';
+import { ValidationError } from '../errors/scrubber-error.js';
+
+export class Validator {
+  constructor(config) {
+    this.config = config;
+    this.tokenCounter = new TokenCounter();
+  }
+
+  /**
+   * Validate chunks
+   * @param {Array} chunks - Array of chunks
+   * @returns {Promise<Array>} - Validated chunks
+   */
+  async validate(chunks) {
+    const valid = [];
+    const errors = [];
+
+    for (const chunk of chunks) {
+      const validation = this._validateChunk(chunk);
+
+      if (validation.valid) {
+        valid.push(chunk);
+      } else {
+        errors.push({
+          chunkIndex: chunk.index,
+          errors: validation.errors
+        });
+      }
+    }
+
+    return valid;
+  }
+
+  _validateChunk(chunk) {
+    const errors = [];
+
+    if (this.config.rejectEmptyChunks && !chunk.text.trim()) {
+      errors.push('empty_chunk');
+    }
+
+    if (this.config.enforceMinLength) {
+      const tokens = this.tokenCounter.count(chunk.text);
+      if (tokens < this.config.minTokens) {
+        errors.push(`chunk_too_short: ${tokens} < ${this.config.minTokens}`);
+      }
+    }
+
+    if (this.config.enforceMaxLength) {
+      const tokens = this.tokenCounter.count(chunk.text);
+      if (tokens > this.config.hardMaxTokens) {
+        errors.push(`chunk_too_long: ${tokens} > ${this.config.hardMaxTokens}`);
+      }
+    }
+
+    return {
+      valid: errors.length === 0,
+      errors
+    };
+  }
+}

package/lib/scrubber/telemetry.d.ts

@@ -2,35 +2,28 @@
  * S-MORA Layer 0 Scrubber Telemetry Collection
  * @module smora/scrubber/telemetry
  */
-export interface StageStats {
-    count: number;
-    totalTime: number;
-    errors: number;
-}
-export interface StageSummary {
-    count: number;
-    avgTime: number;
-    totalTime: number;
-    errors: number;
-}
-export interface TelemetrySummary {
-    stages: Record<string, StageStats>;
-    performance: {
-        structural: number;
-        semantic: number;
-        normalization: number;
-        chunking: number;
-        metadata: number;
-        validation: number;
-        total: number;
-    };
-}
 export declare class ScrubberTelemetry {
-    stats: Record<string, StageStats>;
+    stats: any;
     constructor();
-    recordStage(stage: string, duration: number, success?: boolean): void;
-    getStageStats(stage: string): StageSummary;
-    getSummary(): TelemetrySummary;
+    recordStage(stage: any, duration: any, success?: boolean): void;
+    getStageStats(stage: any): {
+        count: any;
+        avgTime: number;
+        totalTime: any;
+        errors: any;
+    };
+    getSummary(): {
+        stages: any;
+        performance: {
+            structural: any;
+            semantic: any;
+            normalization: any;
+            chunking: any;
+            metadata: any;
+            validation: any;
+            total: unknown;
+        };
+    };
     reset(): void;
     assertPerformanceBudget(budget?: number): void;
 }

package/lib/scrubber/telemetry.js

@@ -1,3 +1,4 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber Telemetry Collection
  * @module smora/scrubber/telemetry

package/lib/scrubber/telemetry.ts

@@ -1,99 +1,62 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber Telemetry Collection
  * @module smora/scrubber/telemetry
  */
-
-export interface StageStats {
-  count: number;
-  totalTime: number;
-  errors: number;
-}
-
-export interface StageSummary {
-  count: number;
-  avgTime: number;
-  totalTime: number;
-  errors: number;
-}
-
-export interface TelemetrySummary {
-  stages: Record<string, StageStats>;
-  performance: {
-    structural: number;
-    semantic: number;
-    normalization: number;
-    chunking: number;
-    metadata: number;
-    validation: number;
-    total: number;
-  };
-}
-
 export class ScrubberTelemetry {
-  stats: Record<string, StageStats>;
-
-  constructor() {
-    this.stats = {
-      structural: { count: 0, totalTime: 0, errors: 0 },
-      semantic: { count: 0, totalTime: 0, errors: 0 },
-      normalization: { count: 0, totalTime: 0, errors: 0 },
-      chunking: { count: 0, totalTime: 0, errors: 0 },
-      metadata: { count: 0, totalTime: 0, errors: 0 },
-      validation: { count: 0, totalTime: 0, errors: 0 },
-    };
-  }
-
-  recordStage(stage: string, duration: number, success: boolean = true): void {
-    if (!this.stats[stage]) {
-      this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+    stats;
+    constructor() {
+        this.stats = {
+            structural: { count: 0, totalTime: 0, errors: 0 },
+            semantic: { count: 0, totalTime: 0, errors: 0 },
+            normalization: { count: 0, totalTime: 0, errors: 0 },
+            chunking: { count: 0, totalTime: 0, errors: 0 },
+            metadata: { count: 0, totalTime: 0, errors: 0 },
+            validation: { count: 0, totalTime: 0, errors: 0 },
+        };
+    }
+    recordStage(stage, duration, success = true) {
+        if (!this.stats[stage]) {
+            this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+        }
+        this.stats[stage].count++;
+        this.stats[stage].totalTime += duration;
+        if (!success) {
+            this.stats[stage].errors++;
+        }
+    }
+    getStageStats(stage) {
+        const stats = this.stats[stage] || { count: 0, totalTime: 0, errors: 0 };
+        return {
+            count: stats.count,
+            avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
+            totalTime: stats.totalTime,
+            errors: stats.errors,
+        };
+    }
+    getSummary() {
+        return {
+            stages: this.stats,
+            performance: {
+                structural: this.stats.structural.totalTime,
+                semantic: this.stats.semantic.totalTime,
+                normalization: this.stats.normalization.totalTime,
+                chunking: this.stats.chunking.totalTime,
+                metadata: this.stats.metadata.totalTime,
+                validation: this.stats.validation.totalTime,
+                total: Object.values(this.stats).reduce((sum, s) => sum + s.totalTime, 0),
+            },
+        };
     }
-    this.stats[stage].count++;
-    this.stats[stage].totalTime += duration;
-    if (!success) {
-      this.stats[stage].errors++;
+    reset() {
+        Object.keys(this.stats).forEach((key) => {
+            this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
+        });
     }
-  }
-
-  getStageStats(stage: string): StageSummary {
-    const stats = this.stats[stage] || { count: 0, totalTime: 0, errors: 0 };
-    return {
-      count: stats.count,
-      avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
-      totalTime: stats.totalTime,
-      errors: stats.errors,
-    };
-  }
-
-  getSummary(): TelemetrySummary {
-    return {
-      stages: this.stats,
-      performance: {
-        structural: this.stats.structural.totalTime,
-        semantic: this.stats.semantic.totalTime,
-        normalization: this.stats.normalization.totalTime,
-        chunking: this.stats.chunking.totalTime,
-        metadata: this.stats.metadata.totalTime,
-        validation: this.stats.validation.totalTime,
-        total: Object.values(this.stats).reduce(
-          (sum, s) => sum + s.totalTime,
-          0,
-        ),
-      },
-    };
-  }
-
-  reset(): void {
-    Object.keys(this.stats).forEach((key) => {
-      this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
-    });
-  }
-
-  assertPerformanceBudget(budget: number = 10): void {
-    const summary = this.getSummary();
-    if (summary.performance.total > budget) {
-      throw new Error(
-        `Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`,
-      );
+    assertPerformanceBudget(budget = 10) {
+        const summary = this.getSummary();
+        if (summary.performance.total > budget) {
+            throw new Error(`Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`);
+        }
     }
-  }
 }

package/lib/scrubber/utils/hash.d.ts

@@ -0,0 +1,14 @@
+export declare class HashUtil {
+    /**
+     * Hash content for deduplication
+     * @param {string} content - Content to hash
+     * @returns {string} - SHA256 hash
+     */
+    hash(content: any): string;
+    /**
+     * Fast hash for caching (non-cryptographic)
+     * @param {string} content - Content to hash
+     * @returns {string} - Simple hash
+     */
+    fastHash(content: any): string;
+}

package/lib/scrubber/utils/hash.js

@@ -0,0 +1,37 @@
+// @ts-nocheck
+/**
+ * Content Hashing Utilities
+ * @module smora/scrubber/utils/hash
+ */
+import crypto from 'crypto';
+export class HashUtil {
+    /**
+     * Hash content for deduplication
+     * @param {string} content - Content to hash
+     * @returns {string} - SHA256 hash
+     */
+    hash(content) {
+        const normalized = content
+            .toLowerCase()
+            .trim()
+            .replace(/\s+/g, ' ');
+        return crypto
+            .createHash('sha256')
+            .update(normalized)
+            .digest('hex');
+    }
+    /**
+     * Fast hash for caching (non-cryptographic)
+     * @param {string} content - Content to hash
+     * @returns {string} - Simple hash
+     */
+    fastHash(content) {
+        let hash = 0;
+        for (let i = 0; i < content.length; i++) {
+            const char = content.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash;
+        }
+        return hash.toString(36);
+    }
+}

package/lib/scrubber/utils/hash.ts

@@ -0,0 +1,40 @@
+// @ts-nocheck
+/**
+ * Content Hashing Utilities
+ * @module smora/scrubber/utils/hash
+ */
+import crypto from 'crypto';
+
+export class HashUtil {
+  /**
+   * Hash content for deduplication
+   * @param {string} content - Content to hash
+   * @returns {string} - SHA256 hash
+   */
+  hash(content) {
+    const normalized = content
+      .toLowerCase()
+      .trim()
+      .replace(/\s+/g, ' ');
+
+    return crypto
+      .createHash('sha256')
+      .update(normalized)
+      .digest('hex');
+  }
+
+  /**
+   * Fast hash for caching (non-cryptographic)
+   * @param {string} content - Content to hash
+   * @returns {string} - Simple hash
+   */
+  fastHash(content) {
+    let hash = 0;
+    for (let i = 0; i < content.length; i++) {
+      const char = content.charCodeAt(i);
+      hash = ((hash << 5) - hash) + char;
+      hash = hash & hash;
+    }
+    return hash.toString(36);
+  }
+}

package/lib/scrubber/utils/html-parser.d.ts

@@ -0,0 +1,14 @@
+/**
+ * HTML Parsing Utilities
+ * @module smora/scrubber/utils/html-parser
+ */
+export declare class HTMLParser {
+    /**
+     * Extract text content from HTML
+     * @param {string} html - HTML content
+     * @returns {string} - Extracted text
+     */
+    parse(html: any): any;
+    _extractText(html: any): any;
+    _stripTags(html: any): any;
+}

package/lib/scrubber/utils/html-parser.js

@@ -0,0 +1,38 @@
+// @ts-nocheck
+/**
+ * HTML Parsing Utilities
+ * @module smora/scrubber/utils/html-parser
+ */
+export class HTMLParser {
+    /**
+     * Extract text content from HTML
+     * @param {string} html - HTML content
+     * @returns {string} - Extracted text
+     */
+    parse(html) {
+        return this._extractText(html);
+    }
+    _extractText(html) {
+        // Remove scripts, styles, and comments
+        let text = html;
+        text = text.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+        text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
+        text = text.replace(/<!--[\s\S]*?-->/g, '');
+        // Convert headings to markdown
+        text = text.replace(/<h([1-6])([^>]*)>(.*?)<\/h\1>/gi, (match, level, attrs, content) => {
+            const headingLevel = parseInt(level);
+            const hashes = '#'.repeat(headingLevel);
+            return `${hashes} ${this._stripTags(content)}\n\n`;
+        });
+        // Convert paragraphs
+        text = text.replace(/<p[^>]*>(.*?)<\/p>/gi, '$1\n\n');
+        // Convert lists
+        text = text.replace(/<li[^>]*>(.*?)<\/li>/gi, '- $1\n');
+        // Remove remaining tags
+        text = text.replace(/<[^>]+>/g, '');
+        return text;
+    }
+    _stripTags(html) {
+        return html.replace(/<[^>]+>/g, '');
+    }
+}

package/lib/scrubber/utils/html-parser.ts

@@ -0,0 +1,46 @@
+// @ts-nocheck
+/**
+ * HTML Parsing Utilities
+ * @module smora/scrubber/utils/html-parser
+ */
+
+export class HTMLParser {
+  /**
+   * Extract text content from HTML
+   * @param {string} html - HTML content
+   * @returns {string} - Extracted text
+   */
+  parse(html) {
+    return this._extractText(html);
+  }
+
+  _extractText(html) {
+    // Remove scripts, styles, and comments
+    let text = html;
+    text = text.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+    text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
+    text = text.replace(/<!--[\s\S]*?-->/g, '');
+
+    // Convert headings to markdown
+    text = text.replace(/<h([1-6])([^>]*)>(.*?)<\/h\1>/gi, (match, level, attrs, content) => {
+      const headingLevel = parseInt(level);
+      const hashes = '#'.repeat(headingLevel);
+      return `${hashes} ${this._stripTags(content)}\n\n`;
+    });
+
+    // Convert paragraphs
+    text = text.replace(/<p[^>]*>(.*?)<\/p>/gi, '$1\n\n');
+
+    // Convert lists
+    text = text.replace(/<li[^>]*>(.*?)<\/li>/gi, '- $1\n');
+
+    // Remove remaining tags
+    text = text.replace(/<[^>]+>/g, '');
+
+    return text;
+  }
+
+  _stripTags(html) {
+    return html.replace(/<[^>]+>/g, '');
+  }
+}

package/lib/scrubber/utils/pattern-matcher.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Boilerplate Pattern Matching Utilities
+ * @module smora/scrubber/utils/pattern-matcher
+ */
+export declare class PatternMatcher {
+    constructor();
+    _loadDefaultPatterns(): (string | RegExp)[];
+    getBoilerplatePatterns(): any;
+    addPattern(pattern: any): void;
+    removePattern(index: any): void;
+    isBoilerplate(text: any): any;
+}

package/lib/scrubber/utils/pattern-matcher.js

@@ -0,0 +1,54 @@
+// @ts-nocheck
+/**
+ * Boilerplate Pattern Matching Utilities
+ * @module smora/scrubber/utils/pattern-matcher
+ */
+export class PatternMatcher {
+    constructor() {
+        this.boilerplatePatterns = this._loadDefaultPatterns();
+    }
+    _loadDefaultPatterns() {
+        return [
+            // Legal/Footer
+            /©\s*\d{4}/i,
+            /all rights reserved/i,
+            /copyright\s+\d{4}/i,
+            // Navigation
+            /^home\s*\|/i,
+            /^navigation\s*:|menu\s*:/i,
+            /sidebar/i,
+            // Meta
+            /^last\s+updated?\s*:/i,
+            /cookie\s+policy/i,
+            /privacy\s+policy/i,
+            // Auto-generated
+            /^table\s+of\s+contents?$/i,
+            /^contents\s*$/i,
+            /jump\s+to\s+(section|navigation)/i,
+            // Strings
+            'home | docs | contact',
+            'skip to main content',
+            'this site uses cookies'
+        ];
+    }
+    getBoilerplatePatterns() {
+        return this.boilerplatePatterns;
+    }
+    addPattern(pattern) {
+        this.boilerplatePatterns.push(pattern);
+    }
+    removePattern(index) {
+        if (index >= 0 && index < this.boilerplatePatterns.length) {
+            this.boilerplatePatterns.splice(index, 1);
+        }
+    }
+    isBoilerplate(text) {
+        const lowerText = text.toLowerCase().trim();
+        return this.boilerplatePatterns.some(pattern => {
+            if (pattern instanceof RegExp) {
+                return pattern.test(lowerText);
+            }
+            return lowerText.includes(pattern);
+        });
+    }
+}

package/lib/scrubber/utils/pattern-matcher.ts

@@ -0,0 +1,64 @@
+// @ts-nocheck
+/**
+ * Boilerplate Pattern Matching Utilities
+ * @module smora/scrubber/utils/pattern-matcher
+ */
+
+export class PatternMatcher {
+  constructor() {
+    this.boilerplatePatterns = this._loadDefaultPatterns();
+  }
+
+  _loadDefaultPatterns() {
+    return [
+      // Legal/Footer
+      /©\s*\d{4}/i,
+      /all rights reserved/i,
+      /copyright\s+\d{4}/i,
+
+      // Navigation
+      /^home\s*\|/i,
+      /^navigation\s*:|menu\s*:/i,
+      /sidebar/i,
+
+      // Meta
+      /^last\s+updated?\s*:/i,
+      /cookie\s+policy/i,
+      /privacy\s+policy/i,
+
+      // Auto-generated
+      /^table\s+of\s+contents?$/i,
+      /^contents\s*$/i,
+      /jump\s+to\s+(section|navigation)/i,
+
+      // Strings
+      'home | docs | contact',
+      'skip to main content',
+      'this site uses cookies'
+    ];
+  }
+
+  getBoilerplatePatterns() {
+    return this.boilerplatePatterns;
+  }
+
+  addPattern(pattern) {
+    this.boilerplatePatterns.push(pattern);
+  }
+
+  removePattern(index) {
+    if (index >= 0 && index < this.boilerplatePatterns.length) {
+      this.boilerplatePatterns.splice(index, 1);
+    }
+  }
+
+  isBoilerplate(text) {
+    const lowerText = text.toLowerCase().trim();
+    return this.boilerplatePatterns.some(pattern => {
+      if (pattern instanceof RegExp) {
+        return pattern.test(lowerText);
+      }
+      return lowerText.includes(pattern);
+    });
+  }
+}

package/lib/scrubber/utils/token-counter.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Token Counting Utilities
+ * @module smora/scrubber/utils/token-counter
+ */
+export declare class TokenCounter {
+    /**
+     * Estimate token count (approximation)
+     * @param {string} text - Text to count
+     * @returns {number} - Estimated token count
+     */
+    count(text: any): number;
+    /**
+     * More accurate token count (slower)
+     * @param {string} text - Text to count
+     * @returns {number} - More accurate token count
+     */
+    countAccurate(text: any): any;
+}

package/lib/scrubber/utils/token-counter.js

@@ -0,0 +1,30 @@
+// @ts-nocheck
+/**
+ * Token Counting Utilities
+ * @module smora/scrubber/utils/token-counter
+ */
+export class TokenCounter {
+    /**
+     * Estimate token count (approximation)
+     * @param {string} text - Text to count
+     * @returns {number} - Estimated token count
+     */
+    count(text) {
+        // Simple approximation: ~4 characters per token
+        return Math.ceil(text.length / 4);
+    }
+    /**
+     * More accurate token count (slower)
+     * @param {string} text - Text to count
+     * @returns {number} - More accurate token count
+     */
+    countAccurate(text) {
+        const words = text.split(/\s+/).filter(w => w.length > 0);
+        let tokens = words.length;
+        const punctuationMatches = text.match(/[.,!?;:]/g);
+        if (punctuationMatches) {
+            tokens += punctuationMatches.length;
+        }
+        return tokens;
+    }
+}