@yamo/memory-mesh 2.0.1

package/lib/scrubber/stages/metadata-annotator.js

@@ -0,0 +1,74 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 5: Metadata Annotation
+ * @module smora/scrubber/stages/metadata-annotator
+ */
+
+import { HashUtil } from '../utils/hash.js';
+
+export class MetadataAnnotator {
+  constructor(config) {
+    this.config = config;
+    this.hashUtil = new HashUtil();
+  }
+
+  /**
+   * Add metadata to chunks
+   * @param {Array} chunks - Array of chunks
+   * @param {Object} document - Original document metadata
+   * @returns {Promise<Array>} - Annotated chunks
+   */
+  async annotate(chunks, document) {
+    const headingPath = [];
+
+    return chunks.map((chunk, index) => {
+      const metadata = {
+        ...chunk.metadata,
+        source: this.config.addSource ? document.source : undefined,
+        doc_type: this.config.addSource ? document.type : undefined,
+        section: this.config.addSection ? this._extractSection(chunk) : undefined,
+        heading_path: this.config.addHeadingPath ?
+          this._buildHeadingPath(chunk, headingPath) :
+          undefined,
+        ingestion_timestamp: this.config.addTimestamp ?
+          new Date().toISOString() :
+          undefined,
+        hash: this.config.addHash ?
+          this.hashUtil.hash(chunk.text) :
+          undefined
+      };
+
+      return {
+        ...chunk,
+        metadata: Object.fromEntries(
+          Object.entries(metadata).filter(([_, v]) => v !== undefined)
+        )
+      };
+    });
+  }
+
+  _extractSection(chunk) {
+    if (chunk.metadata.heading) {
+      return chunk.metadata.heading;
+    }
+    return 'unnamed-section';
+  }
+
+  _buildHeadingPath(chunk, currentPath) {
+    const heading = chunk.metadata.heading;
+
+    if (heading && heading !== currentPath[currentPath.length - 1]) {
+      if (currentPath.length === 0 || this._isSubHeading(heading, currentPath[currentPath.length - 1])) {
+        currentPath.push(heading);
+      } else {
+        currentPath.length = 0;
+        currentPath.push(heading);
+      }
+    }
+
+    return [...currentPath];
+  }
+
+  _isSubHeading(heading1, heading2) {
+    return heading1.length > heading2.length;
+  }
+}

package/lib/scrubber/stages/normalizer.js

@@ -0,0 +1,59 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 3: Normalization
+ * @module smora/scrubber/stages/normalizer
+ */
+
+export class Normalizer {
+  constructor(config) {
+    this.config = config;
+  }
+
+  /**
+   * Normalize content structure
+   * @param {string} content - Filtered content
+   * @returns {Promise<string>} - Normalized content
+   */
+  async normalize(content) {
+    let normalized = content;
+
+    if (this.config.normalizeHeadings) {
+      normalized = this._normalizeHeadings(normalized);
+    }
+
+    if (this.config.normalizeLists) {
+      normalized = this._normalizeLists(normalized);
+    }
+
+    if (this.config.normalizePunctuation) {
+      normalized = this._normalizePunctuation(normalized);
+    }
+
+    return normalized;
+  }
+
+  _normalizeHeadings(content) {
+    let normalized = content.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+    normalized = normalized.replace(/^\s*(#{1,6})/gm, '$1');
+    normalized = normalized.replace(/#{7,}/g, '######');
+    return normalized;
+  }
+
+  _normalizeLists(content) {
+    let normalized = content.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+    normalized = normalized.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+        return `${ws}${num}. ${char}`;
+      }
+      return match;
+    });
+    return normalized;
+  }
+
+  _normalizePunctuation(content) {
+    // Remove quotes (both straight and curly)
+    let normalized = content.replace(/["'""''`]/g, '');
+    normalized = normalized.replace(/ +/g, ' ');
+    normalized = normalized.replace(/\.{4,}/g, '...');
+    return normalized;
+  }
+}

package/lib/scrubber/stages/semantic-filter.js

@@ -0,0 +1,61 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 2: Semantic Filtering
+ * @module smora/scrubber/stages/semantic-filter
+ */
+
+import { PatternMatcher } from '../utils/pattern-matcher.js';
+import { HashUtil } from '../utils/hash.js';
+
+export class SemanticFilter {
+  constructor(config) {
+    this.config = config;
+    this.patternMatcher = new PatternMatcher();
+    this.hashUtil = new HashUtil();
+  }
+
+  /**
+   * Filter semantically empty content
+   * @param {string} content - Cleaned content
+   * @returns {Promise<string>} - Filtered content
+   */
+  async filter(content) {
+    const paragraphs = content.split(/\n\n+/);
+
+    let filtered = paragraphs.filter(p => !this._isBoilerplate(p));
+    filtered = await this._removeDuplicates(filtered);
+    filtered = filtered.filter(p => this._hasSignal(p));
+
+    return filtered.join('\n\n');
+  }
+
+  _isBoilerplate(paragraph) {
+    return this.patternMatcher.isBoilerplate(paragraph);
+  }
+
+  async _removeDuplicates(paragraphs) {
+    if (!this.config.removeDuplicates) return paragraphs;
+
+    const seen = new Set();
+    const unique = [];
+
+    for (const para of paragraphs) {
+      const hash = this.hashUtil.hash(para);
+      if (!seen.has(hash)) {
+        seen.add(hash);
+        unique.push(para);
+      }
+    }
+
+    return unique;
+  }
+
+  _hasSignal(paragraph) {
+    const text = paragraph.trim();
+    if (text.length < 10) return false;
+
+    const signalChars = text.replace(/[^a-zA-Z0-9]/g, '').length;
+    const ratio = signalChars / text.length;
+
+    return ratio >= (this.config.minSignalRatio || 0.3);
+  }
+}

package/lib/scrubber/stages/structural-cleaner.js

@@ -0,0 +1,82 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 1: Structural Cleaning
+ * @module smora/scrubber/stages/structural-cleaner
+ */
+
+import { HTMLParser } from '../utils/html-parser.js';
+import { StructuralCleaningError, ScrubberError } from '../errors/scrubber-error.js';
+
+export class StructuralCleaner {
+  constructor(config) {
+    this.config = config;
+    this.htmlParser = new HTMLParser();
+  }
+
+  /**
+   * Clean document structure
+   * @param {string} content - Raw document content
+   * @returns {Promise<string>} - Cleaned content
+   */
+  async clean(content) {
+    try {
+      const type = this._detectType(content);
+      let cleaned = content;
+
+      if (type === 'html') {
+        cleaned = await this._cleanHTML(cleaned);
+        // HTML may have markdown headings, normalize them
+        cleaned = await this._cleanMarkdown(cleaned);
+      } else if (type === 'markdown') {
+        cleaned = await this._cleanMarkdown(cleaned);
+      }
+
+      cleaned = this._collapseWhitespace(cleaned);
+      cleaned = this._normalizeLineBreaks(cleaned);
+
+      return cleaned;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ScrubberError(
+        `Failed to clean content: ${message}`,
+        { stage: 'structural-cleaner', originalError: error }
+      );
+    }
+  }
+
+  _detectType(content) {
+    if (content.trim().startsWith('<')) return 'html';
+    if (/^#{1,6}\s/.test(content) || /^#{1,6}[A-Za-z]/.test(content)) return 'markdown';
+    return 'text';
+  }
+
+  async _cleanHTML(content) {
+    return this.htmlParser.parse(content);
+  }
+
+  async _cleanMarkdown(content) {
+    let cleaned = content;
+    // Add space after heading markers when missing
+    cleaned = cleaned.replace(/(#{1,6})([^\s#])/g, '$1 $2');
+    // Add space after list markers when missing
+    cleaned = cleaned.replace(/(\s*)([-*+])(\S)/g, '$1$2 $3');
+    // Add space after numbered list markers when missing
+    cleaned = cleaned.replace(/(\s*)(\d+)(\S)/g, (match, ws, num, char) => {
+      // Only if it looks like a numbered list (digit followed by non-dot, non-space)
+      if (!/\.\s/.test(match.substring(ws.length + num.length))) {
+        return `${ws}${num}. ${char}`;
+      }
+      return match;
+    });
+    return cleaned;
+  }
+
+  _collapseWhitespace(content) {
+    let cleaned = content.replace(/[ \t]+/g, ' ');
+    cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+    return cleaned;
+  }
+
+  _normalizeLineBreaks(content) {
+    return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+  }
+}

package/lib/scrubber/stages/validator.js

@@ -0,0 +1,66 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 6: Validation
+ * @module smora/scrubber/stages/validator
+ */
+
+import { TokenCounter } from '../utils/token-counter.js';
+import { ValidationError } from '../errors/scrubber-error.js';
+
+export class Validator {
+  constructor(config) {
+    this.config = config;
+    this.tokenCounter = new TokenCounter();
+  }
+
+  /**
+   * Validate chunks
+   * @param {Array} chunks - Array of chunks
+   * @returns {Promise<Array>} - Validated chunks
+   */
+  async validate(chunks) {
+    const valid = [];
+    const errors = [];
+
+    for (const chunk of chunks) {
+      const validation = this._validateChunk(chunk);
+
+      if (validation.valid) {
+        valid.push(chunk);
+      } else {
+        errors.push({
+          chunkIndex: chunk.index,
+          errors: validation.errors
+        });
+      }
+    }
+
+    return valid;
+  }
+
+  _validateChunk(chunk) {
+    const errors = [];
+
+    if (this.config.rejectEmptyChunks && !chunk.text.trim()) {
+      errors.push('empty_chunk');
+    }
+
+    if (this.config.enforceMinLength) {
+      const tokens = this.tokenCounter.count(chunk.text);
+      if (tokens < this.config.minTokens) {
+        errors.push(`chunk_too_short: ${tokens} < ${this.config.minTokens}`);
+      }
+    }
+
+    if (this.config.enforceMaxLength) {
+      const tokens = this.tokenCounter.count(chunk.text);
+      if (tokens > this.config.hardMaxTokens) {
+        errors.push(`chunk_too_long: ${tokens} > ${this.config.hardMaxTokens}`);
+      }
+    }
+
+    return {
+      valid: errors.length === 0,
+      errors
+    };
+  }
+}

package/lib/scrubber/telemetry.js

@@ -0,0 +1,66 @@
+/**
+ * S-MORA Layer 0 Scrubber Telemetry Collection
+ * @module smora/scrubber/telemetry
+ */
+
+export class ScrubberTelemetry {
+  constructor() {
+    this.stats = {
+      structural: { count: 0, totalTime: 0, errors: 0 },
+      semantic: { count: 0, totalTime: 0, errors: 0 },
+      normalization: { count: 0, totalTime: 0, errors: 0 },
+      chunking: { count: 0, totalTime: 0, errors: 0 },
+      metadata: { count: 0, totalTime: 0, errors: 0 },
+      validation: { count: 0, totalTime: 0, errors: 0 }
+    };
+  }
+
+  recordStage(stage, duration, success = true) {
+    if (!this.stats[stage]) {
+      this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+    }
+    this.stats[stage].count++;
+    this.stats[stage].totalTime += duration;
+    if (!success) this.stats[stage].errors++;
+  }
+
+  getStageStats(stage) {
+    const stats = this.stats[stage];
+    return {
+      count: stats.count,
+      avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
+      totalTime: stats.totalTime,
+      errors: stats.errors
+    };
+  }
+
+  getSummary() {
+    return {
+      stages: this.stats,
+      performance: {
+        structural: this.stats.structural.totalTime,
+        semantic: this.stats.semantic.totalTime,
+        normalization: this.stats.normalization.totalTime,
+        chunking: this.stats.chunking.totalTime,
+        metadata: this.stats.metadata.totalTime,
+        validation: this.stats.validation.totalTime,
+        total: Object.values(this.stats).reduce((sum, s) => sum + s.totalTime, 0)
+      }
+    };
+  }
+
+  reset() {
+    Object.keys(this.stats).forEach(key => {
+      this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
+    });
+  }
+
+  assertPerformanceBudget(budget = 10) {
+    const summary = this.getSummary();
+    if (summary.performance.total > budget) {
+      throw new Error(
+        `Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`
+      );
+    }
+  }
+}

package/lib/scrubber/utils/hash.js

@@ -0,0 +1,39 @@
+/**
+ * Content Hashing Utilities
+ * @module smora/scrubber/utils/hash
+ */
+import crypto from 'crypto';
+
+export class HashUtil {
+  /**
+   * Hash content for deduplication
+   * @param {string} content - Content to hash
+   * @returns {string} - SHA256 hash
+   */
+  hash(content) {
+    const normalized = content
+      .toLowerCase()
+      .trim()
+      .replace(/\s+/g, ' ');
+
+    return crypto
+      .createHash('sha256')
+      .update(normalized)
+      .digest('hex');
+  }
+
+  /**
+   * Fast hash for caching (non-cryptographic)
+   * @param {string} content - Content to hash
+   * @returns {string} - Simple hash
+   */
+  fastHash(content) {
+    let hash = 0;
+    for (let i = 0; i < content.length; i++) {
+      const char = content.charCodeAt(i);
+      hash = ((hash << 5) - hash) + char;
+      hash = hash & hash;
+    }
+    return hash.toString(36);
+  }
+}

package/lib/scrubber/utils/html-parser.js

@@ -0,0 +1,45 @@
+/**
+ * HTML Parsing Utilities
+ * @module smora/scrubber/utils/html-parser
+ */
+
+export class HTMLParser {
+  /**
+   * Extract text content from HTML
+   * @param {string} html - HTML content
+   * @returns {string} - Extracted text
+   */
+  parse(html) {
+    return this._extractText(html);
+  }
+
+  _extractText(html) {
+    // Remove scripts, styles, and comments
+    let text = html;
+    text = text.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+    text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
+    text = text.replace(/<!--[\s\S]*?-->/g, '');
+
+    // Convert headings to markdown
+    text = text.replace(/<h([1-6])([^>]*)>(.*?)<\/h\1>/gi, (match, level, attrs, content) => {
+      const headingLevel = parseInt(level);
+      const hashes = '#'.repeat(headingLevel);
+      return `${hashes} ${this._stripTags(content)}\n\n`;
+    });
+
+    // Convert paragraphs
+    text = text.replace(/<p[^>]*>(.*?)<\/p>/gi, '$1\n\n');
+
+    // Convert lists
+    text = text.replace(/<li[^>]*>(.*?)<\/li>/gi, '- $1\n');
+
+    // Remove remaining tags
+    text = text.replace(/<[^>]+>/g, '');
+
+    return text;
+  }
+
+  _stripTags(html) {
+    return html.replace(/<[^>]+>/g, '');
+  }
+}

package/lib/scrubber/utils/pattern-matcher.js

@@ -0,0 +1,63 @@
+/**
+ * Boilerplate Pattern Matching Utilities
+ * @module smora/scrubber/utils/pattern-matcher
+ */
+
+export class PatternMatcher {
+  constructor() {
+    this.boilerplatePatterns = this._loadDefaultPatterns();
+  }
+
+  _loadDefaultPatterns() {
+    return [
+      // Legal/Footer
+      /©\s*\d{4}/i,
+      /all rights reserved/i,
+      /copyright\s+\d{4}/i,
+
+      // Navigation
+      /^home\s*\|/i,
+      /^navigation\s*:|menu\s*:/i,
+      /sidebar/i,
+
+      // Meta
+      /^last\s+updated?\s*:/i,
+      /cookie\s+policy/i,
+      /privacy\s+policy/i,
+
+      // Auto-generated
+      /^table\s+of\s+contents?$/i,
+      /^contents\s*$/i,
+      /jump\s+to\s+(section|navigation)/i,
+
+      // Strings
+      'home | docs | contact',
+      'skip to main content',
+      'this site uses cookies'
+    ];
+  }
+
+  getBoilerplatePatterns() {
+    return this.boilerplatePatterns;
+  }
+
+  addPattern(pattern) {
+    this.boilerplatePatterns.push(pattern);
+  }
+
+  removePattern(index) {
+    if (index >= 0 && index < this.boilerplatePatterns.length) {
+      this.boilerplatePatterns.splice(index, 1);
+    }
+  }
+
+  isBoilerplate(text) {
+    const lowerText = text.toLowerCase().trim();
+    return this.boilerplatePatterns.some(pattern => {
+      if (pattern instanceof RegExp) {
+        return pattern.test(lowerText);
+      }
+      return lowerText.includes(pattern);
+    });
+  }
+}

package/lib/scrubber/utils/token-counter.js

@@ -0,0 +1,31 @@
+/**
+ * Token Counting Utilities
+ * @module smora/scrubber/utils/token-counter
+ */
+
+export class TokenCounter {
+  /**
+   * Estimate token count (approximation)
+   * @param {string} text - Text to count
+   * @returns {number} - Estimated token count
+   */
+  count(text) {
+    // Simple approximation: ~4 characters per token
+    return Math.ceil(text.length / 4);
+  }
+
+  /**
+   * More accurate token count (slower)
+   * @param {string} text - Text to count
+   * @returns {number} - More accurate token count
+   */
+  countAccurate(text) {
+    const words = text.split(/\s+/).filter(w => w.length > 0);
+    let tokens = words.length;
+    const punctuationMatches = text.match(/[.,!?;:]/g);
+    if (punctuationMatches) {
+      tokens += punctuationMatches.length;
+    }
+    return tokens;
+  }
+}