@yamo/memory-mesh 3.0.0 → 3.0.1

package/lib/memory/search/keyword-search.ts

@@ -1,174 +1,136 @@
+// @ts-nocheck
 /**
  * Simple Keyword Search Engine (In-Memory)
  * Provides basic TF-IDF style retrieval to complement vector search
  */
-
-export interface KeywordDoc {
-  content: string;
-  metadata?: any;
-}
-
-export interface KeywordSearchResult extends KeywordDoc {
-  id: string;
-  score: number;
-  matches: string[];
-}
-
-export interface SearchOptions {
-  limit?: number;
-}
-
 export class KeywordSearch {
-  index: Map<string, Map<string, number>>; // token -> Map<docId, tf>
-  docLengths: Map<string, number>; // docId -> length
-  idf: Map<string, number>; // token -> idf value
-  docs: Map<string, KeywordDoc>; // docId -> content (optional, for snippet)
-  isDirty: boolean;
-
-  constructor() {
-    this.index = new Map();
-    this.docLengths = new Map();
-    this.idf = new Map();
-    this.docs = new Map();
-    this.isDirty = false;
-  }
-
-  /**
-   * Tokenize text into normalized terms
-   * @param {string} text
-   * @returns {string[]} tokens
-   */
-  tokenize(text: string): string[] {
-    if (!text) {
-      return [];
+    index; // token -> Map<docId, tf>
+    docLengths; // docId -> length
+    idf; // token -> idf value
+    docs; // docId -> content (optional, for snippet)
+    isDirty;
+    constructor() {
+        this.index = new Map();
+        this.docLengths = new Map();
+        this.idf = new Map();
+        this.docs = new Map();
+        this.isDirty = false;
     }
-    return text
-      .toLowerCase()
-      .replace(/[^\w\s]/g, "") // Remove punctuation
-      .split(/\s+/)
-      .filter((t) => t.length > 2) // Filter stopwords/short
-      .map((t) => t.substring(0, 20)); // Truncate
-  }
-
-  /**
-   * Add a document to the index
-   * @param {string} id
-   * @param {string} content
-   * @param {Object} [metadata]
-   */
-  add(id: string, content: string, metadata: any = {}): void {
-    const tokens = this.tokenize(content);
-    const termFreqs = new Map<string, number>();
-
-    tokens.forEach((t) => {
-      termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
-    });
-
-    this.docLengths.set(id, tokens.length);
-    this.docs.set(id, { content, metadata });
-
-    // Update index
-    for (const [token, freq] of termFreqs.entries()) {
-      if (!this.index.has(token)) {
-        this.index.set(token, new Map());
-      }
-      this.index.get(token)!.set(id, freq);
+    /**
+     * Tokenize text into normalized terms
+     * @param {string} text
+     * @returns {string[]} tokens
+     */
+    tokenize(text) {
+        if (!text) {
+            return [];
+        }
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, "") // Remove punctuation
+            .split(/\s+/)
+            .filter((t) => t.length > 2) // Filter stopwords/short
+            .map((t) => t.substring(0, 20)); // Truncate
     }
-
-    this.isDirty = true;
-  }
-
-  /**
-   * Remove a document
-   * @param {string} id
-   */
-  remove(id: string): void {
-    this.docLengths.delete(id);
-    this.docs.delete(id);
-
-    // This is expensive O(Vocab), but okay for small scale
-    for (const docMap of this.index.values()) {
-      docMap.delete(id);
+    /**
+     * Add a document to the index
+     * @param {string} id
+     * @param {string} content
+     * @param {Object} [metadata]
+     */
+    add(id, content, metadata = {}) {
+        const tokens = this.tokenize(content);
+        const termFreqs = new Map();
+        tokens.forEach((t) => {
+            termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
+        });
+        this.docLengths.set(id, tokens.length);
+        this.docs.set(id, { content, metadata });
+        // Update index
+        for (const [token, freq] of termFreqs.entries()) {
+            if (!this.index.has(token)) {
+                this.index.set(token, new Map());
+            }
+            this.index.get(token).set(id, freq);
+        }
+        this.isDirty = true;
     }
-    this.isDirty = true;
-  }
-
-  /**
-   * Recalculate IDF scores
-   */
-  _computeStats(): void {
-    if (!this.isDirty) {
-      return;
+    /**
+     * Remove a document
+     * @param {string} id
+     */
+    remove(id) {
+        this.docLengths.delete(id);
+        this.docs.delete(id);
+        // This is expensive O(Vocab), but okay for small scale
+        for (const docMap of this.index.values()) {
+            docMap.delete(id);
+        }
+        this.isDirty = true;
     }
-
-    const N = this.docLengths.size;
-    this.idf.clear();
-
-    for (const [token, docMap] of this.index.entries()) {
-      const df = docMap.size;
-      // Standard IDF: log(N / (df + 1)) + 1
-      const idf = Math.log(N / (df + 1)) + 1;
-      this.idf.set(token, idf);
+    /**
+     * Recalculate IDF scores
+     */
+    _computeStats() {
+        if (!this.isDirty) {
+            return;
+        }
+        const N = this.docLengths.size;
+        this.idf.clear();
+        for (const [token, docMap] of this.index.entries()) {
+            const df = docMap.size;
+            // Standard IDF: log(N / (df + 1)) + 1
+            const idf = Math.log(N / (df + 1)) + 1;
+            this.idf.set(token, idf);
+        }
+        this.isDirty = false;
     }
-
-    this.isDirty = false;
-  }
-
-  /**
-   * Search for query terms
-   * @param {string} query
-   * @param {Object} options
-   * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
-   */
-  search(query: string, options: SearchOptions = {}): KeywordSearchResult[] {
-    this._computeStats();
-
-    const tokens = this.tokenize(query);
-    const scores = new Map<string, number>(); // docId -> score
-    const matches = new Map<string, string[]>(); // docId -> matched tokens
-
-    const limit = options.limit || 10;
-
-    for (const token of tokens) {
-      const docMap = this.index.get(token);
-      if (!docMap) {
-        continue;
-      }
-
-      const idf = this.idf.get(token) || 0;
-
-      for (const [docId, tf] of docMap.entries()) {
-        // TF-IDF Score
-        // Score = tf * idf * (normalization?)
-        // Simple variant:
-        const score = tf * idf;
-
-        scores.set(docId, (scores.get(docId) || 0) + score);
-
-        if (!matches.has(docId)) {
-          matches.set(docId, []);
+    /**
+     * Search for query terms
+     * @param {string} query
+     * @param {Object} options
+     * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
+     */
+    search(query, options = {}) {
+        this._computeStats();
+        const tokens = this.tokenize(query);
+        const scores = new Map(); // docId -> score
+        const matches = new Map(); // docId -> matched tokens
+        const limit = options.limit || 10;
+        for (const token of tokens) {
+            const docMap = this.index.get(token);
+            if (!docMap) {
+                continue;
+            }
+            const idf = this.idf.get(token) || 0;
+            for (const [docId, tf] of docMap.entries()) {
+                // TF-IDF Score
+                // Score = tf * idf * (normalization?)
+                // Simple variant:
+                const score = tf * idf;
+                scores.set(docId, (scores.get(docId) || 0) + score);
+                if (!matches.has(docId)) {
+                    matches.set(docId, []);
+                }
+                matches.get(docId).push(token);
+            }
         }
-        matches.get(docId)!.push(token);
-      }
+        // Convert to array and sort
+        return Array.from(scores.entries())
+            .map(([id, score]) => ({
+            id,
+            score,
+            matches: matches.get(id) || [],
+            ...this.docs.get(id),
+        }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    /**
+     * Bulk load records
+     * @param {Array} records
+     */
+    load(records) {
+        records.forEach((r) => this.add(r.id, r.content, r.metadata));
     }
-
-    // Convert to array and sort
-    return Array.from(scores.entries())
-      .map(([id, score]) => ({
-        id,
-        score,
-        matches: matches.get(id) || [],
-        ...this.docs.get(id)!,
-      }))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, limit);
-  }
-
-  /**
-   * Bulk load records
-   * @param {Array} records
-   */
-  load(records: { id: string; content: string; metadata?: any }[]): void {
-    records.forEach((r) => this.add(r.id, r.content, r.metadata));
-  }
 }

package/lib/scrubber/config/defaults.d.ts

@@ -2,52 +2,45 @@
  * S-MORA Layer 0 Scrubber Default Configuration
  * @module smora/scrubber/config/defaults
  */
-export interface StructuralConfig {
-    stripHTML: boolean;
-    normalizeMarkdown: boolean;
-    collapseWhitespace: boolean;
-    removeScripts: boolean;
-    removeStyles: boolean;
-}
-export interface SemanticConfig {
-    removeDuplicates: boolean;
-    removeBoilerplate: boolean;
-    minSignalRatio: number;
-    boilerplatePatterns: string;
-}
-export interface NormalizationConfig {
-    normalizeHeadings: boolean;
-    normalizeLists: boolean;
-    normalizePunctuation: boolean;
-}
-export interface ChunkingConfig {
-    maxTokens: number;
-    minTokens: number;
-    hardMaxTokens: number;
-    splitOnHeadings: boolean;
-    preserveContext: boolean;
-}
-export interface MetadataConfig {
-    addSource: boolean;
-    addSection: boolean;
-    addHeadingPath: boolean;
-    addTimestamp: boolean;
-    addHash: boolean;
-}
-export interface ValidationConfig {
-    enforceMinLength: boolean;
-    enforceMaxLength: boolean;
-    rejectEmptyChunks: boolean;
-}
-export interface ScrubberConfig {
+export declare const defaultScrubberConfig: {
     enabled: boolean;
-    structural: StructuralConfig;
-    semantic: SemanticConfig;
-    normalization: NormalizationConfig;
-    chunking: ChunkingConfig;
-    metadata: MetadataConfig;
-    validation: ValidationConfig;
+    structural: {
+        stripHTML: boolean;
+        normalizeMarkdown: boolean;
+        collapseWhitespace: boolean;
+        removeScripts: boolean;
+        removeStyles: boolean;
+    };
+    semantic: {
+        removeDuplicates: boolean;
+        removeBoilerplate: boolean;
+        minSignalRatio: number;
+        boilerplatePatterns: string;
+    };
+    normalization: {
+        normalizeHeadings: boolean;
+        normalizeLists: boolean;
+        normalizePunctuation: boolean;
+    };
+    chunking: {
+        maxTokens: number;
+        minTokens: number;
+        hardMaxTokens: number;
+        splitOnHeadings: boolean;
+        preserveContext: boolean;
+    };
+    metadata: {
+        addSource: boolean;
+        addSection: boolean;
+        addHeadingPath: boolean;
+        addTimestamp: boolean;
+        addHash: boolean;
+    };
+    validation: {
+        enforceMinLength: boolean;
+        enforceMaxLength: boolean;
+        rejectEmptyChunks: boolean;
+    };
     logTransformations: boolean;
     cachePatterns: boolean;
-}
-export declare const defaultScrubberConfig: ScrubberConfig;
+};

package/lib/scrubber/config/defaults.js

@@ -1,3 +1,4 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber Default Configuration
  * @module smora/scrubber/config/defaults

package/lib/scrubber/config/defaults.ts

@@ -1,117 +1,55 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber Default Configuration
  * @module smora/scrubber/config/defaults
  */
-
-export interface StructuralConfig {
-  stripHTML: boolean;
-  normalizeMarkdown: boolean;
-  collapseWhitespace: boolean;
-  removeScripts: boolean;
-  removeStyles: boolean;
-}
-
-export interface SemanticConfig {
-  removeDuplicates: boolean;
-  removeBoilerplate: boolean;
-  minSignalRatio: number;
-  boilerplatePatterns: string;
-}
-
-export interface NormalizationConfig {
-  normalizeHeadings: boolean;
-  normalizeLists: boolean;
-  normalizePunctuation: boolean;
-}
-
-export interface ChunkingConfig {
-  maxTokens: number;
-  minTokens: number;
-  hardMaxTokens: number;
-  splitOnHeadings: boolean;
-  preserveContext: boolean;
-}
-
-export interface MetadataConfig {
-  addSource: boolean;
-  addSection: boolean;
-  addHeadingPath: boolean;
-  addTimestamp: boolean;
-  addHash: boolean;
-}
-
-export interface ValidationConfig {
-  enforceMinLength: boolean;
-  enforceMaxLength: boolean;
-  rejectEmptyChunks: boolean;
-}
-
-export interface ScrubberConfig {
-  enabled: boolean;
-  structural: StructuralConfig;
-  semantic: SemanticConfig;
-  normalization: NormalizationConfig;
-  chunking: ChunkingConfig;
-  metadata: MetadataConfig;
-  validation: ValidationConfig;
-  logTransformations: boolean;
-  cachePatterns: boolean;
-}
-
-export const defaultScrubberConfig: ScrubberConfig = {
-  // Master switch - enabled by default for security (PII/sensitive data protection)
-  enabled: true,
-
-  // Stage 1: Structural Cleaning
-  structural: {
-    stripHTML: true,
-    normalizeMarkdown: true,
-    collapseWhitespace: true,
-    removeScripts: true,
-    removeStyles: true,
-  },
-
-  // Stage 2: Semantic Filtering
-  semantic: {
-    removeDuplicates: true,
-    removeBoilerplate: true,
-    minSignalRatio: 0.3,
-    boilerplatePatterns: "default",
-  },
-
-  // Stage 3: Normalization
-  normalization: {
-    normalizeHeadings: true,
-    normalizeLists: true,
-    normalizePunctuation: true,
-  },
-
-  // Stage 4: Chunking
-  chunking: {
-    maxTokens: 500,
-    minTokens: 10,
-    hardMaxTokens: 2000,
-    splitOnHeadings: true,
-    preserveContext: true,
-  },
-
-  // Stage 5: Metadata Annotation
-  metadata: {
-    addSource: true,
-    addSection: true,
-    addHeadingPath: true,
-    addTimestamp: true,
-    addHash: true,
-  },
-
-  // Stage 6: Validation
-  validation: {
-    enforceMinLength: true,
-    enforceMaxLength: true,
-    rejectEmptyChunks: true,
-  },
-
-  // Performance
-  logTransformations: false,
-  cachePatterns: true,
+export const defaultScrubberConfig = {
+    // Master switch - enabled by default for security (PII/sensitive data protection)
+    enabled: true,
+    // Stage 1: Structural Cleaning
+    structural: {
+        stripHTML: true,
+        normalizeMarkdown: true,
+        collapseWhitespace: true,
+        removeScripts: true,
+        removeStyles: true,
+    },
+    // Stage 2: Semantic Filtering
+    semantic: {
+        removeDuplicates: true,
+        removeBoilerplate: true,
+        minSignalRatio: 0.3,
+        boilerplatePatterns: "default",
+    },
+    // Stage 3: Normalization
+    normalization: {
+        normalizeHeadings: true,
+        normalizeLists: true,
+        normalizePunctuation: true,
+    },
+    // Stage 4: Chunking
+    chunking: {
+        maxTokens: 500,
+        minTokens: 10,
+        hardMaxTokens: 2000,
+        splitOnHeadings: true,
+        preserveContext: true,
+    },
+    // Stage 5: Metadata Annotation
+    metadata: {
+        addSource: true,
+        addSection: true,
+        addHeadingPath: true,
+        addTimestamp: true,
+        addHash: true,
+    },
+    // Stage 6: Validation
+    validation: {
+        enforceMinLength: true,
+        enforceMaxLength: true,
+        rejectEmptyChunks: true,
+    },
+    // Performance
+    logTransformations: false,
+    cachePatterns: true,
 };

package/lib/scrubber/errors/scrubber-error.d.ts

@@ -0,0 +1,22 @@
+/**
+ * S-MORA Layer 0 Scrubber Error Classes
+ * @module smora/scrubber/errors/scrubber-error
+ */
+export declare class ScrubberError extends Error {
+    constructor(message: any, details?: {});
+    toJSON(): {
+        name: string;
+        message: string;
+        details: any;
+        timestamp: any;
+    };
+}
+export declare class StructuralCleaningError extends ScrubberError {
+    constructor(message: any, details?: {});
+}
+export declare class ChunkingError extends ScrubberError {
+    constructor(message: any, details?: {});
+}
+export declare class ValidationError extends ScrubberError {
+    constructor(message: any, details?: {});
+}

package/lib/scrubber/errors/scrubber-error.js

@@ -0,0 +1,39 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber Error Classes
+ * @module smora/scrubber/errors/scrubber-error
+ */
+export class ScrubberError extends Error {
+    constructor(message, details = {}) {
+        super(message);
+        this.name = 'ScrubberError';
+        this.details = details;
+        this.timestamp = new Date().toISOString();
+    }
+    toJSON() {
+        return {
+            name: this.name,
+            message: this.message,
+            details: this.details,
+            timestamp: this.timestamp
+        };
+    }
+}
+export class StructuralCleaningError extends ScrubberError {
+    constructor(message, details = {}) {
+        super(message, details);
+        this.name = 'StructuralCleaningError';
+    }
+}
+export class ChunkingError extends ScrubberError {
+    constructor(message, details = {}) {
+        super(message, details);
+        this.name = 'ChunkingError';
+    }
+}
+export class ValidationError extends ScrubberError {
+    constructor(message, details = {}) {
+        super(message, details);
+        this.name = 'ValidationError';
+    }
+}

package/lib/scrubber/errors/scrubber-error.ts

@@ -0,0 +1,44 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber Error Classes
+ * @module smora/scrubber/errors/scrubber-error
+ */
+
+export class ScrubberError extends Error {
+  constructor(message, details = {}) {
+    super(message);
+    this.name = 'ScrubberError';
+    this.details = details;
+    this.timestamp = new Date().toISOString();
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+      message: this.message,
+      details: this.details,
+      timestamp: this.timestamp
+    };
+  }
+}
+
+export class StructuralCleaningError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'StructuralCleaningError';
+  }
+}
+
+export class ChunkingError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'ChunkingError';
+  }
+}
+
+export class ValidationError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'ValidationError';
+  }
+}

package/lib/scrubber/index.d.ts

@@ -3,4 +3,3 @@
  * PII and sensitive data sanitization
  */
 export { Scrubber } from "./scrubber.js";
-export { ScrubberConfig } from "./config/defaults.js";

package/lib/scrubber/index.js

@@ -1,3 +1,4 @@
+// @ts-nocheck
 /**
  * YAMO Scrubber Module
  * PII and sensitive data sanitization