@yamo/memory-mesh 2.3.2 → 3.0.0

package/lib/{search/keyword-search.js → memory/search/keyword-search.ts}

@@ -3,40 +3,64 @@
  * Provides basic TF-IDF style retrieval to complement vector search
  */
 
+export interface KeywordDoc {
+  content: string;
+  metadata?: any;
+}
+
+export interface KeywordSearchResult extends KeywordDoc {
+  id: string;
+  score: number;
+  matches: string[];
+}
+
+export interface SearchOptions {
+  limit?: number;
+}
+
 export class KeywordSearch {
+  index: Map<string, Map<string, number>>; // token -> Map<docId, tf>
+  docLengths: Map<string, number>; // docId -> length
+  idf: Map<string, number>; // token -> idf value
+  docs: Map<string, KeywordDoc>; // docId -> content (optional, for snippet)
+  isDirty: boolean;
+
   constructor() {
-    this.index = new Map(); // token -> Map<docId, tf>
-    this.docLengths = new Map(); // docId -> length
-    this.idf = new Map(); // token -> idf value
-    this.docs = new Map(); // docId -> content (optional, for snippet)
+    this.index = new Map();
+    this.docLengths = new Map();
+    this.idf = new Map();
+    this.docs = new Map();
     this.isDirty = false;
   }
 
   /**
    * Tokenize text into normalized terms
-   * @param {string} text 
+   * @param {string} text
    * @returns {string[]} tokens
    */
-  tokenize(text) {
-    if (!text) return [];
-    return text.toLowerCase()
-      .replace(/[^\w\s]/g, '') // Remove punctuation
+  tokenize(text: string): string[] {
+    if (!text) {
+      return [];
+    }
+    return text
+      .toLowerCase()
+      .replace(/[^\w\s]/g, "") // Remove punctuation
       .split(/\s+/)
-      .filter(t => t.length > 2) // Filter stopwords/short
-      .map(t => t.substring(0, 20)); // Truncate
+      .filter((t) => t.length > 2) // Filter stopwords/short
+      .map((t) => t.substring(0, 20)); // Truncate
   }
 
   /**
    * Add a document to the index
-   * @param {string} id 
-   * @param {string} content 
+   * @param {string} id
+   * @param {string} content
    * @param {Object} [metadata]
    */
-  add(id, content, metadata = {}) {
+  add(id: string, content: string, metadata: any = {}): void {
     const tokens = this.tokenize(content);
-    const termFreqs = new Map();
+    const termFreqs = new Map<string, number>();
 
-    tokens.forEach(t => {
+    tokens.forEach((t) => {
       termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
     });
 
@@ -48,7 +72,7 @@ export class KeywordSearch {
       if (!this.index.has(token)) {
         this.index.set(token, new Map());
       }
-      this.index.get(token).set(id, freq);
+      this.index.get(token)!.set(id, freq);
     }
 
     this.isDirty = true;
@@ -56,12 +80,12 @@ export class KeywordSearch {
 
   /**
    * Remove a document
-   * @param {string} id 
+   * @param {string} id
    */
-  remove(id) {
+  remove(id: string): void {
     this.docLengths.delete(id);
     this.docs.delete(id);
-    
+
     // This is expensive O(Vocab), but okay for small scale
     for (const docMap of this.index.values()) {
       docMap.delete(id);
@@ -72,9 +96,11 @@ export class KeywordSearch {
   /**
    * Recalculate IDF scores
    */
-  _computeStats() {
-    if (!this.isDirty) return;
-    
+  _computeStats(): void {
+    if (!this.isDirty) {
+      return;
+    }
+
     const N = this.docLengths.size;
     this.idf.clear();
 
@@ -94,18 +120,20 @@ export class KeywordSearch {
    * @param {Object} options
    * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
    */
-  search(query, options = {}) {
+  search(query: string, options: SearchOptions = {}): KeywordSearchResult[] {
     this._computeStats();
-    
+
     const tokens = this.tokenize(query);
-    const scores = new Map(); // docId -> score
-    const matches = new Map(); // docId -> matched tokens
+    const scores = new Map<string, number>(); // docId -> score
+    const matches = new Map<string, string[]>(); // docId -> matched tokens
 
     const limit = options.limit || 10;
 
     for (const token of tokens) {
       const docMap = this.index.get(token);
-      if (!docMap) continue;
+      if (!docMap) {
+        continue;
+      }
 
       const idf = this.idf.get(token) || 0;
 
@@ -114,11 +142,13 @@ export class KeywordSearch {
         // Score = tf * idf * (normalization?)
         // Simple variant:
         const score = tf * idf;
-        
+
         scores.set(docId, (scores.get(docId) || 0) + score);
-        
-        if (!matches.has(docId)) matches.set(docId, []);
-        matches.get(docId).push(token);
+
+        if (!matches.has(docId)) {
+          matches.set(docId, []);
+        }
+        matches.get(docId)!.push(token);
       }
     }
 
@@ -128,7 +158,7 @@ export class KeywordSearch {
         id,
         score,
         matches: matches.get(id) || [],
-        ...this.docs.get(id)
+        ...this.docs.get(id)!,
       }))
       .sort((a, b) => b.score - a.score)
       .slice(0, limit);
@@ -136,9 +166,9 @@ export class KeywordSearch {
 
   /**
    * Bulk load records
-   * @param {Array} records 
+   * @param {Array} records
    */
-  load(records) {
-    records.forEach(r => this.add(r.id, r.content, r.metadata));
+  load(records: { id: string; content: string; metadata?: any }[]): void {
+    records.forEach((r) => this.add(r.id, r.content, r.metadata));
   }
 }

package/lib/scrubber/config/defaults.d.ts

@@ -0,0 +1,53 @@
+/**
+ * S-MORA Layer 0 Scrubber Default Configuration
+ * @module smora/scrubber/config/defaults
+ */
+export interface StructuralConfig {
+    stripHTML: boolean;
+    normalizeMarkdown: boolean;
+    collapseWhitespace: boolean;
+    removeScripts: boolean;
+    removeStyles: boolean;
+}
+export interface SemanticConfig {
+    removeDuplicates: boolean;
+    removeBoilerplate: boolean;
+    minSignalRatio: number;
+    boilerplatePatterns: string;
+}
+export interface NormalizationConfig {
+    normalizeHeadings: boolean;
+    normalizeLists: boolean;
+    normalizePunctuation: boolean;
+}
+export interface ChunkingConfig {
+    maxTokens: number;
+    minTokens: number;
+    hardMaxTokens: number;
+    splitOnHeadings: boolean;
+    preserveContext: boolean;
+}
+export interface MetadataConfig {
+    addSource: boolean;
+    addSection: boolean;
+    addHeadingPath: boolean;
+    addTimestamp: boolean;
+    addHash: boolean;
+}
+export interface ValidationConfig {
+    enforceMinLength: boolean;
+    enforceMaxLength: boolean;
+    rejectEmptyChunks: boolean;
+}
+export interface ScrubberConfig {
+    enabled: boolean;
+    structural: StructuralConfig;
+    semantic: SemanticConfig;
+    normalization: NormalizationConfig;
+    chunking: ChunkingConfig;
+    metadata: MetadataConfig;
+    validation: ValidationConfig;
+    logTransformations: boolean;
+    cachePatterns: boolean;
+}
+export declare const defaultScrubberConfig: ScrubberConfig;

package/lib/scrubber/config/defaults.js

@@ -2,61 +2,53 @@
  * S-MORA Layer 0 Scrubber Default Configuration
  * @module smora/scrubber/config/defaults
  */
-
 export const defaultScrubberConfig = {
-  // Master switch
-  enabled: false,
-
-  // Stage 1: Structural Cleaning
-  structural: {
-    stripHTML: true,
-    normalizeMarkdown: true,
-    collapseWhitespace: true,
-    removeScripts: true,
-    removeStyles: true
-  },
-
-  // Stage 2: Semantic Filtering
-  semantic: {
-    removeDuplicates: true,
-    removeBoilerplate: true,
-    minSignalRatio: 0.3,
-    boilerplatePatterns: 'default'
-  },
-
-  // Stage 3: Normalization
-  normalization: {
-    normalizeHeadings: true,
-    normalizeLists: true,
-    normalizePunctuation: true
-  },
-
-  // Stage 4: Chunking
-  chunking: {
-    maxTokens: 500,
-    minTokens: 10,
-    hardMaxTokens: 2000,
-    splitOnHeadings: true,
-    preserveContext: true
-  },
-
-  // Stage 5: Metadata Annotation
-  metadata: {
-    addSource: true,
-    addSection: true,
-    addHeadingPath: true,
-    addTimestamp: true,
-    addHash: true
-  },
-
-  // Stage 6: Validation
-  validation: {
-    enforceMinLength: true,
-    enforceMaxLength: true,
-    rejectEmptyChunks: true
-  },
-
-  // Performance
-  logTransformations: false,
-  cachePatterns: true
-};
+    // Master switch - enabled by default for security (PII/sensitive data protection)
+    enabled: true,
+    // Stage 1: Structural Cleaning
+    structural: {
+        stripHTML: true,
+        normalizeMarkdown: true,
+        collapseWhitespace: true,
+        removeScripts: true,
+        removeStyles: true,
+    },
+    // Stage 2: Semantic Filtering
+    semantic: {
+        removeDuplicates: true,
+        removeBoilerplate: true,
+        minSignalRatio: 0.3,
+        boilerplatePatterns: "default",
+    },
+    // Stage 3: Normalization
+    normalization: {
+        normalizeHeadings: true,
+        normalizeLists: true,
+        normalizePunctuation: true,
+    },
+    // Stage 4: Chunking
+    chunking: {
+        maxTokens: 500,
+        minTokens: 10,
+        hardMaxTokens: 2000,
+        splitOnHeadings: true,
+        preserveContext: true,
+    },
+    // Stage 5: Metadata Annotation
+    metadata: {
+        addSource: true,
+        addSection: true,
+        addHeadingPath: true,
+        addTimestamp: true,
+        addHash: true,
+    },
+    // Stage 6: Validation
+    validation: {
+        enforceMinLength: true,
+        enforceMaxLength: true,
+        rejectEmptyChunks: true,
+    },
+    // Performance
+    logTransformations: false,
+    cachePatterns: true,
+};

package/lib/scrubber/config/defaults.ts

@@ -0,0 +1,117 @@
+/**
+ * S-MORA Layer 0 Scrubber Default Configuration
+ * @module smora/scrubber/config/defaults
+ */
+
+export interface StructuralConfig {
+  stripHTML: boolean;
+  normalizeMarkdown: boolean;
+  collapseWhitespace: boolean;
+  removeScripts: boolean;
+  removeStyles: boolean;
+}
+
+export interface SemanticConfig {
+  removeDuplicates: boolean;
+  removeBoilerplate: boolean;
+  minSignalRatio: number;
+  boilerplatePatterns: string;
+}
+
+export interface NormalizationConfig {
+  normalizeHeadings: boolean;
+  normalizeLists: boolean;
+  normalizePunctuation: boolean;
+}
+
+export interface ChunkingConfig {
+  maxTokens: number;
+  minTokens: number;
+  hardMaxTokens: number;
+  splitOnHeadings: boolean;
+  preserveContext: boolean;
+}
+
+export interface MetadataConfig {
+  addSource: boolean;
+  addSection: boolean;
+  addHeadingPath: boolean;
+  addTimestamp: boolean;
+  addHash: boolean;
+}
+
+export interface ValidationConfig {
+  enforceMinLength: boolean;
+  enforceMaxLength: boolean;
+  rejectEmptyChunks: boolean;
+}
+
+export interface ScrubberConfig {
+  enabled: boolean;
+  structural: StructuralConfig;
+  semantic: SemanticConfig;
+  normalization: NormalizationConfig;
+  chunking: ChunkingConfig;
+  metadata: MetadataConfig;
+  validation: ValidationConfig;
+  logTransformations: boolean;
+  cachePatterns: boolean;
+}
+
+export const defaultScrubberConfig: ScrubberConfig = {
+  // Master switch - enabled by default for security (PII/sensitive data protection)
+  enabled: true,
+
+  // Stage 1: Structural Cleaning
+  structural: {
+    stripHTML: true,
+    normalizeMarkdown: true,
+    collapseWhitespace: true,
+    removeScripts: true,
+    removeStyles: true,
+  },
+
+  // Stage 2: Semantic Filtering
+  semantic: {
+    removeDuplicates: true,
+    removeBoilerplate: true,
+    minSignalRatio: 0.3,
+    boilerplatePatterns: "default",
+  },
+
+  // Stage 3: Normalization
+  normalization: {
+    normalizeHeadings: true,
+    normalizeLists: true,
+    normalizePunctuation: true,
+  },
+
+  // Stage 4: Chunking
+  chunking: {
+    maxTokens: 500,
+    minTokens: 10,
+    hardMaxTokens: 2000,
+    splitOnHeadings: true,
+    preserveContext: true,
+  },
+
+  // Stage 5: Metadata Annotation
+  metadata: {
+    addSource: true,
+    addSection: true,
+    addHeadingPath: true,
+    addTimestamp: true,
+    addHash: true,
+  },
+
+  // Stage 6: Validation
+  validation: {
+    enforceMinLength: true,
+    enforceMaxLength: true,
+    rejectEmptyChunks: true,
+  },
+
+  // Performance
+  logTransformations: false,
+  cachePatterns: true,
+};

package/lib/scrubber/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * YAMO Scrubber Module
+ * PII and sensitive data sanitization
+ */
+export { Scrubber } from "./scrubber.js";
+export { ScrubberConfig } from "./config/defaults.js";

package/lib/scrubber/index.js

@@ -1,25 +1,5 @@
 /**
- * S-MORA Layer 0 Scrubber
- * Deterministic ingestion-time preprocessing layer
- * @module smora/scrubber
+ * YAMO Scrubber Module
+ * PII and sensitive data sanitization
  */
-
-export { defaultScrubberConfig } from './config/defaults.js';
-export {
-  ScrubberError,
-  StructuralCleaningError,
-  ChunkingError,
-  ValidationError
-} from './errors/scrubber-error.js';
-export { ScrubberTelemetry } from './telemetry.js';
-export { Scrubber } from './scrubber.js';
-export { HashUtil } from './utils/hash.js';
-export { TokenCounter } from './utils/token-counter.js';
-export { PatternMatcher } from './utils/pattern-matcher.js';
-export { HTMLParser } from './utils/html-parser.js';
-export { StructuralCleaner } from './stages/structural-cleaner.js';
-export { SemanticFilter } from './stages/semantic-filter.js';
-export { Normalizer } from './stages/normalizer.js';
-export { Chunker } from './stages/chunker.js';
-export { MetadataAnnotator } from './stages/metadata-annotator.js';
-export { Validator } from './stages/validator.js';
+export { Scrubber } from "./scrubber.js";

package/lib/scrubber/index.ts

@@ -0,0 +1,7 @@
+/**
+ * YAMO Scrubber Module
+ * PII and sensitive data sanitization
+ */
+
+export { Scrubber } from "./scrubber.js";
+export { ScrubberConfig } from "./config/defaults.js";

package/lib/scrubber/scrubber.d.ts

@@ -0,0 +1,61 @@
+/**
+ * S-MORA Layer 0 Scrubber - Main Orchestrator
+ * @module smora/scrubber/scrubber
+ */
+import { StructuralCleaner } from "./stages/structural-cleaner.js";
+import { SemanticFilter } from "./stages/semantic-filter.js";
+import { Normalizer } from "./stages/normalizer.js";
+import { Chunker } from "./stages/chunker.js";
+import { MetadataAnnotator } from "./stages/metadata-annotator.js";
+import { Validator } from "./stages/validator.js";
+import { ScrubberTelemetry, TelemetrySummary, StageSummary } from "./telemetry.js";
+import { ScrubberConfig } from "./config/defaults.js";
+export interface ScrubberDocument {
+    content: string;
+    source: string;
+    type: string;
+}
+export interface Chunk {
+    text: string;
+    [key: string]: any;
+}
+export interface ScrubberResult {
+    chunks: Chunk[];
+    metadata: {
+        source: string;
+        type: string;
+        processingTimestamp: string;
+        [key: string]: any;
+    };
+    telemetry: Partial<Record<string, StageSummary>> & {
+        totalDuration?: number;
+    };
+    success?: boolean;
+    error?: string;
+}
+export declare class Scrubber {
+    config: ScrubberConfig;
+    stages: any;
+    telemetry: ScrubberTelemetry;
+    constructor(config?: Partial<ScrubberConfig>);
+    /**
+     * Main entry point - process a raw document
+     * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+     * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+     */
+    process(document: ScrubberDocument): Promise<ScrubberResult>;
+    _executeStage<T>(stageName: string, stageFn: () => Promise<T> | T): Promise<T>;
+    _initializeStages(): {
+        structural: StructuralCleaner;
+        semantic: SemanticFilter;
+        normalizer: Normalizer;
+        chunker: Chunker;
+        metadata: MetadataAnnotator;
+        validator: Validator;
+    };
+    getMetrics(): TelemetrySummary;
+    healthCheck(): Promise<{
+        status: string;
+    }>;
+}
+export default Scrubber;