@yamo/memory-mesh 2.3.2 → 3.0.0

package/lib/scrubber/scrubber.js

@@ -2,129 +2,107 @@
  * S-MORA Layer 0 Scrubber - Main Orchestrator
  * @module smora/scrubber/scrubber
  */
-
-import { StructuralCleaner } from './stages/structural-cleaner.js';
-import { SemanticFilter } from './stages/semantic-filter.js';
-import { Normalizer } from './stages/normalizer.js';
-import { Chunker } from './stages/chunker.js';
-import { MetadataAnnotator } from './stages/metadata-annotator.js';
-import { Validator } from './stages/validator.js';
-import { ScrubberTelemetry } from './telemetry.js';
-import { ScrubberError } from './errors/scrubber-error.js';
-import { defaultScrubberConfig } from './config/defaults.js';
-
+import { StructuralCleaner } from "./stages/structural-cleaner.js";
+import { SemanticFilter } from "./stages/semantic-filter.js";
+import { Normalizer } from "./stages/normalizer.js";
+import { Chunker } from "./stages/chunker.js";
+import { MetadataAnnotator } from "./stages/metadata-annotator.js";
+import { Validator } from "./stages/validator.js";
+import { ScrubberTelemetry, } from "./telemetry.js";
+// import { ScrubberError } from './errors/scrubber-error'; // Assuming this exists or I should check
+import { defaultScrubberConfig } from "./config/defaults.js";
 export class Scrubber {
-  constructor(config = {}) {
-    this.config = { ...defaultScrubberConfig, ...config };
-    this.stages = this._initializeStages();
-    this.telemetry = new ScrubberTelemetry();
-  }
-
-  /**
-   * Main entry point - process a raw document
-   * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
-   * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
-   */
-  async process(document) {
-    const startTime = Date.now();
-    const result = {
-      chunks: [],
-      metadata: {
-        source: document.source,
-        type: document.type,
-        processingTimestamp: new Date().toISOString()
-      },
-      telemetry: {}
-    };
-
-    try {
-      // If disabled, return empty chunks
-      if (!this.config.enabled) {
-        result.success = true;
-        result.telemetry.totalDuration = Date.now() - startTime;
-        return result;
-      }
-
-      // Stage 1: Structural Cleaning
-      const cleaned = await this._executeStage('structural', () =>
-        this.stages.structural.clean(document.content)
-      );
-      result.telemetry.structural = this.telemetry.getStageStats('structural');
-
-      // Stage 2: Semantic Filtering
-      const filtered = await this._executeStage('semantic', () =>
-        this.stages.semantic.filter(cleaned)
-      );
-      result.telemetry.semantic = this.telemetry.getStageStats('semantic');
-
-      // Stage 3: Normalization
-      const normalized = await this._executeStage('normalization', () =>
-        this.stages.normalizer.normalize(filtered)
-      );
-      result.telemetry.normalization = this.telemetry.getStageStats('normalization');
-
-      // Stage 4: Chunking
-      const chunks = await this._executeStage('chunking', () =>
-        this.stages.chunker.chunk(normalized)
-      );
-      result.telemetry.chunking = this.telemetry.getStageStats('chunking');
-
-      // Stage 5: Metadata Annotation
-      const annotated = await this._executeStage('metadata', () =>
-        this.stages.metadata.annotate(chunks, document)
-      );
-      result.telemetry.metadata = this.telemetry.getStageStats('metadata');
-
-      // Stage 6: Validation
-      result.chunks = await this._executeStage('validation', () =>
-        this.stages.validator.validate(annotated)
-      );
-      result.telemetry.validation = this.telemetry.getStageStats('validation');
-
-      result.telemetry.totalDuration = Date.now() - startTime;
-      result.success = true;
-
-      return result;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      result.success = false;
-      result.error = message;
-      result.telemetry.totalDuration = Date.now() - startTime;
+    config;
+    stages; // Using any for stages as they are not yet converted
+    telemetry;
+    constructor(config = {}) {
+        this.config = { ...defaultScrubberConfig, ...config };
+        this.stages = this._initializeStages();
+        this.telemetry = new ScrubberTelemetry();
     }
-  }
-
-  async _executeStage(stageName, stageFn) {
-    const startTime = Date.now();
-    try {
-      const result = await stageFn();
-      const duration = Date.now() - startTime;
-      this.telemetry.recordStage(stageName, duration, true);
-      return result;
-    } catch (error) {
-      const duration = Date.now() - startTime;
-      this.telemetry.recordStage(stageName, duration, false);
-      throw error;
+    /**
+     * Main entry point - process a raw document
+     * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+     * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+     */
+    async process(document) {
+        const startTime = Date.now();
+        const result = {
+            chunks: [],
+            metadata: {
+                source: document.source,
+                type: document.type,
+                processingTimestamp: new Date().toISOString(),
+            },
+            telemetry: {},
+        };
+        try {
+            // If disabled, return empty chunks
+            if (!this.config.enabled) {
+                result.success = true;
+                result.telemetry.totalDuration = Date.now() - startTime;
+                return result;
+            }
+            // Stage 1: Structural Cleaning
+            const cleaned = await this._executeStage("structural", () => this.stages.structural.clean(document.content));
+            result.telemetry.structural = this.telemetry.getStageStats("structural");
+            // Stage 2: Semantic Filtering
+            const filtered = await this._executeStage("semantic", () => this.stages.semantic.filter(cleaned));
+            result.telemetry.semantic = this.telemetry.getStageStats("semantic");
+            // Stage 3: Normalization
+            const normalized = await this._executeStage("normalization", () => this.stages.normalizer.normalize(filtered));
+            result.telemetry.normalization =
+                this.telemetry.getStageStats("normalization");
+            // Stage 4: Chunking
+            const chunks = await this._executeStage("chunking", () => this.stages.chunker.chunk(normalized));
+            result.telemetry.chunking = this.telemetry.getStageStats("chunking");
+            // Stage 5: Metadata Annotation
+            const annotated = await this._executeStage("metadata", () => this.stages.metadata.annotate(chunks, document));
+            result.telemetry.metadata = this.telemetry.getStageStats("metadata");
+            // Stage 6: Validation
+            result.chunks = await this._executeStage("validation", () => this.stages.validator.validate(annotated));
+            result.telemetry.validation = this.telemetry.getStageStats("validation");
+            result.telemetry.totalDuration = Date.now() - startTime;
+            result.success = true;
+            return result;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            result.success = false;
+            result.error = message;
+            result.telemetry.totalDuration = Date.now() - startTime;
+            return result;
+        }
+    }
+    async _executeStage(stageName, stageFn) {
+        const startTime = Date.now();
+        try {
+            const result = await stageFn();
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, true);
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, false);
+            throw error;
+        }
+    }
+    _initializeStages() {
+        return {
+            structural: new StructuralCleaner(this.config.structural),
+            semantic: new SemanticFilter(this.config.semantic),
+            normalizer: new Normalizer(this.config.normalization),
+            chunker: new Chunker(this.config.chunking),
+            metadata: new MetadataAnnotator(this.config.metadata),
+            validator: new Validator(this.config.validation),
+        };
+    }
+    getMetrics() {
+        return this.telemetry.getSummary();
+    }
+    healthCheck() {
+        return Promise.resolve({ status: "healthy" });
     }
-  }
-
-  _initializeStages() {
-    return {
-      structural: new StructuralCleaner(this.config.structural),
-      semantic: new SemanticFilter(this.config.semantic),
-      normalizer: new Normalizer(this.config.normalization),
-      chunker: new Chunker(this.config.chunking),
-      metadata: new MetadataAnnotator(this.config.metadata),
-      validator: new Validator(this.config.validation)
-    };
-  }
-
-  getMetrics() {
-    return this.telemetry.getSummary();
-  }
-
-  async healthCheck() {
-    return { status: 'healthy' };
-  }
 }
-
 export default Scrubber;

package/lib/scrubber/scrubber.ts

@@ -0,0 +1,168 @@
+/**
+ * S-MORA Layer 0 Scrubber - Main Orchestrator
+ * @module smora/scrubber/scrubber
+ */
+
+import { StructuralCleaner } from "./stages/structural-cleaner.js";
+import { SemanticFilter } from "./stages/semantic-filter.js";
+import { Normalizer } from "./stages/normalizer.js";
+import { Chunker } from "./stages/chunker.js";
+import { MetadataAnnotator } from "./stages/metadata-annotator.js";
+import { Validator } from "./stages/validator.js";
+import {
+  ScrubberTelemetry,
+  TelemetrySummary,
+  StageSummary,
+} from "./telemetry.js";
+// import { ScrubberError } from './errors/scrubber-error'; // Assuming this exists or I should check
+import { defaultScrubberConfig, ScrubberConfig } from "./config/defaults.js";
+
+// Interfaces for input/output
+export interface ScrubberDocument {
+  content: string;
+  source: string;
+  type: string;
+}
+
+export interface Chunk {
+  text: string;
+  [key: string]: any;
+}
+
+export interface ScrubberResult {
+  chunks: Chunk[];
+  metadata: {
+    source: string;
+    type: string;
+    processingTimestamp: string;
+    [key: string]: any;
+  };
+  telemetry: Partial<Record<string, StageSummary>> & { totalDuration?: number };
+  success?: boolean;
+  error?: string;
+}
+
+export class Scrubber {
+  config: ScrubberConfig;
+  stages: any; // Using any for stages as they are not yet converted
+  telemetry: ScrubberTelemetry;
+
+  constructor(config: Partial<ScrubberConfig> = {}) {
+    this.config = { ...defaultScrubberConfig, ...config };
+    this.stages = this._initializeStages();
+    this.telemetry = new ScrubberTelemetry();
+  }
+
+  /**
+   * Main entry point - process a raw document
+   * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+   * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+   */
+  async process(document: ScrubberDocument): Promise<ScrubberResult> {
+    const startTime = Date.now();
+    const result: ScrubberResult = {
+      chunks: [],
+      metadata: {
+        source: document.source,
+        type: document.type,
+        processingTimestamp: new Date().toISOString(),
+      },
+      telemetry: {},
+    };
+
+    try {
+      // If disabled, return empty chunks
+      if (!this.config.enabled) {
+        result.success = true;
+        result.telemetry.totalDuration = Date.now() - startTime;
+        return result;
+      }
+
+      // Stage 1: Structural Cleaning
+      const cleaned = await this._executeStage("structural", () =>
+        this.stages.structural.clean(document.content),
+      );
+      result.telemetry.structural = this.telemetry.getStageStats("structural");
+
+      // Stage 2: Semantic Filtering
+      const filtered = await this._executeStage("semantic", () =>
+        this.stages.semantic.filter(cleaned),
+      );
+      result.telemetry.semantic = this.telemetry.getStageStats("semantic");
+
+      // Stage 3: Normalization
+      const normalized = await this._executeStage("normalization", () =>
+        this.stages.normalizer.normalize(filtered),
+      );
+      result.telemetry.normalization =
+        this.telemetry.getStageStats("normalization");
+
+      // Stage 4: Chunking
+      const chunks = await this._executeStage("chunking", () =>
+        this.stages.chunker.chunk(normalized),
+      );
+      result.telemetry.chunking = this.telemetry.getStageStats("chunking");
+
+      // Stage 5: Metadata Annotation
+      const annotated = await this._executeStage("metadata", () =>
+        this.stages.metadata.annotate(chunks, document),
+      );
+      result.telemetry.metadata = this.telemetry.getStageStats("metadata");
+
+      // Stage 6: Validation
+      result.chunks = await this._executeStage("validation", () =>
+        this.stages.validator.validate(annotated),
+      );
+      result.telemetry.validation = this.telemetry.getStageStats("validation");
+
+      result.telemetry.totalDuration = Date.now() - startTime;
+      result.success = true;
+
+      return result;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      result.success = false;
+      result.error = message;
+      result.telemetry.totalDuration = Date.now() - startTime;
+      return result;
+    }
+  }
+
+  async _executeStage<T>(
+    stageName: string,
+    stageFn: () => Promise<T> | T,
+  ): Promise<T> {
+    const startTime = Date.now();
+    try {
+      const result = await stageFn();
+      const duration = Date.now() - startTime;
+      this.telemetry.recordStage(stageName, duration, true);
+      return result;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      this.telemetry.recordStage(stageName, duration, false);
+      throw error;
+    }
+  }
+
+  _initializeStages() {
+    return {
+      structural: new StructuralCleaner(this.config.structural),
+      semantic: new SemanticFilter(this.config.semantic),
+      normalizer: new Normalizer(this.config.normalization),
+      chunker: new Chunker(this.config.chunking),
+      metadata: new MetadataAnnotator(this.config.metadata),
+      validator: new Validator(this.config.validation),
+    };
+  }
+
+  getMetrics(): TelemetrySummary {
+    return this.telemetry.getSummary();
+  }
+
+  healthCheck(): Promise<{ status: string }> {
+    return Promise.resolve({ status: "healthy" });
+  }
+}
+
+export default Scrubber;

package/lib/scrubber/stages/chunker.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Type definitions for chunker.js
+ */
+
+export interface ChunkerConfig {
+  maxSize?: number;
+  [key: string]: any;
+}
+
+export class Chunker {
+  constructor(config?: ChunkerConfig);
+  chunk(content: string): Promise<string[]>;
+}

package/lib/scrubber/stages/metadata-annotator.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Type definitions for metadata-annotator.js
+ */
+
+export interface AnnotatorConfig {
+  includeTimestamp?: boolean;
+  [key: string]: any;
+}
+
+export interface AnnotatedData {
+  content: string;
+  metadata: Record<string, any>;
+}
+
+export class MetadataAnnotator {
+  constructor(config?: AnnotatorConfig);
+  annotate(content: string): Promise<AnnotatedData>;
+}

package/lib/scrubber/stages/normalizer.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Type definitions for normalizer.js
+ */
+
+export interface NormalizerConfig {
+  lowercase?: boolean;
+  [key: string]: any;
+}
+
+export class Normalizer {
+  constructor(config?: NormalizerConfig);
+  normalize(content: string): Promise<string>;
+}

package/lib/scrubber/stages/semantic-filter.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Type definitions for semantic-filter.js
+ */
+
+export interface FilterConfig {
+  threshold?: number;
+  [key: string]: any;
+}
+
+export class SemanticFilter {
+  constructor(config?: FilterConfig);
+  filter(content: string): Promise<string>;
+}

package/lib/scrubber/stages/structural-cleaner.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Type definitions for structural-cleaner.js
+ */
+
+export interface CleanConfig {
+  preserveStructure?: boolean;
+  [key: string]: any;
+}
+
+export class StructuralCleaner {
+  constructor(config?: CleanConfig);
+  clean(content: string): Promise<string>;
+}

package/lib/scrubber/stages/validator.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Type definitions for validator.js
+ */
+
+export interface ValidatorConfig {
+  strict?: boolean;
+  [key: string]: any;
+}
+
+export interface ValidationResult {
+  valid: boolean;
+  errors?: string[];
+}
+
+export class Validator {
+  constructor(config?: ValidatorConfig);
+  validate(content: string): Promise<ValidationResult>;
+}

package/lib/scrubber/telemetry.d.ts

@@ -0,0 +1,36 @@
+/**
+ * S-MORA Layer 0 Scrubber Telemetry Collection
+ * @module smora/scrubber/telemetry
+ */
+export interface StageStats {
+    count: number;
+    totalTime: number;
+    errors: number;
+}
+export interface StageSummary {
+    count: number;
+    avgTime: number;
+    totalTime: number;
+    errors: number;
+}
+export interface TelemetrySummary {
+    stages: Record<string, StageStats>;
+    performance: {
+        structural: number;
+        semantic: number;
+        normalization: number;
+        chunking: number;
+        metadata: number;
+        validation: number;
+        total: number;
+    };
+}
+export declare class ScrubberTelemetry {
+    stats: Record<string, StageStats>;
+    constructor();
+    recordStage(stage: string, duration: number, success?: boolean): void;
+    getStageStats(stage: string): StageSummary;
+    getSummary(): TelemetrySummary;
+    reset(): void;
+    assertPerformanceBudget(budget?: number): void;
+}

package/lib/scrubber/telemetry.js

@@ -2,65 +2,60 @@
  * S-MORA Layer 0 Scrubber Telemetry Collection
  * @module smora/scrubber/telemetry
  */
-
 export class ScrubberTelemetry {
-  constructor() {
-    this.stats = {
-      structural: { count: 0, totalTime: 0, errors: 0 },
-      semantic: { count: 0, totalTime: 0, errors: 0 },
-      normalization: { count: 0, totalTime: 0, errors: 0 },
-      chunking: { count: 0, totalTime: 0, errors: 0 },
-      metadata: { count: 0, totalTime: 0, errors: 0 },
-      validation: { count: 0, totalTime: 0, errors: 0 }
-    };
-  }
-
-  recordStage(stage, duration, success = true) {
-    if (!this.stats[stage]) {
-      this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+    stats;
+    constructor() {
+        this.stats = {
+            structural: { count: 0, totalTime: 0, errors: 0 },
+            semantic: { count: 0, totalTime: 0, errors: 0 },
+            normalization: { count: 0, totalTime: 0, errors: 0 },
+            chunking: { count: 0, totalTime: 0, errors: 0 },
+            metadata: { count: 0, totalTime: 0, errors: 0 },
+            validation: { count: 0, totalTime: 0, errors: 0 },
+        };
     }
-    this.stats[stage].count++;
-    this.stats[stage].totalTime += duration;
-    if (!success) this.stats[stage].errors++;
-  }
-
-  getStageStats(stage) {
-    const stats = this.stats[stage];
-    return {
-      count: stats.count,
-      avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
-      totalTime: stats.totalTime,
-      errors: stats.errors
-    };
-  }
-
-  getSummary() {
-    return {
-      stages: this.stats,
-      performance: {
-        structural: this.stats.structural.totalTime,
-        semantic: this.stats.semantic.totalTime,
-        normalization: this.stats.normalization.totalTime,
-        chunking: this.stats.chunking.totalTime,
-        metadata: this.stats.metadata.totalTime,
-        validation: this.stats.validation.totalTime,
-        total: Object.values(this.stats).reduce((sum, s) => sum + s.totalTime, 0)
-      }
-    };
-  }
-
-  reset() {
-    Object.keys(this.stats).forEach(key => {
-      this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
-    });
-  }
-
-  assertPerformanceBudget(budget = 10) {
-    const summary = this.getSummary();
-    if (summary.performance.total > budget) {
-      throw new Error(
-        `Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`
-      );
+    recordStage(stage, duration, success = true) {
+        if (!this.stats[stage]) {
+            this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+        }
+        this.stats[stage].count++;
+        this.stats[stage].totalTime += duration;
+        if (!success) {
+            this.stats[stage].errors++;
+        }
+    }
+    getStageStats(stage) {
+        const stats = this.stats[stage] || { count: 0, totalTime: 0, errors: 0 };
+        return {
+            count: stats.count,
+            avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
+            totalTime: stats.totalTime,
+            errors: stats.errors,
+        };
+    }
+    getSummary() {
+        return {
+            stages: this.stats,
+            performance: {
+                structural: this.stats.structural.totalTime,
+                semantic: this.stats.semantic.totalTime,
+                normalization: this.stats.normalization.totalTime,
+                chunking: this.stats.chunking.totalTime,
+                metadata: this.stats.metadata.totalTime,
+                validation: this.stats.validation.totalTime,
+                total: Object.values(this.stats).reduce((sum, s) => sum + s.totalTime, 0),
+            },
+        };
+    }
+    reset() {
+        Object.keys(this.stats).forEach((key) => {
+            this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
+        });
+    }
+    assertPerformanceBudget(budget = 10) {
+        const summary = this.getSummary();
+        if (summary.performance.total > budget) {
+            throw new Error(`Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`);
+        }
     }
-  }
 }