@yamo/memory-mesh 2.3.2 → 3.0.1

package/lib/scrubber/config/defaults.d.ts

@@ -0,0 +1,46 @@
+/**
+ * S-MORA Layer 0 Scrubber Default Configuration
+ * @module smora/scrubber/config/defaults
+ */
+export declare const defaultScrubberConfig: {
+    enabled: boolean;
+    structural: {
+        stripHTML: boolean;
+        normalizeMarkdown: boolean;
+        collapseWhitespace: boolean;
+        removeScripts: boolean;
+        removeStyles: boolean;
+    };
+    semantic: {
+        removeDuplicates: boolean;
+        removeBoilerplate: boolean;
+        minSignalRatio: number;
+        boilerplatePatterns: string;
+    };
+    normalization: {
+        normalizeHeadings: boolean;
+        normalizeLists: boolean;
+        normalizePunctuation: boolean;
+    };
+    chunking: {
+        maxTokens: number;
+        minTokens: number;
+        hardMaxTokens: number;
+        splitOnHeadings: boolean;
+        preserveContext: boolean;
+    };
+    metadata: {
+        addSource: boolean;
+        addSection: boolean;
+        addHeadingPath: boolean;
+        addTimestamp: boolean;
+        addHash: boolean;
+    };
+    validation: {
+        enforceMinLength: boolean;
+        enforceMaxLength: boolean;
+        rejectEmptyChunks: boolean;
+    };
+    logTransformations: boolean;
+    cachePatterns: boolean;
+};

package/lib/scrubber/config/defaults.js

@@ -1,62 +1,55 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber Default Configuration
  * @module smora/scrubber/config/defaults
  */
-
 export const defaultScrubberConfig = {
-  // Master switch
-  enabled: false,
-
-  // Stage 1: Structural Cleaning
-  structural: {
-    stripHTML: true,
-    normalizeMarkdown: true,
-    collapseWhitespace: true,
-    removeScripts: true,
-    removeStyles: true
-  },
-
-  // Stage 2: Semantic Filtering
-  semantic: {
-    removeDuplicates: true,
-    removeBoilerplate: true,
-    minSignalRatio: 0.3,
-    boilerplatePatterns: 'default'
-  },
-
-  // Stage 3: Normalization
-  normalization: {
-    normalizeHeadings: true,
-    normalizeLists: true,
-    normalizePunctuation: true
-  },
-
-  // Stage 4: Chunking
-  chunking: {
-    maxTokens: 500,
-    minTokens: 10,
-    hardMaxTokens: 2000,
-    splitOnHeadings: true,
-    preserveContext: true
-  },
-
-  // Stage 5: Metadata Annotation
-  metadata: {
-    addSource: true,
-    addSection: true,
-    addHeadingPath: true,
-    addTimestamp: true,
-    addHash: true
-  },
-
-  // Stage 6: Validation
-  validation: {
-    enforceMinLength: true,
-    enforceMaxLength: true,
-    rejectEmptyChunks: true
-  },
-
-  // Performance
-  logTransformations: false,
-  cachePatterns: true
-};
+    // Master switch - enabled by default for security (PII/sensitive data protection)
+    enabled: true,
+    // Stage 1: Structural Cleaning
+    structural: {
+        stripHTML: true,
+        normalizeMarkdown: true,
+        collapseWhitespace: true,
+        removeScripts: true,
+        removeStyles: true,
+    },
+    // Stage 2: Semantic Filtering
+    semantic: {
+        removeDuplicates: true,
+        removeBoilerplate: true,
+        minSignalRatio: 0.3,
+        boilerplatePatterns: "default",
+    },
+    // Stage 3: Normalization
+    normalization: {
+        normalizeHeadings: true,
+        normalizeLists: true,
+        normalizePunctuation: true,
+    },
+    // Stage 4: Chunking
+    chunking: {
+        maxTokens: 500,
+        minTokens: 10,
+        hardMaxTokens: 2000,
+        splitOnHeadings: true,
+        preserveContext: true,
+    },
+    // Stage 5: Metadata Annotation
+    metadata: {
+        addSource: true,
+        addSection: true,
+        addHeadingPath: true,
+        addTimestamp: true,
+        addHash: true,
+    },
+    // Stage 6: Validation
+    validation: {
+        enforceMinLength: true,
+        enforceMaxLength: true,
+        rejectEmptyChunks: true,
+    },
+    // Performance
+    logTransformations: false,
+    cachePatterns: true,
+};

package/lib/scrubber/config/defaults.ts

@@ -0,0 +1,55 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber Default Configuration
+ * @module smora/scrubber/config/defaults
+ */
+export const defaultScrubberConfig = {
+    // Master switch - enabled by default for security (PII/sensitive data protection)
+    enabled: true,
+    // Stage 1: Structural Cleaning
+    structural: {
+        stripHTML: true,
+        normalizeMarkdown: true,
+        collapseWhitespace: true,
+        removeScripts: true,
+        removeStyles: true,
+    },
+    // Stage 2: Semantic Filtering
+    semantic: {
+        removeDuplicates: true,
+        removeBoilerplate: true,
+        minSignalRatio: 0.3,
+        boilerplatePatterns: "default",
+    },
+    // Stage 3: Normalization
+    normalization: {
+        normalizeHeadings: true,
+        normalizeLists: true,
+        normalizePunctuation: true,
+    },
+    // Stage 4: Chunking
+    chunking: {
+        maxTokens: 500,
+        minTokens: 10,
+        hardMaxTokens: 2000,
+        splitOnHeadings: true,
+        preserveContext: true,
+    },
+    // Stage 5: Metadata Annotation
+    metadata: {
+        addSource: true,
+        addSection: true,
+        addHeadingPath: true,
+        addTimestamp: true,
+        addHash: true,
+    },
+    // Stage 6: Validation
+    validation: {
+        enforceMinLength: true,
+        enforceMaxLength: true,
+        rejectEmptyChunks: true,
+    },
+    // Performance
+    logTransformations: false,
+    cachePatterns: true,
+};

package/lib/scrubber/errors/scrubber-error.d.ts

@@ -0,0 +1,22 @@
+/**
+ * S-MORA Layer 0 Scrubber Error Classes
+ * @module smora/scrubber/errors/scrubber-error
+ */
+export declare class ScrubberError extends Error {
+    constructor(message: any, details?: {});
+    toJSON(): {
+        name: string;
+        message: string;
+        details: any;
+        timestamp: any;
+    };
+}
+export declare class StructuralCleaningError extends ScrubberError {
+    constructor(message: any, details?: {});
+}
+export declare class ChunkingError extends ScrubberError {
+    constructor(message: any, details?: {});
+}
+export declare class ValidationError extends ScrubberError {
+    constructor(message: any, details?: {});
+}

package/lib/scrubber/errors/scrubber-error.js

@@ -1,43 +1,39 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber Error Classes
  * @module smora/scrubber/errors/scrubber-error
  */
-
 export class ScrubberError extends Error {
-  constructor(message, details = {}) {
-    super(message);
-    this.name = 'ScrubberError';
-    this.details = details;
-    this.timestamp = new Date().toISOString();
-  }
-
-  toJSON() {
-    return {
-      name: this.name,
-      message: this.message,
-      details: this.details,
-      timestamp: this.timestamp
-    };
-  }
+    constructor(message, details = {}) {
+        super(message);
+        this.name = 'ScrubberError';
+        this.details = details;
+        this.timestamp = new Date().toISOString();
+    }
+    toJSON() {
+        return {
+            name: this.name,
+            message: this.message,
+            details: this.details,
+            timestamp: this.timestamp
+        };
+    }
 }
-
 export class StructuralCleaningError extends ScrubberError {
-  constructor(message, details = {}) {
-    super(message, details);
-    this.name = 'StructuralCleaningError';
-  }
+    constructor(message, details = {}) {
+        super(message, details);
+        this.name = 'StructuralCleaningError';
+    }
 }
-
 export class ChunkingError extends ScrubberError {
-  constructor(message, details = {}) {
-    super(message, details);
-    this.name = 'ChunkingError';
-  }
+    constructor(message, details = {}) {
+        super(message, details);
+        this.name = 'ChunkingError';
+    }
 }
-
 export class ValidationError extends ScrubberError {
-  constructor(message, details = {}) {
-    super(message, details);
-    this.name = 'ValidationError';
-  }
-}
+    constructor(message, details = {}) {
+        super(message, details);
+        this.name = 'ValidationError';
+    }
+}

package/lib/scrubber/errors/scrubber-error.ts

@@ -0,0 +1,44 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber Error Classes
+ * @module smora/scrubber/errors/scrubber-error
+ */
+
+export class ScrubberError extends Error {
+  constructor(message, details = {}) {
+    super(message);
+    this.name = 'ScrubberError';
+    this.details = details;
+    this.timestamp = new Date().toISOString();
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+      message: this.message,
+      details: this.details,
+      timestamp: this.timestamp
+    };
+  }
+}
+
+export class StructuralCleaningError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'StructuralCleaningError';
+  }
+}
+
+export class ChunkingError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'ChunkingError';
+  }
+}
+
+export class ValidationError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'ValidationError';
+  }
+}

package/lib/scrubber/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * YAMO Scrubber Module
+ * PII and sensitive data sanitization
+ */
+export { Scrubber } from "./scrubber.js";

package/lib/scrubber/index.js

@@ -1,25 +1,6 @@
+// @ts-nocheck
 /**
- * S-MORA Layer 0 Scrubber
- * Deterministic ingestion-time preprocessing layer
- * @module smora/scrubber
+ * YAMO Scrubber Module
+ * PII and sensitive data sanitization
  */
-
-export { defaultScrubberConfig } from './config/defaults.js';
-export {
-  ScrubberError,
-  StructuralCleaningError,
-  ChunkingError,
-  ValidationError
-} from './errors/scrubber-error.js';
-export { ScrubberTelemetry } from './telemetry.js';
-export { Scrubber } from './scrubber.js';
-export { HashUtil } from './utils/hash.js';
-export { TokenCounter } from './utils/token-counter.js';
-export { PatternMatcher } from './utils/pattern-matcher.js';
-export { HTMLParser } from './utils/html-parser.js';
-export { StructuralCleaner } from './stages/structural-cleaner.js';
-export { SemanticFilter } from './stages/semantic-filter.js';
-export { Normalizer } from './stages/normalizer.js';
-export { Chunker } from './stages/chunker.js';
-export { MetadataAnnotator } from './stages/metadata-annotator.js';
-export { Validator } from './stages/validator.js';
+export { Scrubber } from "./scrubber.js";

package/lib/scrubber/index.ts

@@ -0,0 +1,6 @@
+// @ts-nocheck
+/**
+ * YAMO Scrubber Module
+ * PII and sensitive data sanitization
+ */
+export { Scrubber } from "./scrubber.js";

package/lib/scrubber/scrubber.d.ts

@@ -0,0 +1,44 @@
+/**
+ * S-MORA Layer 0 Scrubber - Main Orchestrator
+ * @module smora/scrubber/scrubber
+ */
+import { StructuralCleaner } from "./stages/structural-cleaner.js";
+import { SemanticFilter } from "./stages/semantic-filter.js";
+import { Normalizer } from "./stages/normalizer.js";
+import { Chunker } from "./stages/chunker.js";
+import { MetadataAnnotator } from "./stages/metadata-annotator.js";
+import { Validator } from "./stages/validator.js";
+export declare class Scrubber {
+    config: any;
+    stages: any;
+    telemetry: any;
+    constructor(config?: {});
+    /**
+     * Main entry point - process a raw document
+     * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+     * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+     */
+    process(document: any): Promise<{
+        chunks: any[];
+        metadata: {
+            source: any;
+            type: any;
+            processingTimestamp: string;
+        };
+        telemetry: {};
+    }>;
+    _executeStage(stageName: any, stageFn: any): Promise<any>;
+    _initializeStages(): {
+        structural: StructuralCleaner;
+        semantic: SemanticFilter;
+        normalizer: Normalizer;
+        chunker: Chunker;
+        metadata: MetadataAnnotator;
+        validator: Validator;
+    };
+    getMetrics(): any;
+    healthCheck(): Promise<{
+        status: string;
+    }>;
+}
+export default Scrubber;

package/lib/scrubber/scrubber.js

@@ -1,130 +1,109 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Main Orchestrator
  * @module smora/scrubber/scrubber
  */
-
-import { StructuralCleaner } from './stages/structural-cleaner.js';
-import { SemanticFilter } from './stages/semantic-filter.js';
-import { Normalizer } from './stages/normalizer.js';
-import { Chunker } from './stages/chunker.js';
-import { MetadataAnnotator } from './stages/metadata-annotator.js';
-import { Validator } from './stages/validator.js';
-import { ScrubberTelemetry } from './telemetry.js';
-import { ScrubberError } from './errors/scrubber-error.js';
-import { defaultScrubberConfig } from './config/defaults.js';
-
+import { StructuralCleaner } from "./stages/structural-cleaner.js";
+import { SemanticFilter } from "./stages/semantic-filter.js";
+import { Normalizer } from "./stages/normalizer.js";
+import { Chunker } from "./stages/chunker.js";
+import { MetadataAnnotator } from "./stages/metadata-annotator.js";
+import { Validator } from "./stages/validator.js";
+import { ScrubberTelemetry, } from "./telemetry.js";
+// import { ScrubberError } from './errors/scrubber-error'; // Assuming this exists or I should check
+import { defaultScrubberConfig } from "./config/defaults.js";
 export class Scrubber {
-  constructor(config = {}) {
-    this.config = { ...defaultScrubberConfig, ...config };
-    this.stages = this._initializeStages();
-    this.telemetry = new ScrubberTelemetry();
-  }
-
-  /**
-   * Main entry point - process a raw document
-   * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
-   * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
-   */
-  async process(document) {
-    const startTime = Date.now();
-    const result = {
-      chunks: [],
-      metadata: {
-        source: document.source,
-        type: document.type,
-        processingTimestamp: new Date().toISOString()
-      },
-      telemetry: {}
-    };
-
-    try {
-      // If disabled, return empty chunks
-      if (!this.config.enabled) {
-        result.success = true;
-        result.telemetry.totalDuration = Date.now() - startTime;
-        return result;
-      }
-
-      // Stage 1: Structural Cleaning
-      const cleaned = await this._executeStage('structural', () =>
-        this.stages.structural.clean(document.content)
-      );
-      result.telemetry.structural = this.telemetry.getStageStats('structural');
-
-      // Stage 2: Semantic Filtering
-      const filtered = await this._executeStage('semantic', () =>
-        this.stages.semantic.filter(cleaned)
-      );
-      result.telemetry.semantic = this.telemetry.getStageStats('semantic');
-
-      // Stage 3: Normalization
-      const normalized = await this._executeStage('normalization', () =>
-        this.stages.normalizer.normalize(filtered)
-      );
-      result.telemetry.normalization = this.telemetry.getStageStats('normalization');
-
-      // Stage 4: Chunking
-      const chunks = await this._executeStage('chunking', () =>
-        this.stages.chunker.chunk(normalized)
-      );
-      result.telemetry.chunking = this.telemetry.getStageStats('chunking');
-
-      // Stage 5: Metadata Annotation
-      const annotated = await this._executeStage('metadata', () =>
-        this.stages.metadata.annotate(chunks, document)
-      );
-      result.telemetry.metadata = this.telemetry.getStageStats('metadata');
-
-      // Stage 6: Validation
-      result.chunks = await this._executeStage('validation', () =>
-        this.stages.validator.validate(annotated)
-      );
-      result.telemetry.validation = this.telemetry.getStageStats('validation');
-
-      result.telemetry.totalDuration = Date.now() - startTime;
-      result.success = true;
-
-      return result;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      result.success = false;
-      result.error = message;
-      result.telemetry.totalDuration = Date.now() - startTime;
+    config;
+    stages; // Using any for stages as they are not yet converted
+    telemetry;
+    constructor(config = {}) {
+        this.config = { ...defaultScrubberConfig, ...config };
+        this.stages = this._initializeStages();
+        this.telemetry = new ScrubberTelemetry();
     }
-  }
-
-  async _executeStage(stageName, stageFn) {
-    const startTime = Date.now();
-    try {
-      const result = await stageFn();
-      const duration = Date.now() - startTime;
-      this.telemetry.recordStage(stageName, duration, true);
-      return result;
-    } catch (error) {
-      const duration = Date.now() - startTime;
-      this.telemetry.recordStage(stageName, duration, false);
-      throw error;
+    /**
+     * Main entry point - process a raw document
+     * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+     * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+     */
+    async process(document) {
+        const startTime = Date.now();
+        const result = {
+            chunks: [],
+            metadata: {
+                source: document.source,
+                type: document.type,
+                processingTimestamp: new Date().toISOString(),
+            },
+            telemetry: {},
+        };
+        try {
+            // If disabled, return empty chunks
+            if (!this.config.enabled) {
+                result.success = true;
+                result.telemetry.totalDuration = Date.now() - startTime;
+                return result;
+            }
+            // Stage 1: Structural Cleaning
+            const cleaned = await this._executeStage("structural", () => this.stages.structural.clean(document.content));
+            result.telemetry.structural = this.telemetry.getStageStats("structural");
+            // Stage 2: Semantic Filtering
+            const filtered = await this._executeStage("semantic", () => this.stages.semantic.filter(cleaned));
+            result.telemetry.semantic = this.telemetry.getStageStats("semantic");
+            // Stage 3: Normalization
+            const normalized = await this._executeStage("normalization", () => this.stages.normalizer.normalize(filtered));
+            result.telemetry.normalization =
+                this.telemetry.getStageStats("normalization");
+            // Stage 4: Chunking
+            const chunks = await this._executeStage("chunking", () => this.stages.chunker.chunk(normalized));
+            result.telemetry.chunking = this.telemetry.getStageStats("chunking");
+            // Stage 5: Metadata Annotation
+            const annotated = await this._executeStage("metadata", () => this.stages.metadata.annotate(chunks, document));
+            result.telemetry.metadata = this.telemetry.getStageStats("metadata");
+            // Stage 6: Validation
+            result.chunks = await this._executeStage("validation", () => this.stages.validator.validate(annotated));
+            result.telemetry.validation = this.telemetry.getStageStats("validation");
+            result.telemetry.totalDuration = Date.now() - startTime;
+            result.success = true;
+            return result;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            result.success = false;
+            result.error = message;
+            result.telemetry.totalDuration = Date.now() - startTime;
+            return result;
+        }
+    }
+    async _executeStage(stageName, stageFn) {
+        const startTime = Date.now();
+        try {
+            const result = await stageFn();
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, true);
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, false);
+            throw error;
+        }
+    }
+    _initializeStages() {
+        return {
+            structural: new StructuralCleaner(this.config.structural),
+            semantic: new SemanticFilter(this.config.semantic),
+            normalizer: new Normalizer(this.config.normalization),
+            chunker: new Chunker(this.config.chunking),
+            metadata: new MetadataAnnotator(this.config.metadata),
+            validator: new Validator(this.config.validation),
+        };
+    }
+    getMetrics() {
+        return this.telemetry.getSummary();
+    }
+    healthCheck() {
+        return Promise.resolve({ status: "healthy" });
     }
-  }
-
-  _initializeStages() {
-    return {
-      structural: new StructuralCleaner(this.config.structural),
-      semantic: new SemanticFilter(this.config.semantic),
-      normalizer: new Normalizer(this.config.normalization),
-      chunker: new Chunker(this.config.chunking),
-      metadata: new MetadataAnnotator(this.config.metadata),
-      validator: new Validator(this.config.validation)
-    };
-  }
-
-  getMetrics() {
-    return this.telemetry.getSummary();
-  }
-
-  async healthCheck() {
-    return { status: 'healthy' };
-  }
 }
-
 export default Scrubber;