@yamo/memory-mesh 3.0.0 → 3.0.2

package/lib/scrubber/index.ts

@@ -1,7 +1,6 @@
+// @ts-nocheck
 /**
  * YAMO Scrubber Module
  * PII and sensitive data sanitization
  */
-
 export { Scrubber } from "./scrubber.js";
-export { ScrubberConfig } from "./config/defaults.js";

package/lib/scrubber/scrubber.d.ts

@@ -8,43 +8,26 @@ import { Normalizer } from "./stages/normalizer.js";
 import { Chunker } from "./stages/chunker.js";
 import { MetadataAnnotator } from "./stages/metadata-annotator.js";
 import { Validator } from "./stages/validator.js";
-import { ScrubberTelemetry, TelemetrySummary, StageSummary } from "./telemetry.js";
-import { ScrubberConfig } from "./config/defaults.js";
-export interface ScrubberDocument {
-    content: string;
-    source: string;
-    type: string;
-}
-export interface Chunk {
-    text: string;
-    [key: string]: any;
-}
-export interface ScrubberResult {
-    chunks: Chunk[];
-    metadata: {
-        source: string;
-        type: string;
-        processingTimestamp: string;
-        [key: string]: any;
-    };
-    telemetry: Partial<Record<string, StageSummary>> & {
-        totalDuration?: number;
-    };
-    success?: boolean;
-    error?: string;
-}
 export declare class Scrubber {
-    config: ScrubberConfig;
+    config: any;
     stages: any;
-    telemetry: ScrubberTelemetry;
-    constructor(config?: Partial<ScrubberConfig>);
+    telemetry: any;
+    constructor(config?: {});
     /**
      * Main entry point - process a raw document
      * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
      * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
      */
-    process(document: ScrubberDocument): Promise<ScrubberResult>;
-    _executeStage<T>(stageName: string, stageFn: () => Promise<T> | T): Promise<T>;
+    process(document: any): Promise<{
+        chunks: any[];
+        metadata: {
+            source: any;
+            type: any;
+            processingTimestamp: string;
+        };
+        telemetry: {};
+    }>;
+    _executeStage(stageName: any, stageFn: any): Promise<any>;
     _initializeStages(): {
         structural: StructuralCleaner;
         semantic: SemanticFilter;
@@ -53,7 +36,7 @@ export declare class Scrubber {
         metadata: MetadataAnnotator;
         validator: Validator;
     };
-    getMetrics(): TelemetrySummary;
+    getMetrics(): any;
     healthCheck(): Promise<{
         status: string;
     }>;

package/lib/scrubber/scrubber.js

@@ -1,3 +1,4 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Main Orchestrator
  * @module smora/scrubber/scrubber

package/lib/scrubber/scrubber.ts

@@ -1,168 +1,109 @@
+// @ts-nocheck
 /**
  * S-MORA Layer 0 Scrubber - Main Orchestrator
  * @module smora/scrubber/scrubber
  */
-
 import { StructuralCleaner } from "./stages/structural-cleaner.js";
 import { SemanticFilter } from "./stages/semantic-filter.js";
 import { Normalizer } from "./stages/normalizer.js";
 import { Chunker } from "./stages/chunker.js";
 import { MetadataAnnotator } from "./stages/metadata-annotator.js";
 import { Validator } from "./stages/validator.js";
-import {
-  ScrubberTelemetry,
-  TelemetrySummary,
-  StageSummary,
-} from "./telemetry.js";
+import { ScrubberTelemetry, } from "./telemetry.js";
 // import { ScrubberError } from './errors/scrubber-error'; // Assuming this exists or I should check
-import { defaultScrubberConfig, ScrubberConfig } from "./config/defaults.js";
-
-// Interfaces for input/output
-export interface ScrubberDocument {
-  content: string;
-  source: string;
-  type: string;
-}
-
-export interface Chunk {
-  text: string;
-  [key: string]: any;
-}
-
-export interface ScrubberResult {
-  chunks: Chunk[];
-  metadata: {
-    source: string;
-    type: string;
-    processingTimestamp: string;
-    [key: string]: any;
-  };
-  telemetry: Partial<Record<string, StageSummary>> & { totalDuration?: number };
-  success?: boolean;
-  error?: string;
-}
-
+import { defaultScrubberConfig } from "./config/defaults.js";
 export class Scrubber {
-  config: ScrubberConfig;
-  stages: any; // Using any for stages as they are not yet converted
-  telemetry: ScrubberTelemetry;
-
-  constructor(config: Partial<ScrubberConfig> = {}) {
-    this.config = { ...defaultScrubberConfig, ...config };
-    this.stages = this._initializeStages();
-    this.telemetry = new ScrubberTelemetry();
-  }
-
-  /**
-   * Main entry point - process a raw document
-   * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
-   * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
-   */
-  async process(document: ScrubberDocument): Promise<ScrubberResult> {
-    const startTime = Date.now();
-    const result: ScrubberResult = {
-      chunks: [],
-      metadata: {
-        source: document.source,
-        type: document.type,
-        processingTimestamp: new Date().toISOString(),
-      },
-      telemetry: {},
-    };
-
-    try {
-      // If disabled, return empty chunks
-      if (!this.config.enabled) {
-        result.success = true;
-        result.telemetry.totalDuration = Date.now() - startTime;
-        return result;
-      }
-
-      // Stage 1: Structural Cleaning
-      const cleaned = await this._executeStage("structural", () =>
-        this.stages.structural.clean(document.content),
-      );
-      result.telemetry.structural = this.telemetry.getStageStats("structural");
-
-      // Stage 2: Semantic Filtering
-      const filtered = await this._executeStage("semantic", () =>
-        this.stages.semantic.filter(cleaned),
-      );
-      result.telemetry.semantic = this.telemetry.getStageStats("semantic");
-
-      // Stage 3: Normalization
-      const normalized = await this._executeStage("normalization", () =>
-        this.stages.normalizer.normalize(filtered),
-      );
-      result.telemetry.normalization =
-        this.telemetry.getStageStats("normalization");
-
-      // Stage 4: Chunking
-      const chunks = await this._executeStage("chunking", () =>
-        this.stages.chunker.chunk(normalized),
-      );
-      result.telemetry.chunking = this.telemetry.getStageStats("chunking");
-
-      // Stage 5: Metadata Annotation
-      const annotated = await this._executeStage("metadata", () =>
-        this.stages.metadata.annotate(chunks, document),
-      );
-      result.telemetry.metadata = this.telemetry.getStageStats("metadata");
-
-      // Stage 6: Validation
-      result.chunks = await this._executeStage("validation", () =>
-        this.stages.validator.validate(annotated),
-      );
-      result.telemetry.validation = this.telemetry.getStageStats("validation");
-
-      result.telemetry.totalDuration = Date.now() - startTime;
-      result.success = true;
-
-      return result;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      result.success = false;
-      result.error = message;
-      result.telemetry.totalDuration = Date.now() - startTime;
-      return result;
+    config;
+    stages; // Using any for stages as they are not yet converted
+    telemetry;
+    constructor(config = {}) {
+        this.config = { ...defaultScrubberConfig, ...config };
+        this.stages = this._initializeStages();
+        this.telemetry = new ScrubberTelemetry();
+    }
+    /**
+     * Main entry point - process a raw document
+     * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+     * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+     */
+    async process(document) {
+        const startTime = Date.now();
+        const result = {
+            chunks: [],
+            metadata: {
+                source: document.source,
+                type: document.type,
+                processingTimestamp: new Date().toISOString(),
+            },
+            telemetry: {},
+        };
+        try {
+            // If disabled, return empty chunks
+            if (!this.config.enabled) {
+                result.success = true;
+                result.telemetry.totalDuration = Date.now() - startTime;
+                return result;
+            }
+            // Stage 1: Structural Cleaning
+            const cleaned = await this._executeStage("structural", () => this.stages.structural.clean(document.content));
+            result.telemetry.structural = this.telemetry.getStageStats("structural");
+            // Stage 2: Semantic Filtering
+            const filtered = await this._executeStage("semantic", () => this.stages.semantic.filter(cleaned));
+            result.telemetry.semantic = this.telemetry.getStageStats("semantic");
+            // Stage 3: Normalization
+            const normalized = await this._executeStage("normalization", () => this.stages.normalizer.normalize(filtered));
+            result.telemetry.normalization =
+                this.telemetry.getStageStats("normalization");
+            // Stage 4: Chunking
+            const chunks = await this._executeStage("chunking", () => this.stages.chunker.chunk(normalized));
+            result.telemetry.chunking = this.telemetry.getStageStats("chunking");
+            // Stage 5: Metadata Annotation
+            const annotated = await this._executeStage("metadata", () => this.stages.metadata.annotate(chunks, document));
+            result.telemetry.metadata = this.telemetry.getStageStats("metadata");
+            // Stage 6: Validation
+            result.chunks = await this._executeStage("validation", () => this.stages.validator.validate(annotated));
+            result.telemetry.validation = this.telemetry.getStageStats("validation");
+            result.telemetry.totalDuration = Date.now() - startTime;
+            result.success = true;
+            return result;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            result.success = false;
+            result.error = message;
+            result.telemetry.totalDuration = Date.now() - startTime;
+            return result;
+        }
+    }
+    async _executeStage(stageName, stageFn) {
+        const startTime = Date.now();
+        try {
+            const result = await stageFn();
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, true);
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            this.telemetry.recordStage(stageName, duration, false);
+            throw error;
+        }
+    }
+    _initializeStages() {
+        return {
+            structural: new StructuralCleaner(this.config.structural),
+            semantic: new SemanticFilter(this.config.semantic),
+            normalizer: new Normalizer(this.config.normalization),
+            chunker: new Chunker(this.config.chunking),
+            metadata: new MetadataAnnotator(this.config.metadata),
+            validator: new Validator(this.config.validation),
+        };
+    }
+    getMetrics() {
+        return this.telemetry.getSummary();
     }
-  }
-
-  async _executeStage<T>(
-    stageName: string,
-    stageFn: () => Promise<T> | T,
-  ): Promise<T> {
-    const startTime = Date.now();
-    try {
-      const result = await stageFn();
-      const duration = Date.now() - startTime;
-      this.telemetry.recordStage(stageName, duration, true);
-      return result;
-    } catch (error) {
-      const duration = Date.now() - startTime;
-      this.telemetry.recordStage(stageName, duration, false);
-      throw error;
+    healthCheck() {
+        return Promise.resolve({ status: "healthy" });
     }
-  }
-
-  _initializeStages() {
-    return {
-      structural: new StructuralCleaner(this.config.structural),
-      semantic: new SemanticFilter(this.config.semantic),
-      normalizer: new Normalizer(this.config.normalization),
-      chunker: new Chunker(this.config.chunking),
-      metadata: new MetadataAnnotator(this.config.metadata),
-      validator: new Validator(this.config.validation),
-    };
-  }
-
-  getMetrics(): TelemetrySummary {
-    return this.telemetry.getSummary();
-  }
-
-  healthCheck(): Promise<{ status: string }> {
-    return Promise.resolve({ status: "healthy" });
-  }
 }
-
 export default Scrubber;

package/lib/scrubber/stages/chunker.d.ts

@@ -1,13 +1,25 @@
 /**
- * Type definitions for chunker.js
+ * S-MORA Layer 0 Scrubber - Stage 4: Chunking
+ * @module smora/scrubber/stages/chunker
  */
-
-export interface ChunkerConfig {
-  maxSize?: number;
-  [key: string]: any;
-}
-
-export class Chunker {
-  constructor(config?: ChunkerConfig);
-  chunk(content: string): Promise<string[]>;
+export declare class Chunker {
+    constructor(config: any);
+    /**
+     * Split content into chunks
+     * @param {string} content - Normalized content
+     * @returns {Promise<Array>} - Array of chunks with metadata
+     */
+    chunk(content: any): Promise<{
+        index: number;
+        text: any;
+        metadata: {
+            tokens: any;
+            heading: any;
+            position: number;
+        };
+    }[]>;
+    _isHeading(line: any): boolean;
+    _shouldStartNewChunk(currentChunk: any, para: any, paraTokens: any, isHeading: any): boolean;
+    _extractInitialHeading(content: any): any;
+    _extractHeadingText(headingLine: any): any;
 }

package/lib/scrubber/stages/chunker.js

@@ -0,0 +1,86 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 4: Chunking
+ * @module smora/scrubber/stages/chunker
+ */
+import { TokenCounter } from '../utils/token-counter.js';
+import { ScrubberError } from '../errors/scrubber-error.js';
+export class Chunker {
+    constructor(config) {
+        this.config = config;
+        this.tokenCounter = new TokenCounter();
+    }
+    /**
+     * Split content into chunks
+     * @param {string} content - Normalized content
+     * @returns {Promise<Array>} - Array of chunks with metadata
+     */
+    async chunk(content) {
+        try {
+            const chunks = [];
+            const paragraphs = content.split(/\n\n+/);
+            let currentChunk = {
+                text: '',
+                tokens: 0,
+                heading: this._extractInitialHeading(content)
+            };
+            for (const para of paragraphs) {
+                const isHeading = this._isHeading(para);
+                const paraTokens = this.tokenCounter.count(para);
+                if (this._shouldStartNewChunk(currentChunk, para, paraTokens, isHeading)) {
+                    if (currentChunk.tokens >= this.config.minTokens) {
+                        chunks.push({ ...currentChunk });
+                    }
+                    currentChunk = {
+                        text: '',
+                        tokens: 0,
+                        heading: isHeading ? this._extractHeadingText(para) : currentChunk.heading
+                    };
+                }
+                currentChunk.text += (currentChunk.text ? '\n\n' : '') + para;
+                currentChunk.tokens += paraTokens;
+                if (currentChunk.tokens > this.config.hardMaxTokens) {
+                    chunks.push({ ...currentChunk });
+                    currentChunk = { text: '', tokens: 0, heading: null };
+                }
+            }
+            if (currentChunk.tokens >= this.config.minTokens) {
+                chunks.push(currentChunk);
+            }
+            return chunks.map((chunk, index) => ({
+                index,
+                text: chunk.text.trim(),
+                metadata: {
+                    tokens: chunk.tokens,
+                    heading: chunk.heading,
+                    position: index
+                }
+            }));
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new ScrubberError(`Failed to chunk content: ${message}`, { stage: 'chunker', originalError: error });
+        }
+    }
+    _isHeading(line) {
+        return /^#{1,6}\s/.test(line);
+    }
+    _shouldStartNewChunk(currentChunk, para, paraTokens, isHeading) {
+        if (this.config.splitOnHeadings && isHeading && currentChunk.tokens > 0) {
+            return true;
+        }
+        const wouldExceed = (currentChunk.tokens + paraTokens) > this.config.maxTokens;
+        if (wouldExceed && currentChunk.tokens > 0) {
+            return true;
+        }
+        return false;
+    }
+    _extractInitialHeading(content) {
+        const match = content.match(/^#{1,6}\s+(.+)$/m);
+        return match ? match[1] : null;
+    }
+    _extractHeadingText(headingLine) {
+        const match = headingLine.match(/^#{1,6}\s+(.+)$/);
+        return match ? match[1] : null;
+    }
+}

package/lib/scrubber/stages/chunker.ts

@@ -0,0 +1,104 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 4: Chunking
+ * @module smora/scrubber/stages/chunker
+ */
+
+import { TokenCounter } from '../utils/token-counter.js';
+import { ChunkingError, ScrubberError } from '../errors/scrubber-error.js';
+
+export class Chunker {
+  constructor(config) {
+    this.config = config;
+    this.tokenCounter = new TokenCounter();
+  }
+
+  /**
+   * Split content into chunks
+   * @param {string} content - Normalized content
+   * @returns {Promise<Array>} - Array of chunks with metadata
+   */
+  async chunk(content) {
+    try {
+      const chunks = [];
+      const paragraphs = content.split(/\n\n+/);
+
+      let currentChunk = {
+        text: '',
+        tokens: 0,
+        heading: this._extractInitialHeading(content)
+      };
+
+      for (const para of paragraphs) {
+        const isHeading = this._isHeading(para);
+        const paraTokens = this.tokenCounter.count(para);
+
+        if (this._shouldStartNewChunk(currentChunk, para, paraTokens, isHeading)) {
+          if (currentChunk.tokens >= this.config.minTokens) {
+            chunks.push({ ...currentChunk });
+          }
+          currentChunk = {
+            text: '',
+            tokens: 0,
+            heading: isHeading ? this._extractHeadingText(para) : currentChunk.heading
+          };
+        }
+
+        currentChunk.text += (currentChunk.text ? '\n\n' : '') + para;
+        currentChunk.tokens += paraTokens;
+
+        if (currentChunk.tokens > this.config.hardMaxTokens) {
+          chunks.push({ ...currentChunk });
+          currentChunk = { text: '', tokens: 0, heading: null };
+        }
+      }
+
+      if (currentChunk.tokens >= this.config.minTokens) {
+        chunks.push(currentChunk);
+      }
+
+      return chunks.map((chunk, index) => ({
+        index,
+        text: chunk.text.trim(),
+        metadata: {
+          tokens: chunk.tokens,
+          heading: chunk.heading,
+          position: index
+        }
+      }));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ScrubberError(
+        `Failed to chunk content: ${message}`,
+        { stage: 'chunker', originalError: error }
+      );
+    }
+  }
+
+  _isHeading(line) {
+    return /^#{1,6}\s/.test(line);
+  }
+
+  _shouldStartNewChunk(currentChunk, para, paraTokens, isHeading) {
+    if (this.config.splitOnHeadings && isHeading && currentChunk.tokens > 0) {
+      return true;
+    }
+
+    const wouldExceed = (currentChunk.tokens + paraTokens) > this.config.maxTokens;
+    if (wouldExceed && currentChunk.tokens > 0) {
+      return true;
+    }
+
+    return false;
+  }
+
+  _extractInitialHeading(content) {
+    const match = content.match(/^#{1,6}\s+(.+)$/m);
+    return match ? match[1] : null;
+  }
+
+  _extractHeadingText(headingLine) {
+    const match = headingLine.match(/^#{1,6}\s+(.+)$/);
+    return match ? match[1] : null;
+  }
+}

package/lib/scrubber/stages/metadata-annotator.d.ts

@@ -1,18 +1,17 @@
 /**
- * Type definitions for metadata-annotator.js
+ * S-MORA Layer 0 Scrubber - Stage 5: Metadata Annotation
+ * @module smora/scrubber/stages/metadata-annotator
  */
-
-export interface AnnotatorConfig {
-  includeTimestamp?: boolean;
-  [key: string]: any;
-}
-
-export interface AnnotatedData {
-  content: string;
-  metadata: Record<string, any>;
-}
-
-export class MetadataAnnotator {
-  constructor(config?: AnnotatorConfig);
-  annotate(content: string): Promise<AnnotatedData>;
+export declare class MetadataAnnotator {
+    constructor(config: any);
+    /**
+     * Add metadata to chunks
+     * @param {Array} chunks - Array of chunks
+     * @param {Object} document - Original document metadata
+     * @returns {Promise<Array>} - Annotated chunks
+     */
+    annotate(chunks: any, document: any): Promise<any>;
+    _extractSection(chunk: any): any;
+    _buildHeadingPath(chunk: any, currentPath: any): any[];
+    _isSubHeading(heading1: any, heading2: any): boolean;
 }

package/lib/scrubber/stages/metadata-annotator.js

@@ -0,0 +1,64 @@
+// @ts-nocheck
+/**
+ * S-MORA Layer 0 Scrubber - Stage 5: Metadata Annotation
+ * @module smora/scrubber/stages/metadata-annotator
+ */
+import { HashUtil } from '../utils/hash.js';
+export class MetadataAnnotator {
+    constructor(config) {
+        this.config = config;
+        this.hashUtil = new HashUtil();
+    }
+    /**
+     * Add metadata to chunks
+     * @param {Array} chunks - Array of chunks
+     * @param {Object} document - Original document metadata
+     * @returns {Promise<Array>} - Annotated chunks
+     */
+    async annotate(chunks, document) {
+        const headingPath = [];
+        return chunks.map((chunk, index) => {
+            const metadata = {
+                ...chunk.metadata,
+                source: this.config.addSource ? document.source : undefined,
+                doc_type: this.config.addSource ? document.type : undefined,
+                section: this.config.addSection ? this._extractSection(chunk) : undefined,
+                heading_path: this.config.addHeadingPath ?
+                    this._buildHeadingPath(chunk, headingPath) :
+                    undefined,
+                ingestion_timestamp: this.config.addTimestamp ?
+                    new Date().toISOString() :
+                    undefined,
+                hash: this.config.addHash ?
+                    this.hashUtil.hash(chunk.text) :
+                    undefined
+            };
+            return {
+                ...chunk,
+                metadata: Object.fromEntries(Object.entries(metadata).filter(([_, v]) => v !== undefined))
+            };
+        });
+    }
+    _extractSection(chunk) {
+        if (chunk.metadata.heading) {
+            return chunk.metadata.heading;
+        }
+        return 'unnamed-section';
+    }
+    _buildHeadingPath(chunk, currentPath) {
+        const heading = chunk.metadata.heading;
+        if (heading && heading !== currentPath[currentPath.length - 1]) {
+            if (currentPath.length === 0 || this._isSubHeading(heading, currentPath[currentPath.length - 1])) {
+                currentPath.push(heading);
+            }
+            else {
+                currentPath.length = 0;
+                currentPath.push(heading);
+            }
+        }
+        return [...currentPath];
+    }
+    _isSubHeading(heading1, heading2) {
+        return heading1.length > heading2.length;
+    }
+}