@yamo/memory-mesh 2.3.2 → 3.0.1

package/lib/scrubber/utils/pattern-matcher.js

@@ -1,63 +1,54 @@
+// @ts-nocheck
 /**
  * Boilerplate Pattern Matching Utilities
  * @module smora/scrubber/utils/pattern-matcher
  */
-
 export class PatternMatcher {
-  constructor() {
-    this.boilerplatePatterns = this._loadDefaultPatterns();
-  }
-
-  _loadDefaultPatterns() {
-    return [
-      // Legal/Footer
-      /©\s*\d{4}/i,
-      /all rights reserved/i,
-      /copyright\s+\d{4}/i,
-
-      // Navigation
-      /^home\s*\|/i,
-      /^navigation\s*:|menu\s*:/i,
-      /sidebar/i,
-
-      // Meta
-      /^last\s+updated?\s*:/i,
-      /cookie\s+policy/i,
-      /privacy\s+policy/i,
-
-      // Auto-generated
-      /^table\s+of\s+contents?$/i,
-      /^contents\s*$/i,
-      /jump\s+to\s+(section|navigation)/i,
-
-      // Strings
-      'home | docs | contact',
-      'skip to main content',
-      'this site uses cookies'
-    ];
-  }
-
-  getBoilerplatePatterns() {
-    return this.boilerplatePatterns;
-  }
-
-  addPattern(pattern) {
-    this.boilerplatePatterns.push(pattern);
-  }
-
-  removePattern(index) {
-    if (index >= 0 && index < this.boilerplatePatterns.length) {
-      this.boilerplatePatterns.splice(index, 1);
+    constructor() {
+        this.boilerplatePatterns = this._loadDefaultPatterns();
     }
-  }
-
-  isBoilerplate(text) {
-    const lowerText = text.toLowerCase().trim();
-    return this.boilerplatePatterns.some(pattern => {
-      if (pattern instanceof RegExp) {
-        return pattern.test(lowerText);
-      }
-      return lowerText.includes(pattern);
-    });
-  }
-}
+    _loadDefaultPatterns() {
+        return [
+            // Legal/Footer
+            /©\s*\d{4}/i,
+            /all rights reserved/i,
+            /copyright\s+\d{4}/i,
+            // Navigation
+            /^home\s*\|/i,
+            /^navigation\s*:|menu\s*:/i,
+            /sidebar/i,
+            // Meta
+            /^last\s+updated?\s*:/i,
+            /cookie\s+policy/i,
+            /privacy\s+policy/i,
+            // Auto-generated
+            /^table\s+of\s+contents?$/i,
+            /^contents\s*$/i,
+            /jump\s+to\s+(section|navigation)/i,
+            // Strings
+            'home | docs | contact',
+            'skip to main content',
+            'this site uses cookies'
+        ];
+    }
+    getBoilerplatePatterns() {
+        return this.boilerplatePatterns;
+    }
+    addPattern(pattern) {
+        this.boilerplatePatterns.push(pattern);
+    }
+    removePattern(index) {
+        if (index >= 0 && index < this.boilerplatePatterns.length) {
+            this.boilerplatePatterns.splice(index, 1);
+        }
+    }
+    isBoilerplate(text) {
+        const lowerText = text.toLowerCase().trim();
+        return this.boilerplatePatterns.some(pattern => {
+            if (pattern instanceof RegExp) {
+                return pattern.test(lowerText);
+            }
+            return lowerText.includes(pattern);
+        });
+    }
+}

package/lib/scrubber/utils/pattern-matcher.ts

@@ -0,0 +1,64 @@
+// @ts-nocheck
+/**
+ * Boilerplate Pattern Matching Utilities
+ * @module smora/scrubber/utils/pattern-matcher
+ */
+
+export class PatternMatcher {
+  constructor() {
+    this.boilerplatePatterns = this._loadDefaultPatterns();
+  }
+
+  _loadDefaultPatterns() {
+    return [
+      // Legal/Footer
+      /©\s*\d{4}/i,
+      /all rights reserved/i,
+      /copyright\s+\d{4}/i,
+
+      // Navigation
+      /^home\s*\|/i,
+      /^navigation\s*:|menu\s*:/i,
+      /sidebar/i,
+
+      // Meta
+      /^last\s+updated?\s*:/i,
+      /cookie\s+policy/i,
+      /privacy\s+policy/i,
+
+      // Auto-generated
+      /^table\s+of\s+contents?$/i,
+      /^contents\s*$/i,
+      /jump\s+to\s+(section|navigation)/i,
+
+      // Strings
+      'home | docs | contact',
+      'skip to main content',
+      'this site uses cookies'
+    ];
+  }
+
+  getBoilerplatePatterns() {
+    return this.boilerplatePatterns;
+  }
+
+  addPattern(pattern) {
+    this.boilerplatePatterns.push(pattern);
+  }
+
+  removePattern(index) {
+    if (index >= 0 && index < this.boilerplatePatterns.length) {
+      this.boilerplatePatterns.splice(index, 1);
+    }
+  }
+
+  isBoilerplate(text) {
+    const lowerText = text.toLowerCase().trim();
+    return this.boilerplatePatterns.some(pattern => {
+      if (pattern instanceof RegExp) {
+        return pattern.test(lowerText);
+      }
+      return lowerText.includes(pattern);
+    });
+  }
+}

package/lib/scrubber/utils/token-counter.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Token Counting Utilities
+ * @module smora/scrubber/utils/token-counter
+ */
+export declare class TokenCounter {
+    /**
+     * Estimate token count (approximation)
+     * @param {string} text - Text to count
+     * @returns {number} - Estimated token count
+     */
+    count(text: any): number;
+    /**
+     * More accurate token count (slower)
+     * @param {string} text - Text to count
+     * @returns {number} - More accurate token count
+     */
+    countAccurate(text: any): any;
+}

package/lib/scrubber/utils/token-counter.js

@@ -1,31 +1,30 @@
+// @ts-nocheck
 /**
  * Token Counting Utilities
  * @module smora/scrubber/utils/token-counter
  */
-
 export class TokenCounter {
-  /**
-   * Estimate token count (approximation)
-   * @param {string} text - Text to count
-   * @returns {number} - Estimated token count
-   */
-  count(text) {
-    // Simple approximation: ~4 characters per token
-    return Math.ceil(text.length / 4);
-  }
-
-  /**
-   * More accurate token count (slower)
-   * @param {string} text - Text to count
-   * @returns {number} - More accurate token count
-   */
-  countAccurate(text) {
-    const words = text.split(/\s+/).filter(w => w.length > 0);
-    let tokens = words.length;
-    const punctuationMatches = text.match(/[.,!?;:]/g);
-    if (punctuationMatches) {
-      tokens += punctuationMatches.length;
+    /**
+     * Estimate token count (approximation)
+     * @param {string} text - Text to count
+     * @returns {number} - Estimated token count
+     */
+    count(text) {
+        // Simple approximation: ~4 characters per token
+        return Math.ceil(text.length / 4);
     }
-    return tokens;
-  }
-}
+    /**
+     * More accurate token count (slower)
+     * @param {string} text - Text to count
+     * @returns {number} - More accurate token count
+     */
+    countAccurate(text) {
+        const words = text.split(/\s+/).filter(w => w.length > 0);
+        let tokens = words.length;
+        const punctuationMatches = text.match(/[.,!?;:]/g);
+        if (punctuationMatches) {
+            tokens += punctuationMatches.length;
+        }
+        return tokens;
+    }
+}

package/lib/scrubber/utils/token-counter.ts

@@ -0,0 +1,32 @@
+// @ts-nocheck
+/**
+ * Token Counting Utilities
+ * @module smora/scrubber/utils/token-counter
+ */
+
+export class TokenCounter {
+  /**
+   * Estimate token count (approximation)
+   * @param {string} text - Text to count
+   * @returns {number} - Estimated token count
+   */
+  count(text) {
+    // Simple approximation: ~4 characters per token
+    return Math.ceil(text.length / 4);
+  }
+
+  /**
+   * More accurate token count (slower)
+   * @param {string} text - Text to count
+   * @returns {number} - More accurate token count
+   */
+  countAccurate(text) {
+    const words = text.split(/\s+/).filter(w => w.length > 0);
+    let tokens = words.length;
+    const punctuationMatches = text.match(/[.,!?;:]/g);
+    if (punctuationMatches) {
+      tokens += punctuationMatches.length;
+    }
+    return tokens;
+  }
+}

package/lib/utils/logger.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Structured Logger using Pino
+ * Provides centralized logging with metadata support, PII redaction, and environment-based formatting
+ */
+import pino from "pino";
+/**
+ * Main logger instance with configuration
+ */
+export declare const logger: pino.Logger<never, boolean>;
+/**
+ * Create a child logger with module-specific context
+ * @param module - Module name for logging context
+ * @returns Child logger instance
+ *
+ * @example
+ * const logger = createLogger('kernel');
+ * logger.info({ action: 'boot' }, 'Kernel starting');
+ */
+export declare function createLogger(module: any): pino.Logger<never, boolean>;

package/lib/utils/logger.js

@@ -0,0 +1,65 @@
+// @ts-nocheck
+/**
+ * Structured Logger using Pino
+ * Provides centralized logging with metadata support, PII redaction, and environment-based formatting
+ */
+import pino from "pino";
+// Determine if running in production
+const isProduction = process.env.NODE_ENV === "production";
+const logLevel = (process.env.LOG_LEVEL || "warn");
+/**
+ * Main logger instance with configuration
+ */
+export const logger = pino({
+    level: logLevel,
+    // Pretty printing in development, JSON in production
+    transport: !isProduction
+        ? {
+            target: "pino-pretty",
+            options: {
+                colorize: true,
+                translateTime: "HH:MM:ss.l",
+                ignore: "pid,hostname,app,version",
+                singleLine: false,
+            },
+        }
+        : undefined,
+    // Base fields included in all logs
+    base: {
+        app: "yamo-os",
+        version: process.env.npm_package_version || "1.1.0",
+    },
+    // Redact sensitive fields from logs
+    redact: {
+        paths: [
+            "apiKey",
+            "password",
+            "token",
+            "secret",
+            "key",
+            "*.apiKey",
+            "*.password",
+            "*.token",
+            "*.secret",
+            "*.key",
+            "OPENAI_API_KEY",
+            "ANTHROPIC_API_KEY",
+            "ZAI_API_KEY",
+        ],
+        censor: "[REDACTED]",
+    },
+    // Timestamp in ISO format
+    timestamp: pino.stdTimeFunctions.isoTime,
+});
+/**
+ * Create a child logger with module-specific context
+ * @param module - Module name for logging context
+ * @returns Child logger instance
+ *
+ * @example
+ * const logger = createLogger('kernel');
+ * logger.info({ action: 'boot' }, 'Kernel starting');
+ */
+export function createLogger(module) {
+    return logger.child({ module });
+}

package/lib/utils/logger.ts

@@ -0,0 +1,65 @@
+// @ts-nocheck
+/**
+ * Structured Logger using Pino
+ * Provides centralized logging with metadata support, PII redaction, and environment-based formatting
+ */
+import pino from "pino";
+// Determine if running in production
+const isProduction = process.env.NODE_ENV === "production";
+const logLevel = (process.env.LOG_LEVEL || "warn");
+/**
+ * Main logger instance with configuration
+ */
+export const logger = pino({
+    level: logLevel,
+    // Pretty printing in development, JSON in production
+    transport: !isProduction
+        ? {
+            target: "pino-pretty",
+            options: {
+                colorize: true,
+                translateTime: "HH:MM:ss.l",
+                ignore: "pid,hostname,app,version",
+                singleLine: false,
+            },
+        }
+        : undefined,
+    // Base fields included in all logs
+    base: {
+        app: "yamo-os",
+        version: process.env.npm_package_version || "1.1.0",
+    },
+    // Redact sensitive fields from logs
+    redact: {
+        paths: [
+            "apiKey",
+            "password",
+            "token",
+            "secret",
+            "key",
+            "*.apiKey",
+            "*.password",
+            "*.token",
+            "*.secret",
+            "*.key",
+            "OPENAI_API_KEY",
+            "ANTHROPIC_API_KEY",
+            "ZAI_API_KEY",
+        ],
+        censor: "[REDACTED]",
+    },
+    // Timestamp in ISO format
+    timestamp: pino.stdTimeFunctions.isoTime,
+});
+/**
+ * Create a child logger with module-specific context
+ * @param module - Module name for logging context
+ * @returns Child logger instance
+ *
+ * @example
+ * const logger = createLogger('kernel');
+ * logger.info({ action: 'boot' }, 'Kernel starting');
+ */
+export function createLogger(module) {
+    return logger.child({ module });
+}

package/lib/utils/skill-metadata.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Extract identity fields (name, intent, description) from .yamo content.
+ *
+ * Priority:
+ *   1. YAML frontmatter (--- … --- block at file start)
+ *   2. Legacy v0.4 root-level compact declarations
+ *   3. Content-hash fallback — deterministic and idempotent
+ */
+export declare function extractSkillIdentity(content: any): {
+    name: any;
+    intent: any;
+    description: any;
+};
+/**
+ * Extract tags from YAML frontmatter.
+ * Returns an array of tag strings, or empty array if no tags found.
+ *
+ * Tags are expected in the format:
+ *   tags: tag1, tag2, tag3
+ *
+ * This function ONLY reads the YAML frontmatter block and does NOT parse
+ * the skill body, following the same safety constraints as extractSkillIdentity.
+ */
+export declare function extractSkillTags(content: any): any;

package/lib/utils/skill-metadata.js

@@ -0,0 +1,133 @@
+// @ts-nocheck
+/**
+ * Structured skill-identity extraction.
+ *
+ * Reads ONLY the YAML frontmatter block (between --- delimiters) or
+ * root-level identity declarations in legacy v0.4 compact format.
+ * The body of .yamo files is LLM-interpreted and MUST NOT be machine-parsed.
+ */
+import crypto from "crypto";
+/** Fields safe to extract for indexing and display. */
+const IDENTITY_FIELDS = new Set(["name", "intent", "description"]);
+/** Pre-computed regexes for legacy root-level identity declarations. */
+const LEGACY_REGEXES = {
+    name: /^name[;:]\s*([^;\n]+);?/m,
+    intent: /^intent[;:]\s*([^;\n]+);?/m,
+    description: /^description[;:]\s*([^;\n]+);?/m,
+};
+/**
+ * Parse flat key: value or key; value lines from a text block.
+ * Only extracts whitelisted identity fields.
+ */
+function parseFlatBlock(block) {
+    const result = {};
+    for (const line of block.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith("#")) {
+            continue;
+        }
+        const colonIdx = trimmed.indexOf(":");
+        const semiIdx = trimmed.indexOf(";");
+        let sepIdx;
+        if (colonIdx > 0 && semiIdx > 0) {
+            sepIdx = Math.min(colonIdx, semiIdx);
+        }
+        else if (colonIdx > 0) {
+            sepIdx = colonIdx;
+        }
+        else if (semiIdx > 0) {
+            sepIdx = semiIdx;
+        }
+        else {
+            continue;
+        }
+        const key = trimmed.substring(0, sepIdx).trim();
+        let value = trimmed.substring(sepIdx + 1).trim();
+        // Strip trailing semicolon (legacy compact format)
+        if (value.endsWith(";")) {
+            value = value.slice(0, -1).trim();
+        }
+        if (IDENTITY_FIELDS.has(key) && value) {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+/**
+ * Extract identity fields (name, intent, description) from .yamo content.
+ *
+ * Priority:
+ *   1. YAML frontmatter (--- … --- block at file start)
+ *   2. Legacy v0.4 root-level compact declarations
+ *   3. Content-hash fallback — deterministic and idempotent
+ */
+export function extractSkillIdentity(content) {
+    // 1. YAML frontmatter
+    if (content.startsWith("---")) {
+        const endIdx = content.indexOf("---", 3);
+        if (endIdx !== -1) {
+            const fields = parseFlatBlock(content.substring(3, endIdx));
+            if (fields.name) {
+                return {
+                    name: fields.name,
+                    intent: fields.intent || "general_procedure",
+                    description: fields.description || "",
+                };
+            }
+        }
+    }
+    // 2. Legacy v0.4 root-level identity declarations.
+    // Safe: name/intent/description do not appear as body section headers.
+    const legacyFields = {};
+    for (const field of IDENTITY_FIELDS) {
+        const match = content.match(LEGACY_REGEXES[field]);
+        if (match) {
+            legacyFields[field] = match[1].trim();
+        }
+    }
+    if (legacyFields.name) {
+        return {
+            name: legacyFields.name,
+            intent: legacyFields.intent || "general_procedure",
+            description: legacyFields.description || "",
+        };
+    }
+    // 3. Content-hash fallback
+    const shortHash = crypto
+        .createHash("sha256")
+        .update(content)
+        .digest("hex")
+        .substring(0, 12);
+    return {
+        name: `Unnamed_${shortHash}`,
+        intent: "general_procedure",
+        description: "",
+    };
+}
+/**
+ * Extract tags from YAML frontmatter.
+ * Returns an array of tag strings, or empty array if no tags found.
+ *
+ * Tags are expected in the format:
+ *   tags: tag1, tag2, tag3
+ *
+ * This function ONLY reads the YAML frontmatter block and does NOT parse
+ * the skill body, following the same safety constraints as extractSkillIdentity.
+ */
+export function extractSkillTags(content) {
+    // Only parse YAML frontmatter (between --- delimiters)
+    if (content.startsWith("---")) {
+        const endIdx = content.indexOf("---", 3);
+        if (endIdx !== -1) {
+            const frontmatter = content.substring(3, endIdx);
+            const tagsMatch = frontmatter.match(/^tags:\s*(.+)$/m);
+            if (tagsMatch) {
+                return tagsMatch[1]
+                    .split(",")
+                    .map((t) => t.trim())
+                    .filter((t) => t.length > 0);
+            }
+        }
+    }
+    return [];
+}