@yamo/memory-mesh 2.3.2 → 3.0.0

package/lib/scrubber/telemetry.ts

@@ -0,0 +1,99 @@
+/**
+ * S-MORA Layer 0 Scrubber Telemetry Collection
+ * @module smora/scrubber/telemetry
+ */
+
+export interface StageStats {
+  count: number;
+  totalTime: number;
+  errors: number;
+}
+
+export interface StageSummary {
+  count: number;
+  avgTime: number;
+  totalTime: number;
+  errors: number;
+}
+
+export interface TelemetrySummary {
+  stages: Record<string, StageStats>;
+  performance: {
+    structural: number;
+    semantic: number;
+    normalization: number;
+    chunking: number;
+    metadata: number;
+    validation: number;
+    total: number;
+  };
+}
+
+export class ScrubberTelemetry {
+  stats: Record<string, StageStats>;
+
+  constructor() {
+    this.stats = {
+      structural: { count: 0, totalTime: 0, errors: 0 },
+      semantic: { count: 0, totalTime: 0, errors: 0 },
+      normalization: { count: 0, totalTime: 0, errors: 0 },
+      chunking: { count: 0, totalTime: 0, errors: 0 },
+      metadata: { count: 0, totalTime: 0, errors: 0 },
+      validation: { count: 0, totalTime: 0, errors: 0 },
+    };
+  }
+
+  recordStage(stage: string, duration: number, success: boolean = true): void {
+    if (!this.stats[stage]) {
+      this.stats[stage] = { count: 0, totalTime: 0, errors: 0 };
+    }
+    this.stats[stage].count++;
+    this.stats[stage].totalTime += duration;
+    if (!success) {
+      this.stats[stage].errors++;
+    }
+  }
+
+  getStageStats(stage: string): StageSummary {
+    const stats = this.stats[stage] || { count: 0, totalTime: 0, errors: 0 };
+    return {
+      count: stats.count,
+      avgTime: stats.count > 0 ? stats.totalTime / stats.count : 0,
+      totalTime: stats.totalTime,
+      errors: stats.errors,
+    };
+  }
+
+  getSummary(): TelemetrySummary {
+    return {
+      stages: this.stats,
+      performance: {
+        structural: this.stats.structural.totalTime,
+        semantic: this.stats.semantic.totalTime,
+        normalization: this.stats.normalization.totalTime,
+        chunking: this.stats.chunking.totalTime,
+        metadata: this.stats.metadata.totalTime,
+        validation: this.stats.validation.totalTime,
+        total: Object.values(this.stats).reduce(
+          (sum, s) => sum + s.totalTime,
+          0,
+        ),
+      },
+    };
+  }
+
+  reset(): void {
+    Object.keys(this.stats).forEach((key) => {
+      this.stats[key] = { count: 0, totalTime: 0, errors: 0 };
+    });
+  }
+
+  assertPerformanceBudget(budget: number = 10): void {
+    const summary = this.getSummary();
+    if (summary.performance.total > budget) {
+      throw new Error(
+        `Performance budget exceeded: ${summary.performance.total}ms > ${budget}ms`,
+      );
+    }
+  }
+}

package/lib/utils/logger.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Structured Logger using Pino
+ * Provides centralized logging with metadata support, PII redaction, and environment-based formatting
+ */
+import pino from "pino";
+/**
+ * Main logger instance with configuration
+ */
+export declare const logger: pino.Logger<never, boolean>;
+/**
+ * Create a child logger with module-specific context
+ * @param module - Module name for logging context
+ * @returns Child logger instance
+ *
+ * @example
+ * const logger = createLogger('kernel');
+ * logger.info({ action: 'boot' }, 'Kernel starting');
+ */
+export declare function createLogger(module: string): pino.Logger;
+/**
+ * Log levels and their usage:
+ * - fatal: Application crash, immediate attention required
+ * - error: Errors that need investigation
+ * - warn: Degraded state, potential issues
+ * - info: Important operational events (default)
+ * - debug: Detailed diagnostic information
+ * - trace: Very verbose, performance metrics
+ */
+export type { Logger } from "pino";

package/lib/utils/logger.js

@@ -0,0 +1,64 @@
+/**
+ * Structured Logger using Pino
+ * Provides centralized logging with metadata support, PII redaction, and environment-based formatting
+ */
+import pino from "pino";
+// Determine if running in production
+const isProduction = process.env.NODE_ENV === "production";
+const logLevel = (process.env.LOG_LEVEL || "warn");
+/**
+ * Main logger instance with configuration
+ */
+export const logger = pino({
+    level: logLevel,
+    // Pretty printing in development, JSON in production
+    transport: !isProduction
+        ? {
+            target: "pino-pretty",
+            options: {
+                colorize: true,
+                translateTime: "HH:MM:ss.l",
+                ignore: "pid,hostname,app,version",
+                singleLine: false,
+            },
+        }
+        : undefined,
+    // Base fields included in all logs
+    base: {
+        app: "yamo-os",
+        version: process.env.npm_package_version || "1.1.0",
+    },
+    // Redact sensitive fields from logs
+    redact: {
+        paths: [
+            "apiKey",
+            "password",
+            "token",
+            "secret",
+            "key",
+            "*.apiKey",
+            "*.password",
+            "*.token",
+            "*.secret",
+            "*.key",
+            "OPENAI_API_KEY",
+            "ANTHROPIC_API_KEY",
+            "ZAI_API_KEY",
+        ],
+        censor: "[REDACTED]",
+    },
+    // Timestamp in ISO format
+    timestamp: pino.stdTimeFunctions.isoTime,
+});
+/**
+ * Create a child logger with module-specific context
+ * @param module - Module name for logging context
+ * @returns Child logger instance
+ *
+ * @example
+ * const logger = createLogger('kernel');
+ * logger.info({ action: 'boot' }, 'Kernel starting');
+ */
+export function createLogger(module) {
+    return logger.child({ module });
+}

package/lib/utils/logger.ts

@@ -0,0 +1,85 @@
+/**
+ * Structured Logger using Pino
+ * Provides centralized logging with metadata support, PII redaction, and environment-based formatting
+ */
+
+import pino from "pino";
+
+// Determine if running in production
+const isProduction = process.env.NODE_ENV === "production";
+const logLevel = (process.env.LOG_LEVEL || "warn") as pino.Level;
+
+/**
+ * Main logger instance with configuration
+ */
+export const logger = pino({
+  level: logLevel,
+
+  // Pretty printing in development, JSON in production
+  transport: !isProduction
+    ? {
+        target: "pino-pretty",
+        options: {
+          colorize: true,
+          translateTime: "HH:MM:ss.l",
+          ignore: "pid,hostname,app,version",
+          singleLine: false,
+        },
+      }
+    : undefined,
+
+  // Base fields included in all logs
+  base: {
+    app: "yamo-os",
+    version: process.env.npm_package_version || "1.1.0",
+  },
+
+  // Redact sensitive fields from logs
+  redact: {
+    paths: [
+      "apiKey",
+      "password",
+      "token",
+      "secret",
+      "key",
+      "*.apiKey",
+      "*.password",
+      "*.token",
+      "*.secret",
+      "*.key",
+      "OPENAI_API_KEY",
+      "ANTHROPIC_API_KEY",
+      "ZAI_API_KEY",
+    ],
+    censor: "[REDACTED]",
+  },
+
+  // Timestamp in ISO format
+  timestamp: pino.stdTimeFunctions.isoTime,
+});
+
+/**
+ * Create a child logger with module-specific context
+ * @param module - Module name for logging context
+ * @returns Child logger instance
+ *
+ * @example
+ * const logger = createLogger('kernel');
+ * logger.info({ action: 'boot' }, 'Kernel starting');
+ */
+export function createLogger(module: string): pino.Logger {
+  return logger.child({ module });
+}
+
+/**
+ * Log levels and their usage:
+ * - fatal: Application crash, immediate attention required
+ * - error: Errors that need investigation
+ * - warn: Degraded state, potential issues
+ * - info: Important operational events (default)
+ * - debug: Detailed diagnostic information
+ * - trace: Very verbose, performance metrics
+ */
+
+// Re-export types for convenience
+export type { Logger } from "pino";

package/lib/utils/skill-metadata.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Structured skill-identity extraction.
+ *
+ * Reads ONLY the YAML frontmatter block (between --- delimiters) or
+ * root-level identity declarations in legacy v0.4 compact format.
+ * The body of .yamo files is LLM-interpreted and MUST NOT be machine-parsed.
+ */
+export interface SkillIdentity {
+    name: string;
+    intent: string;
+    description: string;
+}
+/**
+ * Extract identity fields (name, intent, description) from .yamo content.
+ *
+ * Priority:
+ *   1. YAML frontmatter (--- … --- block at file start)
+ *   2. Legacy v0.4 root-level compact declarations
+ *   3. Content-hash fallback — deterministic and idempotent
+ */
+export declare function extractSkillIdentity(content: string): SkillIdentity;
+/**
+ * Extract tags from YAML frontmatter.
+ * Returns an array of tag strings, or empty array if no tags found.
+ *
+ * Tags are expected in the format:
+ *   tags: tag1, tag2, tag3
+ *
+ * This function ONLY reads the YAML frontmatter block and does NOT parse
+ * the skill body, following the same safety constraints as extractSkillIdentity.
+ */
+export declare function extractSkillTags(content: string): string[];

package/lib/utils/skill-metadata.js

@@ -0,0 +1,132 @@
+/**
+ * Structured skill-identity extraction.
+ *
+ * Reads ONLY the YAML frontmatter block (between --- delimiters) or
+ * root-level identity declarations in legacy v0.4 compact format.
+ * The body of .yamo files is LLM-interpreted and MUST NOT be machine-parsed.
+ */
+import crypto from "crypto";
+/** Fields safe to extract for indexing and display. */
+const IDENTITY_FIELDS = new Set(["name", "intent", "description"]);
+/** Pre-computed regexes for legacy root-level identity declarations. */
+const LEGACY_REGEXES = {
+    name: /^name[;:]\s*([^;\n]+);?/m,
+    intent: /^intent[;:]\s*([^;\n]+);?/m,
+    description: /^description[;:]\s*([^;\n]+);?/m,
+};
+/**
+ * Parse flat key: value or key; value lines from a text block.
+ * Only extracts whitelisted identity fields.
+ */
+function parseFlatBlock(block) {
+    const result = {};
+    for (const line of block.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith("#")) {
+            continue;
+        }
+        const colonIdx = trimmed.indexOf(":");
+        const semiIdx = trimmed.indexOf(";");
+        let sepIdx;
+        if (colonIdx > 0 && semiIdx > 0) {
+            sepIdx = Math.min(colonIdx, semiIdx);
+        }
+        else if (colonIdx > 0) {
+            sepIdx = colonIdx;
+        }
+        else if (semiIdx > 0) {
+            sepIdx = semiIdx;
+        }
+        else {
+            continue;
+        }
+        const key = trimmed.substring(0, sepIdx).trim();
+        let value = trimmed.substring(sepIdx + 1).trim();
+        // Strip trailing semicolon (legacy compact format)
+        if (value.endsWith(";")) {
+            value = value.slice(0, -1).trim();
+        }
+        if (IDENTITY_FIELDS.has(key) && value) {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+/**
+ * Extract identity fields (name, intent, description) from .yamo content.
+ *
+ * Priority:
+ *   1. YAML frontmatter (--- … --- block at file start)
+ *   2. Legacy v0.4 root-level compact declarations
+ *   3. Content-hash fallback — deterministic and idempotent
+ */
+export function extractSkillIdentity(content) {
+    // 1. YAML frontmatter
+    if (content.startsWith("---")) {
+        const endIdx = content.indexOf("---", 3);
+        if (endIdx !== -1) {
+            const fields = parseFlatBlock(content.substring(3, endIdx));
+            if (fields.name) {
+                return {
+                    name: fields.name,
+                    intent: fields.intent || "general_procedure",
+                    description: fields.description || "",
+                };
+            }
+        }
+    }
+    // 2. Legacy v0.4 root-level identity declarations.
+    // Safe: name/intent/description do not appear as body section headers.
+    const legacyFields = {};
+    for (const field of IDENTITY_FIELDS) {
+        const match = content.match(LEGACY_REGEXES[field]);
+        if (match) {
+            legacyFields[field] = match[1].trim();
+        }
+    }
+    if (legacyFields.name) {
+        return {
+            name: legacyFields.name,
+            intent: legacyFields.intent || "general_procedure",
+            description: legacyFields.description || "",
+        };
+    }
+    // 3. Content-hash fallback
+    const shortHash = crypto
+        .createHash("sha256")
+        .update(content)
+        .digest("hex")
+        .substring(0, 12);
+    return {
+        name: `Unnamed_${shortHash}`,
+        intent: "general_procedure",
+        description: "",
+    };
+}
+/**
+ * Extract tags from YAML frontmatter.
+ * Returns an array of tag strings, or empty array if no tags found.
+ *
+ * Tags are expected in the format:
+ *   tags: tag1, tag2, tag3
+ *
+ * This function ONLY reads the YAML frontmatter block and does NOT parse
+ * the skill body, following the same safety constraints as extractSkillIdentity.
+ */
+export function extractSkillTags(content) {
+    // Only parse YAML frontmatter (between --- delimiters)
+    if (content.startsWith("---")) {
+        const endIdx = content.indexOf("---", 3);
+        if (endIdx !== -1) {
+            const frontmatter = content.substring(3, endIdx);
+            const tagsMatch = frontmatter.match(/^tags:\s*(.+)$/m);
+            if (tagsMatch) {
+                return tagsMatch[1]
+                    .split(",")
+                    .map((t) => t.trim())
+                    .filter((t) => t.length > 0);
+            }
+        }
+    }
+    return [];
+}

package/lib/utils/skill-metadata.ts

@@ -0,0 +1,147 @@
+/**
+ * Structured skill-identity extraction.
+ *
+ * Reads ONLY the YAML frontmatter block (between --- delimiters) or
+ * root-level identity declarations in legacy v0.4 compact format.
+ * The body of .yamo files is LLM-interpreted and MUST NOT be machine-parsed.
+ */
+
+import crypto from "crypto";
+
+export interface SkillIdentity {
+  name: string;
+  intent: string;
+  description: string;
+}
+
+/** Fields safe to extract for indexing and display. */
+const IDENTITY_FIELDS = new Set(["name", "intent", "description"]);
+
+/** Pre-computed regexes for legacy root-level identity declarations. */
+const LEGACY_REGEXES: Record<string, RegExp> = {
+  name: /^name[;:]\s*([^;\n]+);?/m,
+  intent: /^intent[;:]\s*([^;\n]+);?/m,
+  description: /^description[;:]\s*([^;\n]+);?/m,
+};
+
+/**
+ * Parse flat key: value or key; value lines from a text block.
+ * Only extracts whitelisted identity fields.
+ */
+function parseFlatBlock(block: string): Record<string, string> {
+  const result: Record<string, string> = {};
+  for (const line of block.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) {
+      continue;
+    }
+
+    const colonIdx = trimmed.indexOf(":");
+    const semiIdx = trimmed.indexOf(";");
+    let sepIdx: number;
+    if (colonIdx > 0 && semiIdx > 0) {
+      sepIdx = Math.min(colonIdx, semiIdx);
+    } else if (colonIdx > 0) {
+      sepIdx = colonIdx;
+    } else if (semiIdx > 0) {
+      sepIdx = semiIdx;
+    } else {
+      continue;
+    }
+
+    const key = trimmed.substring(0, sepIdx).trim();
+    let value = trimmed.substring(sepIdx + 1).trim();
+    // Strip trailing semicolon (legacy compact format)
+    if (value.endsWith(";")) {
+      value = value.slice(0, -1).trim();
+    }
+
+    if (IDENTITY_FIELDS.has(key) && value) {
+      result[key] = value;
+    }
+  }
+  return result;
+}
+
+/**
+ * Extract identity fields (name, intent, description) from .yamo content.
+ *
+ * Priority:
+ *   1. YAML frontmatter (--- … --- block at file start)
+ *   2. Legacy v0.4 root-level compact declarations
+ *   3. Content-hash fallback — deterministic and idempotent
+ */
+export function extractSkillIdentity(content: string): SkillIdentity {
+  // 1. YAML frontmatter
+  if (content.startsWith("---")) {
+    const endIdx = content.indexOf("---", 3);
+    if (endIdx !== -1) {
+      const fields = parseFlatBlock(content.substring(3, endIdx));
+      if (fields.name) {
+        return {
+          name: fields.name,
+          intent: fields.intent || "general_procedure",
+          description: fields.description || "",
+        };
+      }
+    }
+  }
+
+  // 2. Legacy v0.4 root-level identity declarations.
+  // Safe: name/intent/description do not appear as body section headers.
+  const legacyFields: Record<string, string> = {};
+  for (const field of IDENTITY_FIELDS) {
+    const match = content.match(LEGACY_REGEXES[field]);
+    if (match) {
+      legacyFields[field] = match[1].trim();
+    }
+  }
+  if (legacyFields.name) {
+    return {
+      name: legacyFields.name,
+      intent: legacyFields.intent || "general_procedure",
+      description: legacyFields.description || "",
+    };
+  }
+
+  // 3. Content-hash fallback
+  const shortHash = crypto
+    .createHash("sha256")
+    .update(content)
+    .digest("hex")
+    .substring(0, 12);
+  return {
+    name: `Unnamed_${shortHash}`,
+    intent: "general_procedure",
+    description: "",
+  };
+}
+
+/**
+ * Extract tags from YAML frontmatter.
+ * Returns an array of tag strings, or empty array if no tags found.
+ *
+ * Tags are expected in the format:
+ *   tags: tag1, tag2, tag3
+ *
+ * This function ONLY reads the YAML frontmatter block and does NOT parse
+ * the skill body, following the same safety constraints as extractSkillIdentity.
+ */
+export function extractSkillTags(content: string): string[] {
+  // Only parse YAML frontmatter (between --- delimiters)
+  if (content.startsWith("---")) {
+    const endIdx = content.indexOf("---", 3);
+    if (endIdx !== -1) {
+      const frontmatter = content.substring(3, endIdx);
+      const tagsMatch = frontmatter.match(/^tags:\s*(.+)$/m);
+      if (tagsMatch) {
+        return tagsMatch[1]
+          .split(",")
+          .map((t) => t.trim())
+          .filter((t) => t.length > 0);
+      }
+    }
+  }
+
+  return [];
+}

package/lib/yamo/emitter.d.ts

@@ -0,0 +1,73 @@
+/**
+ * YAMO Emitter - Constructs structured YAMO blocks for auditability
+ *
+ * Based on YAMO Protocol specification:
+ * - Semicolon-terminated key-value pairs
+ * - Agent/Intent/Context/Constraints/Meta/Output structure
+ * - Supports reflect, retain, recall operations
+ *
+ * Reference: Hindsight project's yamo_integration.py
+ */
+export interface ReflectBlockParams {
+    topic?: string;
+    memoryCount: number;
+    agentId?: string;
+    reflection: string;
+    confidence?: number;
+}
+export interface RetainBlockParams {
+    content: string;
+    metadata?: any;
+    id: string;
+    agentId?: string;
+    memoryType?: string;
+}
+export interface RecallBlockParams {
+    query: string;
+    resultCount: number;
+    limit?: number;
+    agentId?: string;
+    searchType?: string;
+}
+export interface DeleteBlockParams {
+    id: string;
+    agentId?: string;
+    reason?: string;
+}
+export interface ValidationResult {
+    valid: boolean;
+    errors: string[];
+}
+/**
+ * YamoEmitter class for building YAMO protocol blocks
+ * YAMO (Yet Another Multi-agent Orchestration) blocks provide
+ * structured reasoning traces for AI agent operations.
+ */
+export declare class YamoEmitter {
+    /**
+     * Build a YAMO block for reflect operation
+     * Reflect operations synthesize insights from existing memories
+     */
+    static buildReflectBlock(params: ReflectBlockParams): string;
+    /**
+     * Build a YAMO block for retain (add) operation
+     * Retain operations store new memories into the system
+     */
+    static buildRetainBlock(params: RetainBlockParams): string;
+    /**
+     * Build a YAMO block for recall (search) operation
+     * Recall operations retrieve memories based on semantic similarity
+     */
+    static buildRecallBlock(params: RecallBlockParams): string;
+    /**
+     * Build a YAMO block for delete operation (optional)
+     * Delete operations remove memories from the system
+     */
+    static buildDeleteBlock(params: DeleteBlockParams): string;
+    /**
+     * Validate a YAMO block structure
+     * Checks for required sections and proper formatting
+     */
+    static validateBlock(yamoBlock: string): ValidationResult;
+}
+export default YamoEmitter;