@yamo/memory-mesh 2.0.1

package/lib/reporting/skill-report-generator.js

@@ -0,0 +1,283 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+
+/**
+ * Skill Execution Report Generator
+ *
+ * Generates JSON reports for skill executions, capturing:
+ * - Skill metadata (name, version, type)
+ * - Execution details (duration, status, provider)
+ * - Input/output metrics
+ * - Quality indicators
+ */
+export class SkillReportGenerator {
+    constructor(options = {}) {
+        this.reportsDir = options.reportsDir || this._getReportsDir();
+        this.version = '1.0.0';
+    }
+
+    _getReportsDir() {
+        // @ts-ignore
+        const home = process.env.HOME || process.env.USERPROFILE || process.cwd();
+        return path.join(home, '.yamo', 'reports');
+    }
+
+    /**
+     * Generate a unique report ID
+     * @param {string} sessionId - Session identifier
+     * @returns {string} Report ID
+     */
+    _generateReportId(sessionId) {
+        const timestamp = Date.now();
+        const shortSession = sessionId ? sessionId.substring(0, 8) : 'unknown';
+        return `skill_execution_${timestamp}_${shortSession}`;
+    }
+
+    /**
+     * Extract skill type from file path or name
+     * @param {string} skillName - Name of the skill
+     * @param {string[]} contextFiles - Context files used
+     * @returns {string} Skill type
+     */
+    _getSkillType(skillName, contextFiles = []) {
+        if (skillName === 'LLMClient') return 'direct';
+
+        const skillFile = contextFiles.find(f => f.endsWith('.yamo'));
+        if (!skillFile) return 'unknown';
+
+        if (skillFile.includes('utility/')) return 'utility';
+        if (skillFile.includes('generator/')) return 'generator';
+        if (skillFile.includes('protocol/')) return 'protocol';
+        if (skillFile.includes('system-skills/')) return 'system';
+
+        return 'custom';
+    }
+
+    /**
+     * Parse skill metadata from .yamo file path
+     * @param {string[]} contextFiles - Context files
+     * @returns {Object} Skill metadata
+     */
+    _parseSkillMetadata(contextFiles = []) {
+        const skillFile = contextFiles.find(f => f.endsWith('.yamo'));
+        if (!skillFile) {
+            return { version: null, description: null };
+        }
+
+        // Return basic info - full parsing would require reading the file
+        return {
+            version: '1.0.0', // Default version
+            file: skillFile
+        };
+    }
+
+    /**
+     * Create a report object from execution data
+     * @param {Object} executionData - Data from skill execution
+     * @returns {Object} Report object
+     */
+    createReport(executionData) {
+        const {
+            skill,
+            sessionId,
+            duration,
+            provider,
+            model,
+            promptLength,
+            responseLength,
+            contextFiles = [],
+            parameters = {},
+            status = 'success',
+            error = null,
+            artifactsCreated = [],
+            memoryCaptured = false
+        } = executionData;
+
+        const reportId = this._generateReportId(sessionId);
+        const skillMeta = this._parseSkillMetadata(contextFiles);
+        const skillType = this._getSkillType(skill, contextFiles);
+
+        return {
+            report_id: reportId,
+            timestamp: new Date().toISOString(),
+            skill: {
+                name: skill,
+                version: skillMeta.version,
+                type: skillType,
+                file: skillMeta.file || null
+            },
+            execution: {
+                session_id: sessionId,
+                duration_ms: Math.round(duration),
+                status,
+                error: error ? String(error) : null,
+                provider,
+                model
+            },
+            input: {
+                prompt_length: promptLength,
+                context_files: contextFiles,
+                parameters
+            },
+            output: {
+                response_length: responseLength,
+                artifacts_created: artifactsCreated,
+                tokens_used: null // Could be populated if provider returns token count
+            },
+            quality: {
+                memory_captured: memoryCaptured,
+                artifacts_saved: artifactsCreated.length > 0
+            },
+            meta: {
+                generator: 'yamo-skills',
+                version: this.version
+            }
+        };
+    }
+
+    /**
+     * Generate filename for a report
+     * @param {Object} report - Report object
+     * @returns {string} Filename
+     */
+    getReportFilename(report) {
+        // Format: skill-{name}_{timestamp}_{ms}.json
+        // Include milliseconds for uniqueness when multiple reports per second
+        const safeName = report.skill.name.toLowerCase().replace(/[^a-z0-9]/g, '-');
+        const timestamp = report.timestamp.replace(/[:.]/g, '-').replace('T', '_').slice(0, 19);
+        const ms = report.timestamp.slice(20, 23) || '000';
+        return `skill-${safeName}_${timestamp}-${ms}.json`;
+    }
+
+    /**
+     * Ensure reports directory exists
+     */
+    async _ensureReportsDir() {
+        try {
+            await fs.mkdir(this.reportsDir, { recursive: true });
+        } catch (error) {
+            const e = error instanceof Error ? error : new Error(String(error));
+            // @ts-ignore
+            if (e.code !== 'EEXIST') {
+                throw e;
+            }
+        }
+    }
+
+    /**
+     * Save a report to disk
+     * @param {Object} report - Report object to save
+     * @returns {Promise<string>} Path to saved report
+     */
+    async saveReport(report) {
+        await this._ensureReportsDir();
+
+        const filename = this.getReportFilename(report);
+        const filepath = path.join(this.reportsDir, filename);
+
+        await fs.writeFile(filepath, JSON.stringify(report, null, 2), 'utf8');
+
+        return filepath;
+    }
+
+    /**
+     * Generate and save a report in one call
+     * @param {Object} executionData - Data from skill execution
+     * @returns {Promise<Object>} Report object with filepath
+     */
+    async generateAndSave(executionData) {
+        const report = this.createReport(executionData);
+        const filepath = await this.saveReport(report);
+
+        return {
+            ...report,
+            _filepath: filepath
+        };
+    }
+
+    /**
+     * List recent reports
+     * @param {number} limit - Maximum number of reports to return
+     * @returns {Promise<string[]>} Array of report filenames
+     */
+    async listReports(limit = 10) {
+        try {
+            await this._ensureReportsDir();
+            const files = await fs.readdir(this.reportsDir);
+
+            // Filter JSON files and sort by name (descending = newest first)
+            const reports = files
+                .filter(f => f.endsWith('.json'))
+                .sort((a, b) => b.localeCompare(a))
+                .slice(0, limit);
+
+            return reports;
+        } catch (error) {
+            return [];
+        }
+    }
+
+    /**
+     * Read a specific report
+     * @param {string} filename - Report filename
+     * @returns {Promise<Object|null>} Report object or null
+     */
+    async readReport(filename) {
+        try {
+            const filepath = path.join(this.reportsDir, filename);
+            const content = await fs.readFile(filepath, 'utf8');
+            return JSON.parse(content);
+        } catch (error) {
+            return null;
+        }
+    }
+
+    /**
+     * Get aggregate statistics from recent reports
+     * @param {number} limit - Number of reports to analyze
+     * @returns {Promise<Object>} Statistics object
+     */
+    async getStats(limit = 100) {
+        const reportFiles = await this.listReports(limit);
+        const stats = {
+            total_reports: reportFiles.length,
+            skills_used: {},
+            providers_used: {},
+            success_count: 0,
+            error_count: 0,
+            total_duration_ms: 0,
+            avg_duration_ms: 0
+        };
+
+        for (const filename of reportFiles) {
+            const report = await this.readReport(filename);
+            if (!report) continue;
+
+            // Count skills
+            const skillName = report.skill?.name || 'unknown';
+            stats.skills_used[skillName] = (stats.skills_used[skillName] || 0) + 1;
+
+            // Count providers
+            const provider = report.execution?.provider || 'unknown';
+            stats.providers_used[provider] = (stats.providers_used[provider] || 0) + 1;
+
+            // Count success/error
+            if (report.execution?.status === 'success') {
+                stats.success_count++;
+            } else {
+                stats.error_count++;
+            }
+
+            // Sum duration
+            stats.total_duration_ms += report.execution?.duration_ms || 0;
+        }
+
+        if (stats.total_reports > 0) {
+            stats.avg_duration_ms = Math.round(stats.total_duration_ms / stats.total_reports);
+        }
+
+        return stats;
+    }
+}
+
+export default SkillReportGenerator;

package/lib/scrubber/.gitkeep

@@ -0,0 +1 @@
+# This directory contains S-MORA Layer 0 Scrubber components

package/lib/scrubber/config/defaults.js

@@ -0,0 +1,62 @@
+/**
+ * S-MORA Layer 0 Scrubber Default Configuration
+ * @module smora/scrubber/config/defaults
+ */
+
+export const defaultScrubberConfig = {
+  // Master switch
+  enabled: false,
+
+  // Stage 1: Structural Cleaning
+  structural: {
+    stripHTML: true,
+    normalizeMarkdown: true,
+    collapseWhitespace: true,
+    removeScripts: true,
+    removeStyles: true
+  },
+
+  // Stage 2: Semantic Filtering
+  semantic: {
+    removeDuplicates: true,
+    removeBoilerplate: true,
+    minSignalRatio: 0.3,
+    boilerplatePatterns: 'default'
+  },
+
+  // Stage 3: Normalization
+  normalization: {
+    normalizeHeadings: true,
+    normalizeLists: true,
+    normalizePunctuation: true
+  },
+
+  // Stage 4: Chunking
+  chunking: {
+    maxTokens: 500,
+    minTokens: 10,
+    hardMaxTokens: 2000,
+    splitOnHeadings: true,
+    preserveContext: true
+  },
+
+  // Stage 5: Metadata Annotation
+  metadata: {
+    addSource: true,
+    addSection: true,
+    addHeadingPath: true,
+    addTimestamp: true,
+    addHash: true
+  },
+
+  // Stage 6: Validation
+  validation: {
+    enforceMinLength: true,
+    enforceMaxLength: true,
+    rejectEmptyChunks: true
+  },
+
+  // Performance
+  logTransformations: false,
+  cachePatterns: true
+};

package/lib/scrubber/errors/scrubber-error.js

@@ -0,0 +1,43 @@
+/**
+ * S-MORA Layer 0 Scrubber Error Classes
+ * @module smora/scrubber/errors/scrubber-error
+ */
+
+export class ScrubberError extends Error {
+  constructor(message, details = {}) {
+    super(message);
+    this.name = 'ScrubberError';
+    this.details = details;
+    this.timestamp = new Date().toISOString();
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+      message: this.message,
+      details: this.details,
+      timestamp: this.timestamp
+    };
+  }
+}
+
+export class StructuralCleaningError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'StructuralCleaningError';
+  }
+}
+
+export class ChunkingError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'ChunkingError';
+  }
+}
+
+export class ValidationError extends ScrubberError {
+  constructor(message, details = {}) {
+    super(message, details);
+    this.name = 'ValidationError';
+  }
+}

package/lib/scrubber/index.js

@@ -0,0 +1,25 @@
+/**
+ * S-MORA Layer 0 Scrubber
+ * Deterministic ingestion-time preprocessing layer
+ * @module smora/scrubber
+ */
+
+export { defaultScrubberConfig } from './config/defaults.js';
+export {
+  ScrubberError,
+  StructuralCleaningError,
+  ChunkingError,
+  ValidationError
+} from './errors/scrubber-error.js';
+export { ScrubberTelemetry } from './telemetry.js';
+export { Scrubber } from './scrubber.js';
+export { HashUtil } from './utils/hash.js';
+export { TokenCounter } from './utils/token-counter.js';
+export { PatternMatcher } from './utils/pattern-matcher.js';
+export { HTMLParser } from './utils/html-parser.js';
+export { StructuralCleaner } from './stages/structural-cleaner.js';
+export { SemanticFilter } from './stages/semantic-filter.js';
+export { Normalizer } from './stages/normalizer.js';
+export { Chunker } from './stages/chunker.js';
+export { MetadataAnnotator } from './stages/metadata-annotator.js';
+export { Validator } from './stages/validator.js';

package/lib/scrubber/scrubber.js

@@ -0,0 +1,130 @@
+/**
+ * S-MORA Layer 0 Scrubber - Main Orchestrator
+ * @module smora/scrubber/scrubber
+ */
+
+import { StructuralCleaner } from './stages/structural-cleaner.js';
+import { SemanticFilter } from './stages/semantic-filter.js';
+import { Normalizer } from './stages/normalizer.js';
+import { Chunker } from './stages/chunker.js';
+import { MetadataAnnotator } from './stages/metadata-annotator.js';
+import { Validator } from './stages/validator.js';
+import { ScrubberTelemetry } from './telemetry.js';
+import { ScrubberError } from './errors/scrubber-error.js';
+import { defaultScrubberConfig } from './config/defaults.js';
+
+export class Scrubber {
+  constructor(config = {}) {
+    this.config = { ...defaultScrubberConfig, ...config };
+    this.stages = this._initializeStages();
+    this.telemetry = new ScrubberTelemetry();
+  }
+
+  /**
+   * Main entry point - process a raw document
+   * @param {Object} document - { content: string, source: string, type: 'html'|'md'|'txt' }
+   * @returns {Promise<Object>} - { chunks: Array, metadata: Object, telemetry: Object }
+   */
+  async process(document) {
+    const startTime = Date.now();
+    const result = {
+      chunks: [],
+      metadata: {
+        source: document.source,
+        type: document.type,
+        processingTimestamp: new Date().toISOString()
+      },
+      telemetry: {}
+    };
+
+    try {
+      // If disabled, return empty chunks
+      if (!this.config.enabled) {
+        result.success = true;
+        result.telemetry.totalDuration = Date.now() - startTime;
+        return result;
+      }
+
+      // Stage 1: Structural Cleaning
+      const cleaned = await this._executeStage('structural', () =>
+        this.stages.structural.clean(document.content)
+      );
+      result.telemetry.structural = this.telemetry.getStageStats('structural');
+
+      // Stage 2: Semantic Filtering
+      const filtered = await this._executeStage('semantic', () =>
+        this.stages.semantic.filter(cleaned)
+      );
+      result.telemetry.semantic = this.telemetry.getStageStats('semantic');
+
+      // Stage 3: Normalization
+      const normalized = await this._executeStage('normalization', () =>
+        this.stages.normalizer.normalize(filtered)
+      );
+      result.telemetry.normalization = this.telemetry.getStageStats('normalization');
+
+      // Stage 4: Chunking
+      const chunks = await this._executeStage('chunking', () =>
+        this.stages.chunker.chunk(normalized)
+      );
+      result.telemetry.chunking = this.telemetry.getStageStats('chunking');
+
+      // Stage 5: Metadata Annotation
+      const annotated = await this._executeStage('metadata', () =>
+        this.stages.metadata.annotate(chunks, document)
+      );
+      result.telemetry.metadata = this.telemetry.getStageStats('metadata');
+
+      // Stage 6: Validation
+      result.chunks = await this._executeStage('validation', () =>
+        this.stages.validator.validate(annotated)
+      );
+      result.telemetry.validation = this.telemetry.getStageStats('validation');
+
+      result.telemetry.totalDuration = Date.now() - startTime;
+      result.success = true;
+
+      return result;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      result.success = false;
+      result.error = message;
+      result.telemetry.totalDuration = Date.now() - startTime;
+    }
+  }
+
+  async _executeStage(stageName, stageFn) {
+    const startTime = Date.now();
+    try {
+      const result = await stageFn();
+      const duration = Date.now() - startTime;
+      this.telemetry.recordStage(stageName, duration, true);
+      return result;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      this.telemetry.recordStage(stageName, duration, false);
+      throw error;
+    }
+  }
+
+  _initializeStages() {
+    return {
+      structural: new StructuralCleaner(this.config.structural),
+      semantic: new SemanticFilter(this.config.semantic),
+      normalizer: new Normalizer(this.config.normalization),
+      chunker: new Chunker(this.config.chunking),
+      metadata: new MetadataAnnotator(this.config.metadata),
+      validator: new Validator(this.config.validation)
+    };
+  }
+
+  getMetrics() {
+    return this.telemetry.getSummary();
+  }
+
+  async healthCheck() {
+    return { status: 'healthy' };
+  }
+}
+
+export default Scrubber;

package/lib/scrubber/stages/chunker.js

@@ -0,0 +1,103 @@
+/**
+ * S-MORA Layer 0 Scrubber - Stage 4: Chunking
+ * @module smora/scrubber/stages/chunker
+ */
+
+import { TokenCounter } from '../utils/token-counter.js';
+import { ChunkingError, ScrubberError } from '../errors/scrubber-error.js';
+
+export class Chunker {
+  constructor(config) {
+    this.config = config;
+    this.tokenCounter = new TokenCounter();
+  }
+
+  /**
+   * Split content into chunks
+   * @param {string} content - Normalized content
+   * @returns {Promise<Array>} - Array of chunks with metadata
+   */
+  async chunk(content) {
+    try {
+      const chunks = [];
+      const paragraphs = content.split(/\n\n+/);
+
+      let currentChunk = {
+        text: '',
+        tokens: 0,
+        heading: this._extractInitialHeading(content)
+      };
+
+      for (const para of paragraphs) {
+        const isHeading = this._isHeading(para);
+        const paraTokens = this.tokenCounter.count(para);
+
+        if (this._shouldStartNewChunk(currentChunk, para, paraTokens, isHeading)) {
+          if (currentChunk.tokens >= this.config.minTokens) {
+            chunks.push({ ...currentChunk });
+          }
+          currentChunk = {
+            text: '',
+            tokens: 0,
+            heading: isHeading ? this._extractHeadingText(para) : currentChunk.heading
+          };
+        }
+
+        currentChunk.text += (currentChunk.text ? '\n\n' : '') + para;
+        currentChunk.tokens += paraTokens;
+
+        if (currentChunk.tokens > this.config.hardMaxTokens) {
+          chunks.push({ ...currentChunk });
+          currentChunk = { text: '', tokens: 0, heading: null };
+        }
+      }
+
+      if (currentChunk.tokens >= this.config.minTokens) {
+        chunks.push(currentChunk);
+      }
+
+      return chunks.map((chunk, index) => ({
+        index,
+        text: chunk.text.trim(),
+        metadata: {
+          tokens: chunk.tokens,
+          heading: chunk.heading,
+          position: index
+        }
+      }));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ScrubberError(
+        `Failed to chunk content: ${message}`,
+        { stage: 'chunker', originalError: error }
+      );
+    }
+  }
+
+  _isHeading(line) {
+    return /^#{1,6}\s/.test(line);
+  }
+
+  _shouldStartNewChunk(currentChunk, para, paraTokens, isHeading) {
+    if (this.config.splitOnHeadings && isHeading && currentChunk.tokens > 0) {
+      return true;
+    }
+
+    const wouldExceed = (currentChunk.tokens + paraTokens) > this.config.maxTokens;
+    if (wouldExceed && currentChunk.tokens > 0) {
+      return true;
+    }
+
+    return false;
+  }
+
+  _extractInitialHeading(content) {
+    const match = content.match(/^#{1,6}\s+(.+)$/m);
+    return match ? match[1] : null;
+  }
+
+  _extractHeadingText(headingLine) {
+    const match = headingLine.match(/^#{1,6}\s+(.+)$/);
+    return match ? match[1] : null;
+  }
+}