@yamo/memory-mesh 2.3.2 → 3.0.0

package/lib/yamo/schema.js

@@ -5,121 +5,99 @@
  * Provides schema and table creation for YAMO block persistence.
  * YAMO blocks provide audit trail for all memory operations.
  */
-
 import * as arrow from "apache-arrow";
-
 /**
  * Create YAMO blocks table schema
  * Defines the structure for storing YAMO protocol blocks
- *
- * Schema includes:
- * - Core identifiers (id, agent_id)
- * - Operation tracking (operation_type, yamo_text)
- * - Temporal data (timestamp)
- * - Blockchain fields (block_hash, prev_hash) - nullable for future use
- * - Metadata (JSON string for flexibility)
- *
- * @returns {import('apache-arrow').Schema} Arrow schema for YAMO blocks
+ * @returns {arrow.Schema} Arrow schema for YAMO blocks
  */
 export function createYamoSchema() {
-  return new arrow.Schema([
-    // Core identifiers
-    new arrow.Field('id', new arrow.Utf8(), false),
-    new arrow.Field('agent_id', new arrow.Utf8(), true),
-
-    // Operation tracking
-    new arrow.Field('operation_type', new arrow.Utf8(), false),  // 'retain', 'recall', 'reflect'
-    new arrow.Field('yamo_text', new arrow.Utf8(), false),       // Full YAMO block content
-
-    // Temporal
-    new arrow.Field('timestamp', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
-
-    // Blockchain fields (optional, nullable) - for future anchoring
-    new arrow.Field('block_hash', new arrow.Utf8(), true),   // Hash of this block
-    new arrow.Field('prev_hash', new arrow.Utf8(), true),    // Hash of previous block (for chain)
-
-    // Metadata (JSON string for flexibility)
-    new arrow.Field('metadata', new arrow.Utf8(), true),      // Additional metadata as JSON
-  ]);
+    return new arrow.Schema([
+        // Core identifiers
+        new arrow.Field("id", new arrow.Utf8(), false),
+        new arrow.Field("agent_id", new arrow.Utf8(), true),
+        // Operation tracking
+        new arrow.Field("operation_type", new arrow.Utf8(), false), // 'retain', 'recall', 'reflect'
+        new arrow.Field("yamo_text", new arrow.Utf8(), false), // Full YAMO block content
+        // Temporal
+        new arrow.Field("timestamp", new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
+        // Blockchain fields (optional, nullable) - for future anchoring
+        new arrow.Field("block_hash", new arrow.Utf8(), true), // Hash of this block
+        new arrow.Field("prev_hash", new arrow.Utf8(), true), // Hash of previous block (for chain)
+        // Metadata (JSON string for flexibility)
+        new arrow.Field("metadata", new arrow.Utf8(), true), // Additional metadata as JSON
+    ]);
 }
-
 /**
  * Create YAMO blocks table in LanceDB
  * Creates the table if it doesn't exist, opens it if it does
  *
- * @param {import('@lancedb/lancedb').Connection} db - LanceDB connection
+ * @param {lancedb.Connection} db - LanceDB connection
  * @param {string} [tableName='yamo_blocks'] - Name of the table
- * @returns {Promise<import('@lancedb/lancedb').Table>} The created or opened table
+ * @returns {Promise<lancedb.Table>} The created or opened table
  * @throws {Error} If table creation fails
  */
-export async function createYamoTable(db, tableName = 'yamo_blocks') {
-  try {
-    // Check if table already exists
-    const existingTables = await db.tableNames();
-
-    if (existingTables.includes(tableName)) {
-      // Table exists, open it
-      return await db.openTable(tableName);
+export async function createYamoTable(db, tableName = "yamo_blocks") {
+    try {
+        // Check if table already exists
+        const existingTables = await db.tableNames();
+        if (existingTables.includes(tableName)) {
+            // Table exists, open it
+            return await db.openTable(tableName);
+        }
+        // Create new table with YAMO schema
+        const schema = createYamoSchema();
+        const table = await db.createTable(tableName, [], { schema });
+        return table;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to create YAMO table '${tableName}': ${message}`);
     }
-
-    // Create new table with YAMO schema
-    const schema = createYamoSchema();
-    const table = await db.createTable(tableName, [], { schema });
-
-    return table;
-
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    throw new Error(`Failed to create YAMO table '${tableName}': ${message}`);
-  }
 }
-
 /**
  * Validate a YAMO block record before insertion
  * Checks for required fields and valid values
- *
- * @param {Object} record - Record to validate
- * @param {string} record.id - Block ID
- * @param {string} record.operation_type - Operation type
- * @param {string} record.yamo_text - YAMO block text
- * @returns {Object} Validation result { valid, errors }
  */
 export function validateYamoRecord(record) {
-  const errors = [];
-
-  // Check required fields
-  if (!record.id) {
-    errors.push('Missing required field: id');
-  }
-
-  if (!record.operation_type) {
-    errors.push('Missing required field: operation_type');
-  } else {
-    // Validate operation_type is one of the allowed values
-    const validTypes = ['retain', 'recall', 'reflect'];
-    if (!validTypes.includes(record.operation_type)) {
-      errors.push(`Invalid operation_type: ${record.operation_type}. Must be one of: ${validTypes.join(', ')}`);
+    const errors = [];
+    // Check required fields
+    if (!record.id) {
+        errors.push("Missing required field: id");
+    }
+    if (!record.operation_type) {
+        errors.push("Missing required field: operation_type");
+    }
+    else {
+        // Validate operation_type is one of the allowed values
+        const validTypes = ["retain", "recall", "reflect"];
+        if (!validTypes.includes(record.operation_type)) {
+            errors.push(`Invalid operation_type: ${record.operation_type}. Must be one of: ${validTypes.join(", ")}`);
+        }
+    }
+    if (!record.yamo_text) {
+        errors.push("Missing required field: yamo_text");
     }
-  }
-
-  if (!record.yamo_text) {
-    errors.push('Missing required field: yamo_text');
-  } else {
-    // Validate YAMO block format
-    const requiredSections = ['agent:', 'intent:', 'context:', 'output:', 'log:'];
-    for (const section of requiredSections) {
-      if (!record.yamo_text.includes(section)) {
-        errors.push(`YAMO block missing required section: ${section}`);
-      }
+    else {
+        // Validate YAMO block format
+        const requiredSections = [
+            "agent:",
+            "intent:",
+            "context:",
+            "output:",
+            "log:",
+        ];
+        for (const section of requiredSections) {
+            if (!record.yamo_text.includes(section)) {
+                errors.push(`YAMO block missing required section: ${section}`);
+            }
+        }
     }
-  }
-
-  return {
-    valid: errors.length === 0,
-    errors
-  };
+    return {
+        valid: errors.length === 0,
+        errors,
+    };
 }
-
 /**
  * Generate a YAMO block ID
  * Creates a unique ID for a YAMO block
@@ -128,32 +106,27 @@ export function validateYamoRecord(record) {
  * @returns {string} Generated YAMO block ID
  */
 export function generateYamoId(operationType) {
-  const timestamp = Date.now();
-  const random = Math.random().toString(36).substring(2, 10);
-  return `yamo_${operationType}_${timestamp}_${random}`;
+    const timestamp = Date.now();
+    const random = Math.random().toString(36).substring(2, 10);
+    return `yamo_${operationType}_${timestamp}_${random}`;
 }
-
 /**
  * Check if a table uses YAMO schema
  * Detects if a table has the YAMO block schema structure
  *
- * @param {import('apache-arrow').Schema} schema - Table schema to check
+ * @param {arrow.Schema} schema - Table schema to check
  * @returns {boolean} True if YAMO schema detected
  */
 export function isYamoSchema(schema) {
-  // Check for unique YAMO fields
-  const hasYamoFields = schema.fields.some(f =>
-    f.name === 'operation_type' || f.name === 'yamo_text'
-  );
-
-  return hasYamoFields;
+    // Check for unique YAMO fields
+    const hasYamoFields = schema.fields.some((f) => f.name === "operation_type" || f.name === "yamo_text");
+    return hasYamoFields;
 }
-
 // Export schema function as default for consistency with lancedb/schema.js
 export default {
-  createYamoSchema,
-  createYamoTable,
-  validateYamoRecord,
-  generateYamoId,
-  isYamoSchema
+    createYamoSchema,
+    createYamoTable,
+    validateYamoRecord,
+    generateYamoId,
+    isYamoSchema,
 };

package/lib/yamo/schema.ts

@@ -0,0 +1,165 @@
+/**
+ * YAMO Block Schema Definitions for yamo-memory-mesh
+ * Uses Apache Arrow Schema format for LanceDB JavaScript SDK
+ *
+ * Provides schema and table creation for YAMO block persistence.
+ * YAMO blocks provide audit trail for all memory operations.
+ */
+
+import * as arrow from "apache-arrow";
+import * as lancedb from "@lancedb/lancedb";
+
+/**
+ * Create YAMO blocks table schema
+ * Defines the structure for storing YAMO protocol blocks
+ * @returns {arrow.Schema} Arrow schema for YAMO blocks
+ */
+export function createYamoSchema(): arrow.Schema {
+  return new arrow.Schema([
+    // Core identifiers
+    new arrow.Field("id", new arrow.Utf8(), false),
+    new arrow.Field("agent_id", new arrow.Utf8(), true),
+
+    // Operation tracking
+    new arrow.Field("operation_type", new arrow.Utf8(), false), // 'retain', 'recall', 'reflect'
+    new arrow.Field("yamo_text", new arrow.Utf8(), false), // Full YAMO block content
+
+    // Temporal
+    new arrow.Field(
+      "timestamp",
+      new arrow.Timestamp(arrow.TimeUnit.MILLISECOND),
+      false,
+    ),
+
+    // Blockchain fields (optional, nullable) - for future anchoring
+    new arrow.Field("block_hash", new arrow.Utf8(), true), // Hash of this block
+    new arrow.Field("prev_hash", new arrow.Utf8(), true), // Hash of previous block (for chain)
+
+    // Metadata (JSON string for flexibility)
+    new arrow.Field("metadata", new arrow.Utf8(), true), // Additional metadata as JSON
+  ]);
+}
+
+/**
+ * Create YAMO blocks table in LanceDB
+ * Creates the table if it doesn't exist, opens it if it does
+ *
+ * @param {lancedb.Connection} db - LanceDB connection
+ * @param {string} [tableName='yamo_blocks'] - Name of the table
+ * @returns {Promise<lancedb.Table>} The created or opened table
+ * @throws {Error} If table creation fails
+ */
+export async function createYamoTable(
+  db: lancedb.Connection,
+  tableName: string = "yamo_blocks",
+): Promise<lancedb.Table> {
+  try {
+    // Check if table already exists
+    const existingTables = await db.tableNames();
+
+    if (existingTables.includes(tableName)) {
+      // Table exists, open it
+      return await db.openTable(tableName);
+    }
+
+    // Create new table with YAMO schema
+    const schema = createYamoSchema();
+    const table = await db.createTable(tableName, [], { schema } as any);
+
+    return table;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(`Failed to create YAMO table '${tableName}': ${message}`);
+  }
+}
+
+export interface YamoRecordValidationResult {
+  valid: boolean;
+  errors: string[];
+}
+
+/**
+ * Validate a YAMO block record before insertion
+ * Checks for required fields and valid values
+ */
+export function validateYamoRecord(record: any): YamoRecordValidationResult {
+  const errors: string[] = [];
+
+  // Check required fields
+  if (!record.id) {
+    errors.push("Missing required field: id");
+  }
+
+  if (!record.operation_type) {
+    errors.push("Missing required field: operation_type");
+  } else {
+    // Validate operation_type is one of the allowed values
+    const validTypes = ["retain", "recall", "reflect"];
+    if (!validTypes.includes(record.operation_type)) {
+      errors.push(
+        `Invalid operation_type: ${record.operation_type}. Must be one of: ${validTypes.join(", ")}`,
+      );
+    }
+  }
+
+  if (!record.yamo_text) {
+    errors.push("Missing required field: yamo_text");
+  } else {
+    // Validate YAMO block format
+    const requiredSections = [
+      "agent:",
+      "intent:",
+      "context:",
+      "output:",
+      "log:",
+    ];
+    for (const section of requiredSections) {
+      if (!record.yamo_text.includes(section)) {
+        errors.push(`YAMO block missing required section: ${section}`);
+      }
+    }
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+/**
+ * Generate a YAMO block ID
+ * Creates a unique ID for a YAMO block
+ *
+ * @param {string} operationType - Type of operation
+ * @returns {string} Generated YAMO block ID
+ */
+export function generateYamoId(operationType: string): string {
+  const timestamp = Date.now();
+  const random = Math.random().toString(36).substring(2, 10);
+  return `yamo_${operationType}_${timestamp}_${random}`;
+}
+
+/**
+ * Check if a table uses YAMO schema
+ * Detects if a table has the YAMO block schema structure
+ *
+ * @param {arrow.Schema} schema - Table schema to check
+ * @returns {boolean} True if YAMO schema detected
+ */
+export function isYamoSchema(schema: arrow.Schema): boolean {
+  // Check for unique YAMO fields
+  const hasYamoFields = schema.fields.some(
+    (f) => f.name === "operation_type" || f.name === "yamo_text",
+  );
+
+  return hasYamoFields;
+}
+
+// Export schema function as default for consistency with lancedb/schema.js
+export default {
+  createYamoSchema,
+  createYamoTable,
+  validateYamoRecord,
+  generateYamoId,
+  isYamoSchema,
+};

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@yamo/memory-mesh",
-  "version": "2.3.2",
-  "description": "Portable semantic memory system with Layer 0 Scrubber for YAMO agents",
+  "version": "3.0.0",
+  "description": "Portable semantic memory system with Layer 0 Scrubber for YAMO agents (v3 Singularity Edition)",
   "type": "module",
   "main": "lib/memory/index.js",
-  "types": "index.d.ts",
+  "types": "lib/memory/index.d.ts",
   "bin": {
     "memory-mesh": "./bin/memory_mesh.js",
     "memory-mesh-setup": "./bin/setup.js"
@@ -12,18 +12,21 @@
   "files": [
     "lib/",
     "bin/",
-    "skills/",
-    "index.d.ts"
+    "skills/"
   ],
   "scripts": {
-    "test": "npm run type-check && node --test test/*.test.js",
-    "type-check": "tsc --noEmit"
+    "build": "tsc",
+    "test": "npm run build && node --test test/*.test.js",
+    "type-check": "tsc --noEmit",
+    "prepublishOnly": "npm run build"
   },
   "dependencies": {
     "@lancedb/lancedb": "^0.23.0",
     "@xenova/transformers": "^2.17.0",
     "apache-arrow": "^17.0.0",
-    "onnxruntime-node": "^1.18.0"
+    "onnxruntime-node": "^1.18.0",
+    "pino": "^10.3.1",
+    "pino-pretty": "^13.1.3"
   },
   "author": "Soverane Labs",
   "license": "MIT",

package/index.d.ts

@@ -1,111 +0,0 @@
-export declare class MemoryMesh {
-    constructor();
-    init(): Promise<void>;
-    add(content: string, metadata?: Record<string, any>): Promise<{
-        id: string;
-        content: string;
-        metadata: Record<string, any>;
-        created_at: string;
-    }>;
-    addBatch(entries: Array<{
-        content: string;
-        metadata?: Record<string, any>;
-    }>): Promise<{
-        count: number;
-        success: boolean;
-        ids: string[];
-    }>;
-    search(query: string, options?: {
-        limit?: number;
-        filter?: string;
-        useCache?: boolean;
-    }): Promise<Array<{
-        id: string;
-        content: string;
-        metadata: Record<string, any>;
-        score: number;
-        created_at: string;
-    }>>;
-    get(id: string): Promise<{
-        id: string;
-        content: string;
-        metadata: Record<string, any>;
-        created_at: string;
-        updated_at: string;
-    } | null>;
-    getAll(options?: {
-        limit?: number;
-    }): Promise<Array<any>>;
-    update(id: string, content: string, metadata?: Record<string, any>): Promise<{
-        id: string;
-        content: string;
-        success: boolean;
-    }>;
-    delete(id: string): Promise<{
-        deleted: string;
-        success: boolean;
-    }>;
-    stats(): Promise<{
-        count: number;
-        tableName: string;
-        uri: string;
-        isConnected: boolean;
-        embedding: any;
-    }>;
-    healthCheck(): Promise<{
-        status: string;
-        timestamp: string;
-        checks: Record<string, any>;
-    }>;
-}
-
-export declare class Scrubber {
-    constructor(config?: ScrubberConfig);
-    process(document: {
-        content: string;
-        source: string;
-        type: 'html' | 'md' | 'txt';
-    }): Promise<ScrubberResult>;
-}
-
-export interface ScrubberConfig {
-    enabled?: boolean;
-    structural?: {
-        stripHTML?: boolean;
-        normalizeMarkdown?: boolean;
-        collapseWhitespace?: boolean;
-        removeScripts?: boolean;
-        removeStyles?: boolean;
-    };
-    semantic?: {
-        removeDuplicates?: boolean;
-        removeBoilerplate?: boolean;
-        minSignalRatio?: number;
-    };
-    chunking?: {
-        maxTokens?: number;
-        minTokens?: number;
-        splitOnHeadings?: boolean;
-    };
-    validation?: {
-        enforceMinLength?: boolean;
-    };
-}
-
-export interface ScrubberResult {
-    chunks: Array<{
-        text: string;
-        metadata: Record<string, any>;
-    }>;
-    metadata: {
-        source: string;
-        type: string;
-        processingTimestamp: string;
-    };
-    telemetry: {
-        totalDuration: number;
-        stages: Record<string, any>;
-    };
-    success: boolean;
-    error?: string;
-}

package/lib/embeddings/index.js

@@ -1,2 +0,0 @@
-export { default as EmbeddingService } from './service.js';
-export { default as EmbeddingFactory } from './factory.js';

package/lib/index.js

@@ -1,6 +0,0 @@
-// Core module exports
-export * from './lancedb/index.js';
-export * from './embeddings/index.js';
-export * from './search/index.js';
-export * from './memory/index.js';
-export * from './scrubber/index.js';