mumpix 1.0.19 → 1.0.29

package/src/core/MumpixDB.js

@@ -1,329 +1,249 @@
-'use strict';
-
-/**
- * MumpixDB — main database class
- *
- * const { Mumpix } = require('mumpix')
- * const db = await Mumpix.open('./agent.mumpix', { consistency: 'strict' })
- *
- * await db.remember('User prefers TypeScript')
- * const ans = await db.recall('what language?')
- * const all = await db.list()
- * await db.clear()
- * await db.close()
- */
-
-const { MumpixStore } = require('./store');
-const { MumpixAudit } = require('./audit');
-const { recall, recallMany } = require('./recall');
-const { LicenseManager } = require('./license');
-const { getStoredLicenseKey } = require('./auth');
-const { answerTemporalQuery, materializeEvents, materializeEventIndex, extractEventsFromRecord } = require('../temporal/engine');
-const { appendEventIndex } = require('../temporal/indexes');
-
-const VALID_MODES = ['eventual', 'strict', 'verified'];
+"use strict";
+
+const { MumpixStore } = require("./store");
+const { MumpixAudit } = require("./audit");
+const { recall, recallMany: scoreRecallMany } = require("./recall");
+const { LicenseManager } = require("./license");
+const { getStoredLicenseKey } = require("./auth");
+const {
+  answerTemporalQuery,
+  materializeEvents,
+  materializeEventIndex,
+  extractEventsFromRecord,
+} = require("../temporal/engine");
+const { appendEventIndex } = require("../temporal/indexes");
+
+const VALID_CONSISTENCY = ["eventual", "strict", "verified"];
+
+function envFlag(name) {
+  const value = process.env[name];
+  return value != null && value !== "" && /^(1|true|yes|on)$/i.test(String(value).trim());
+}
+
+function telemetryEnabled(opts = {}) {
+  return typeof opts.telemetry === "boolean" ? opts.telemetry : envFlag("MUMPIX_TELEMETRY");
+}
+
+function storeOpenOptions(consistency, opts) {
+  return {
+    consistency,
+    lock: opts.lock,
+    lockStaleMs: opts.lockStaleMs,
+    walFlushIntervalMs: opts.walFlushIntervalMs,
+    walMaxPendingBytes: opts.walMaxPendingBytes,
+    recallIndex: opts.recallIndex,
+  };
+}
 
 class MumpixDB {
-  /**
-   * @param {MumpixStore} store
-   * @param {MumpixAudit|null} audit
-   * @param {LicenseManager} license
-   * @param {object} opts
-   */
   constructor(store, audit, license, opts = {}) {
     this._store = store;
-    this._audit = audit;   // non-null only in 'verified' mode
+    this._audit = audit;
     this._opts = opts;
     this._closed = false;
     this._license = license;
     this._temporalCache = null;
   }
 
-  // ─────────────────────────────────────────
-  // Factory
-  // ─────────────────────────────────────────
-
-  /**
-   * Open (or create) a Mumpix database.
-   *
-   * @param {string} filePath  Path to .mumpix file
-   * @param {object} [opts]
-   * @param {string} [opts.consistency]  'eventual' | 'strict' | 'verified' (default: 'eventual')
-   * @param {Function} [opts.embedFn]    async (texts: string[]) => number[][] — optional custom embeddings
-   * @param {string} [opts.licenseKey]   Commercial license key for Pro/Enterprise features
-   * @returns {Promise<MumpixDB>}
-   */
   static async open(filePath, opts = {}) {
-    if (!filePath || typeof filePath !== 'string') {
-      throw new TypeError('Mumpix.open() requires a file path string');
+    if (!filePath || typeof filePath !== "string") {
+      throw new TypeError("Mumpix.open() requires a file path string");
     }
 
-    const requestedConsistency = opts.consistency || 'eventual';
-    if (!VALID_MODES.includes(requestedConsistency)) {
-      throw new Error(`Invalid consistency mode "${requestedConsistency}". Valid: ${VALID_MODES.join(', ')}`);
+    const requestedConsistency = opts.consistency || "eventual";
+    if (!VALID_CONSISTENCY.includes(requestedConsistency)) {
+      throw new Error(
+        `Invalid consistency mode "${requestedConsistency}". Valid: ${VALID_CONSISTENCY.join(", ")}`
+      );
     }
 
     const store = new MumpixStore(filePath);
-    let auditLog = null;
+    let audit = null;
+
     try {
-      await store.open({ consistency: requestedConsistency });
+      store.open(storeOpenOptions(requestedConsistency, opts));
 
-      // License initialization & check.
-      // Allow dependency injection for local verification and integration tests.
-      const resolvedLicenseKey = opts.licenseKey || getStoredLicenseKey() || null;
-      const license = new LicenseManager(resolvedLicenseKey);
+      const licenseKey = opts.licenseKey || getStoredLicenseKey() || null;
+      const license =
+        opts.licenseManager instanceof LicenseManager
+          ? opts.licenseManager
+          : new LicenseManager(licenseKey);
       license.setFileContext(store.header && store.header.fileId ? store.header.fileId : null);
-      const licenseReady = await license.init();
-      if (resolvedLicenseKey && !licenseReady) {
-        console.warn(`[DEBUG] License rejected: ${license.validationError}`);
-      }
+      await license.init();
 
-      // Capability-gated mode check with graceful downgrade policy:
-      // if a paid mode (strict/verified) is requested with expired/invalid license,
-      // force eventual mode until the license is renewed.
-      let effectiveConsistency = requestedConsistency;
-      if (requestedConsistency !== 'eventual') {
+      let consistency = requestedConsistency;
+      if (requestedConsistency !== "eventual") {
         try {
-          license.checkLimit('mode', requestedConsistency);
+          license.checkLimit("mode", requestedConsistency);
           license.assertActive(requestedConsistency);
-        } catch (modeErr) {
-          const msg = String(modeErr && modeErr.message ? modeErr.message : modeErr).toLowerCase();
-          console.warn(`[DEBUG] Mode check failed: ${msg}`);
-          const downgradeable =
-            msg.includes('expired') ||
-            msg.includes('invalid for mode') ||
-            msg.includes('license_validation_failed') ||
-            msg.includes('quantum signature verification failed') ||
-            msg.includes('lease window') ||
-            msg.includes('too far in the future');
-          if (!downgradeable) throw modeErr;
-          effectiveConsistency = 'eventual';
+        } catch (err) {
+          const message = String((err && err.message) || err).toLowerCase();
+          const canDowngrade =
+            message.includes("expired") ||
+            message.includes("invalid for mode") ||
+            message.includes("license_validation_failed") ||
+            message.includes("quantum signature verification failed") ||
+            message.includes("lease window") ||
+            message.includes("too far in the future");
+          if (!canDowngrade) throw err;
+          consistency = "eventual";
         }
       }
 
-      if (effectiveConsistency !== requestedConsistency) {
-        console.warn(`[DEBUG] Downgrading consistency to ${effectiveConsistency}`);
-        // Re-open with downgraded mode after releasing current FD/lock first.
+      if (consistency !== requestedConsistency) {
         store.close();
-        await store.open({ consistency: effectiveConsistency });
+        store.open(storeOpenOptions(consistency, opts));
       }
 
-      if (effectiveConsistency === 'verified') {
-        auditLog = new MumpixAudit(filePath);
-        auditLog.open();
+      if (consistency === "verified") {
+        audit = new MumpixAudit(filePath);
+        audit.open();
       }
 
-      return new MumpixDB(store, auditLog, license, {
+      return new MumpixDB(store, audit, license, {
         ...opts,
-        consistency: effectiveConsistency,
+        telemetry: telemetryEnabled(opts),
+        consistency,
         requestedConsistency,
-        downgradedConsistency: effectiveConsistency !== requestedConsistency
-          ? { from: requestedConsistency, to: effectiveConsistency, reason: 'license_not_active' }
-          : null,
+        downgradedConsistency:
+          consistency !== requestedConsistency
+            ? { from: requestedConsistency, to: consistency, reason: "license_not_active" }
+            : null,
       });
-    } catch (error) {
-      if (auditLog) {
-        try { auditLog.close(); } catch (_) {}
+    } catch (err) {
+      if (audit) {
+        try {
+          audit.close();
+        } catch (_) {}
       }
-      try { store.close(); } catch (_) {}
-      throw error;
+      try {
+        store.close();
+      } catch (_) {}
+      throw err;
     }
   }
 
-  // ─────────────────────────────────────────
-  // Core API
-  // ─────────────────────────────────────────
-
-  /**
-   * Store a memory.
-   *
-   * @param {string} content
-   * @returns {{ id: number, written: boolean, consistency: string }}
-   */
-  async remember(content, opts = {}) {
+  async remember(input, meta = {}) {
     this._assertOpen();
-    let text = content;
-    let meta = opts;
 
-    if (content && typeof content === 'object' && !Array.isArray(content)) {
-      text = content.content;
-      meta = {
-        ...opts,
-        workspace: content.workspace ?? opts.workspace,
-        repo: content.repo ?? opts.repo,
-        source: content.source ?? opts.source,
-        ts: content.ts ?? opts.ts,
+    let content = input;
+    let cleanMeta = meta;
+    if (input && typeof input === "object" && !Array.isArray(input)) {
+      content = input.content;
+      cleanMeta = {
+        ...meta,
+        workspace: input.workspace ?? meta.workspace,
+        repo: input.repo ?? meta.repo,
+        source: input.source ?? meta.source,
+        ts: input.ts ?? meta.ts,
       };
     }
 
-    if (typeof text !== 'string' || !text.trim()) {
-      throw new TypeError('remember() requires a non-empty string or { content } object');
+    if (typeof content !== "string" || !content.trim()) {
+      throw new TypeError("remember() requires a non-empty string or { content } object");
     }
 
-    // License Check for Record Limit
-    this._license.checkLimit('records', this._store.records.length);
-
-    const record = this._store.write(text, meta);
+    this._license.checkLimit("records", this._store.records.length);
+    const record = this._store.write(content, cleanMeta);
     this._appendTemporalRecord(record);
+    if (this._audit) this._audit.logWrite(record);
+    return { written: true, id: record.id, consistency: this._store.header.consistency };
+  }
 
-    if (this._audit) {
-      this._audit.logWrite(record);
-    }
+  async rememberAll(items) {
+    this._assertOpen();
+    if (!Array.isArray(items)) throw new TypeError("rememberAll() expects an array");
+    this._license.checkLimit("records", this._store.records.length + items.length);
 
-    return {
+    const normalized = items.map((item) => {
+      if (item && typeof item === "object" && !Array.isArray(item)) {
+        return item;
+      }
+      return { content: item };
+    });
+
+    const records = this._store.writeBatch(normalized);
+    for (const record of records) {
+      this._appendTemporalRecord(record);
+      if (this._audit) this._audit.logWrite(record);
+    }
+    return records.map((record) => ({
       written: true,
       id: record.id,
       consistency: this._store.header.consistency,
-    };
+    }));
   }
 
-  /**
-   * Recall the most semantically relevant memory for a query.
-   *
-   * @param {string} query
-   * @param {object} [opts]
-   * @param {object} [opts.filter]   Pre-filter records: fn(record) → bool
-   * @param {number} [opts.since]    Only consider records newer than this timestamp
-   * @param {string} [opts.mode]     'hybrid' | 'exact' | 'semantic'
-   * @returns {string|null}
-   */
   async recall(query, opts = {}) {
     this._assertOpen();
-    if (typeof query !== 'string' || !query.trim()) {
-      throw new TypeError('recall() requires a non-empty string');
+    if (typeof query !== "string" || !query.trim()) {
+      throw new TypeError("recall() requires a non-empty string");
     }
 
-    const records = this._store.all();
-    if (!records.length) return null;
-
-    const t0 = Date.now();
-    const result = await recall(query, records, {
-      embedFn: this._opts.embedFn,
-      ...opts,
-    });
-    const latencyMs = Date.now() - t0;
-
-    if (this._audit) {
-      this._audit.logRecall(query, result, latencyMs);
-    }
-
-    return result ? result.content : null;
+    const start = Date.now();
+    const hits = await this._rank(query, 1, opts);
+    const hit = hits[0] || null;
+    const latency = Date.now() - start;
+    if (this._audit) this._audit.logRecall(query, hit, latency);
+    return hit ? hit.content : null;
   }
 
-  /**
-   * Return the top-k most relevant memories.
-   *
-   * @param {string} query
-   * @param {number} [k=5]
-   * @param {object} [opts]   Same opts as recall()
-   * @returns {Array<{ id, content, ts, score }>}
-   */
   async recallMany(query, k = 5, opts = {}) {
     this._assertOpen();
-    if (typeof query !== 'string' || !query.trim()) {
-      throw new TypeError('recallMany() requires a non-empty string');
+    if (typeof query !== "string" || !query.trim()) {
+      throw new TypeError("recallMany() requires a non-empty string");
     }
 
-    const records = this._store.all();
-    if (!records.length) return [];
-
-    const results = await recallMany(query, records, {
-      embedFn: this._opts.embedFn,
-      k,
-      ...opts,
-    });
-
-    return results.map(r => ({
-      id: r.id,
-      content: r.content,
-      ts: r.ts,
-      score: r._score,
+    return (await this._rank(query, k, opts)).map((record) => ({
+      id: record.id,
+      content: record.content,
+      ts: record.ts,
+      score: record._score,
     }));
   }
 
-  /**
-   * List all stored memories.
-   *
-   * @returns {Array<{ id, content, ts }>}
-   */
   async list() {
     this._assertOpen();
-    return this._store.all().map(r => ({
-      id: r.id,
-      content: r.content,
-      ts: r.ts,
+    return this._store.all().map((record) => ({
+      id: record.id,
+      content: record.content,
+      ts: record.ts,
     }));
   }
 
-  /**
-   * Materialize normalized temporal events from WAL-backed records.
-   *
-   * @param {object} [opts]
-   * @param {string} [opts.query]  Optional query to shortlist candidate records first
-   * @param {number} [opts.topK=50]
-   * @returns {Array<object>}
-   */
   async events(opts = {}) {
     this._assertOpen();
     const records = this._store.all();
     if (!records.length) return [];
-
-    if (!opts.query) {
-      return this._getTemporalCache().events;
-    }
-
-    if (opts.query && typeof opts.query === 'string' && opts.query.trim()) {
+    if (!opts.query) return this._getTemporalCache().events;
+    if (opts.query && typeof opts.query === "string" && opts.query.trim()) {
       const topK = Number.isFinite(opts.topK) ? Number(opts.topK) : 50;
-      const shortlisted = await recallMany(opts.query, records, {
+      const hits = await scoreRecallMany(opts.query, records, {
         embedFn: this._opts.embedFn,
         k: topK,
       });
-      return materializeEvents(shortlisted);
+      return materializeEvents(hits);
     }
-
     return this._getTemporalCache().events;
   }
 
-  /**
-   * Return derived event indexes built from WAL-backed records.
-   *
-   * @param {object} [opts]
-   * @param {string} [opts.query]
-   * @param {number} [opts.topK=50]
-   * @returns {object}
-   */
   async eventIndex(opts = {}) {
     this._assertOpen();
     const records = this._store.all();
-    if (!records.length) {
-      return materializeEventIndex([]);
-    }
-
-    if (!opts.query) {
-      return this._getTemporalCache();
-    }
-
+    if (!records.length) return materializeEventIndex([]);
+    if (!opts.query) return this._getTemporalCache();
     const topK = Number.isFinite(opts.topK) ? Number(opts.topK) : 50;
-    const shortlisted = await recallMany(opts.query, records, {
+    const hits = await scoreRecallMany(opts.query, records, {
       embedFn: this._opts.embedFn,
       k: topK,
     });
-    return materializeEventIndex(shortlisted);
+    return materializeEventIndex(hits);
   }
 
-  /**
-   * Run a deterministic temporal query over WAL-backed records.
-   *
-   * @param {string} query
-   * @param {object} [opts]
-   * @param {number} [opts.topK=50]
-   * @param {Date|string|number} [opts.now]
-   * @returns {object}
-   */
   async temporalQuery(query, opts = {}) {
     this._assertOpen();
-    if (typeof query !== 'string' || !query.trim()) {
-      throw new TypeError('temporalQuery() requires a non-empty string');
+    if (typeof query !== "string" || !query.trim()) {
+      throw new TypeError("temporalQuery() requires a non-empty string");
     }
 
     const records = this._store.all();
@@ -333,157 +253,126 @@ class MumpixDB {
         kind: null,
         value: null,
         evidence: [],
-        warnings: ['No records available.'],
+        warnings: ["No records available."],
         parsed_query: null,
         events: [],
       };
     }
 
     const topK = Number.isFinite(opts.topK) ? Number(opts.topK) : 50;
-    const shortlisted = await recallMany(query, records, {
+    const hits = await scoreRecallMany(query, records, {
       embedFn: this._opts.embedFn,
       k: topK,
     });
-
     const now =
-      opts.now instanceof Date ? opts.now :
-      (typeof opts.now === 'string' || typeof opts.now === 'number') ? new Date(opts.now) :
-      new Date();
-
-    return answerTemporalQuery(query, shortlisted, { now });
+      opts.now instanceof Date
+        ? opts.now
+        : typeof opts.now === "string" || typeof opts.now === "number"
+          ? new Date(opts.now)
+          : new Date();
+    return answerTemporalQuery(query, hits, { now });
   }
 
-  /**
-   * Delete all memories.
-   *
-   * @returns {{ cleared: boolean, count: number }}
-   */
   async clear() {
     this._assertOpen();
     const count = this._store.clear();
     this._invalidateTemporalCache();
-
-    if (this._audit) {
-      this._audit.logClear(count);
-    }
-
+    if (this._audit) this._audit.logClear(count);
     return { cleared: true, count };
   }
 
-  /**
-   * Get audit log entries (Verified mode only).
-   *
-   * @returns {Array<object>}
-   */
   async audit() {
     this._assertOpen();
     if (!this._audit) {
-      throw new Error('audit() requires consistency: "verified". Current mode: ' + this._store.header.consistency);
+      throw new Error(
+        `audit() requires consistency: "verified". Current mode: ${this._store.header.consistency}`
+      );
     }
     return this._audit.all();
   }
 
-  /**
-   * Get audit log summary (Verified mode only).
-   */
   async auditSummary() {
     this._assertOpen();
-    if (!this._audit) {
-      throw new Error('auditSummary() requires consistency: "verified"');
-    }
+    if (!this._audit) throw new Error('auditSummary() requires consistency: "verified"');
     return this._audit.summary();
   }
 
-  /**
-   * Export audit log as NDJSON string (Verified mode only).
-   */
   async exportAudit() {
     this._assertOpen();
-    if (!this._audit) {
-      throw new Error('exportAudit() requires consistency: "verified"');
-    }
+    if (!this._audit) throw new Error('exportAudit() requires consistency: "verified"');
     return this._audit.export();
   }
 
-  /**
-   * Database metadata and stats.
-   */
   async stats() {
     this._assertOpen();
     return this._store.stats();
   }
 
-  /**
-   * Close the database and release file handles.
-   */
-  async close() {
-    if (this._closed) return;
-
-    // Sync usage metrics in background on close
-    this._license.syncUsage(this._store.stats()).catch(() => { });
-
-    this._store.close();
-    if (this._audit) this._audit.close();
-    this._closed = true;
+  async flush() {
+    this._assertOpen();
+    this._store.flush();
+    return { flushed: true };
   }
 
-  // ─────────────────────────────────────────
-  // Convenience / fluent helpers
-  // ─────────────────────────────────────────
+  async rebuildIndex() {
+    this._assertOpen();
+    return this._store.rebuildIndex();
+  }
 
-  /**
-   * Add multiple memories at once.
-   *
-   * @param {string[]} items
-   */
-  async rememberAll(items) {
+  async indexStats() {
     this._assertOpen();
-    if (!Array.isArray(items)) throw new TypeError('rememberAll() expects an array');
-    const results = [];
-    for (const item of items) {
-      results.push(await this.remember(item));
+    return this._store.indexStats();
+  }
+
+  async close() {
+    if (this._closed) return;
+    try {
+      if (this._opts.telemetry && this._license.hasVerifiedLicense()) {
+        await this._license.syncUsage(this._store.stats());
+      }
+    } finally {
+      this._store.close();
+      if (this._audit) this._audit.close();
+      this._closed = true;
     }
-    return results;
   }
 
-  /**
-   * Check if any memory semantically matches a query.
-   *
-   * @param {string} query
-   * @returns {boolean}
-   */
   async has(query) {
-    const result = await this.recall(query);
-    return result !== null;
+    return (await this.recall(query)) !== null;
   }
 
-  /**
-   * Return the number of stored memories.
-   */
   get size() {
     return this._store.records.length;
   }
 
-  /**
-   * Consistency mode of this instance.
-   */
   get consistency() {
     return this._store.header.consistency;
   }
 
-  /**
-   * File path of the underlying .mumpix file.
-   */
   get path() {
     return this._store.filePath;
   }
 
-  // ─────────────────────────────────────────
-  // Private
-  // ─────────────────────────────────────────
+  async _rank(query, k, opts = {}) {
+    const allRecords = this._store.all();
+    if (!allRecords.length) return [];
+
+    let candidates = null;
+    if (opts.recallIndex !== false && typeof this._store.indexedCandidates === "function") {
+      candidates = this._store.indexedCandidates(query, opts.candidateCap || 400);
+    }
+
+    const records = candidates && candidates.length ? candidates : allRecords;
+    const ranked = await scoreRecallMany(query, records, {
+      embedFn: this._opts.embedFn,
+      k,
+      ...opts,
+    });
+    return ranked.filter((record) => this._store._verifyHash(record));
+  }
 
   _assertOpen() {
-    if (this._closed) throw new Error('Database is closed. Call Mumpix.open() again.');
+    if (this._closed) throw new Error("Database is closed. Call Mumpix.open() again.");
     this._license.assertActive(this._store.header.consistency);
   }
 
@@ -493,8 +382,8 @@ class MumpixDB {
 
   _temporalSignature() {
     const records = this._store.records || [];
-    const last = records.length ? records[records.length - 1] : null;
-    return `${records.length}:${last ? last.id : 0}:${last ? last.ts : 0}`;
+    const tail = records.length ? records[records.length - 1] : null;
+    return `${records.length}:${tail ? tail.id : 0}:${tail ? tail.ts : 0}`;
   }
 
   _getTemporalCache() {
@@ -502,7 +391,6 @@ class MumpixDB {
     if (this._temporalCache && this._temporalCache.signature === signature) {
       return this._temporalCache.index;
     }
-
     const index = materializeEventIndex(this._store.all());
     this._temporalCache = { signature, index };
     return index;