@wipcomputer/memory-crystal 0.7.32 → 0.7.34-alpha.1

package/dist/cloud-crystal.js

@@ -0,0 +1,6 @@
+import {
+  CloudCrystal
+} from "./chunk-J7MRSZIO.js";
+export {
+  CloudCrystal
+};

package/dist/core.d.ts

@@ -0,0 +1,252 @@
+interface CrystalConfig {
+    /** Root directory for all crystal data */
+    dataDir: string;
+    /** Embedding provider: 'openai' | 'ollama' | 'google' */
+    embeddingProvider: 'openai' | 'ollama' | 'google';
+    /** OpenAI API key (required if provider is 'openai') */
+    openaiApiKey?: string;
+    /** OpenAI embedding model (default: text-embedding-3-small) */
+    openaiModel?: string;
+    /** Ollama host (default: http://localhost:11434) */
+    ollamaHost?: string;
+    /** Ollama model (default: nomic-embed-text) */
+    ollamaModel?: string;
+    /** Google API key (required if provider is 'google') */
+    googleApiKey?: string;
+    /** Google embedding model (default: text-embedding-004) */
+    googleModel?: string;
+    /** Remote Worker URL for cloud mirror mode */
+    remoteUrl?: string;
+    /** Remote auth token */
+    remoteToken?: string;
+}
+interface Chunk {
+    id?: number;
+    text: string;
+    embedding?: number[];
+    role: 'user' | 'assistant' | 'system';
+    source_type: string;
+    source_id: string;
+    agent_id: string;
+    token_count: number;
+    created_at: string;
+}
+/** Pre-embedded chunk for delta sync. Includes embedding vector so Nodes don't re-embed. */
+interface ExportedChunk {
+    id: number;
+    text: string;
+    text_hash: string;
+    role: string;
+    source_type: string;
+    source_id: string;
+    agent_id: string;
+    token_count: number;
+    created_at: string;
+    embedding: number[] | null;
+}
+interface Memory {
+    id?: number;
+    text: string;
+    embedding?: number[];
+    category: 'fact' | 'preference' | 'event' | 'opinion' | 'skill' | 'user' | 'feedback' | 'project' | 'reference';
+    confidence: number;
+    source_ids: string;
+    status: 'active' | 'deprecated' | 'deleted';
+    created_at: string;
+    updated_at: string;
+}
+interface SearchResult {
+    text: string;
+    role: string;
+    score: number;
+    source_type: string;
+    source_id: string;
+    agent_id: string;
+    created_at: string;
+    freshness?: "fresh" | "recent" | "aging" | "stale";
+}
+/** Pre-expanded query for unified search API. Skip LLM expansion when you know what you want. */
+interface StructuredQuery {
+    type: 'lex' | 'vec' | 'hyde';
+    text: string;
+}
+interface CrystalStatus {
+    chunks: number;
+    memories: number;
+    sources: number;
+    agents: string[];
+    oldestChunk: string | null;
+    newestChunk: string | null;
+    embeddingProvider: string;
+    dataDir: string;
+    capturedSessions: number;
+    latestCapture: string | null;
+}
+interface SourceCollection {
+    id?: number;
+    name: string;
+    root_path: string;
+    glob_patterns: string;
+    ignore_patterns: string;
+    file_count: number;
+    chunk_count: number;
+    last_sync_at: string | null;
+    created_at: string;
+}
+interface SourceFile {
+    id?: number;
+    collection_id: number;
+    file_path: string;
+    file_hash: string;
+    file_size: number;
+    chunk_count: number;
+    last_indexed_at: string;
+}
+interface SourcesStatus {
+    collections: Array<{
+        name: string;
+        root_path: string;
+        file_count: number;
+        chunk_count: number;
+        last_sync_at: string | null;
+    }>;
+    total_files: number;
+    total_chunks: number;
+}
+interface SyncResult {
+    collection: string;
+    added: number;
+    updated: number;
+    removed: number;
+    chunks_added: number;
+    duration_ms: number;
+}
+declare class Crystal {
+    private config;
+    private lanceDb;
+    private sqliteDb;
+    private chunksTable;
+    private vecDimensions;
+    constructor(config: CrystalConfig);
+    init(): Promise<void>;
+    private initSqliteTables;
+    private initChunksTables;
+    private ensureVecTable;
+    private initLanceTables;
+    embed(texts: string[]): Promise<number[][]>;
+    chunkText(text: string, targetTokens?: number, overlapTokens?: number): string[];
+    ingest(chunks: Chunk[]): Promise<number>;
+    /** Export interface for delta sync payloads. */
+    static readonly DELTA_VERSION = 1;
+    /** Export chunks with IDs greater than sinceId. Returns pre-embedded chunks for delta sync.
+     *  Core calls this to build delta payloads for Nodes. */
+    exportChunksSince(sinceId: number): ExportedChunk[];
+    /** Get the highest chunk ID in the database. Used for watermark tracking. */
+    getMaxChunkId(): number;
+    /** Import pre-embedded chunks from Core. Node calls this to apply delta payloads.
+     *  Skips chunks that already exist (by text_hash). Does NOT re-embed. */
+    importChunks(exported: ExportedChunk[]): number;
+    private recencyWeight;
+    /** Parse relative time strings ("24h", "7d", "30d") or ISO dates into ISO date strings. */
+    private parseSince;
+    private freshnessLabel;
+    search(query: string, limit?: number, filter?: {
+        agent_id?: string;
+        source_type?: string;
+        since?: string;
+        until?: string;
+    }): Promise<SearchResult[]>;
+    /** Deep search: query expansion + LLM re-ranking + position-aware blending.
+     *  Falls back to standard search if no LLM provider is available.
+     *  Supports intent disambiguation, candidateLimit tuning, and explain traces. */
+    deepSearch(query: string, limit?: number, filter?: {
+        agent_id?: string;
+        source_type?: string;
+        since?: string;
+        until?: string;
+    }, options?: {
+        intent?: string;
+        candidateLimit?: number;
+        explain?: boolean;
+    }): Promise<SearchResult[]>;
+    /** Structured search: pass pre-expanded queries to skip LLM expansion.
+     *  Each query is typed (lex, vec, hyde) and searched independently, then fused with RRF. */
+    structuredSearch(queries: StructuredQuery[], limit?: number, filter?: {
+        agent_id?: string;
+        source_type?: string;
+        since?: string;
+        until?: string;
+    }): Promise<SearchResult[]>;
+    /** Vector search via sqlite-vec. Two-step pattern: MATCH first, then JOIN. */
+    private searchVec;
+    /** Full-text search via FTS5 with BM25 scoring. */
+    private searchFTS;
+    /** Build a safe FTS5 query from user input. */
+    private buildFTS5Query;
+    /**
+     * Reciprocal Rank Fusion. Ported from QMD (MIT License, Tobi Lutke, 2024-2026).
+     * Fuses multiple ranked result lists into one using RRF scoring.
+     * Uses text content as dedup key (instead of QMD's file path).
+     */
+    private reciprocalRankFusion;
+    /** LanceDB fallback for search (used when sqlite-vec tables are empty, pre-migration). */
+    private searchLanceFallback;
+    remember(text: string, category?: Memory['category']): Promise<number>;
+    forget(memoryId: number): boolean;
+    status(): Promise<CrystalStatus>;
+    getCaptureState(agentId: string, sourceId: string): {
+        lastMessageCount: number;
+        captureCount: number;
+    };
+    setCaptureState(agentId: string, sourceId: string, messageCount: number, captureCount: number): void;
+    private static readonly DEFAULT_INCLUDE;
+    private static readonly DEFAULT_IGNORE;
+    /** Add a directory as a source collection for indexing. */
+    sourcesAdd(rootPath: string, name: string, options?: {
+        include?: string[];
+        ignore?: string[];
+    }): Promise<SourceCollection>;
+    /** Remove a source collection and its file records. Chunks remain in LanceDB. */
+    sourcesRemove(name: string): boolean;
+    /** Sync a collection: scan files, detect changes, re-index what changed. */
+    sourcesSync(name: string, options?: {
+        dryRun?: boolean;
+        batchSize?: number;
+    }): Promise<SyncResult>;
+    /** Get status of all source collections. */
+    sourcesStatus(): SourcesStatus;
+    /** Scan a directory recursively, matching include/ignore patterns. */
+    private scanDirectory;
+    /** Clean orphaned entries in chunks_vec and chunks_fts that no longer have
+     *  corresponding rows in the chunks table. Returns counts of what was found/cleaned. */
+    cleanOrphans(options?: {
+        dryRun?: boolean;
+    }): {
+        orphanedVec: number;
+        orphanedFts: number;
+        cleanedVec: number;
+        cleanedFts: number;
+        dryRun: boolean;
+    };
+    close(): void;
+}
+declare function resolveConfig(overrides?: Partial<CrystalConfig>): CrystalConfig;
+declare class RemoteCrystal {
+    private url;
+    private token;
+    constructor(url: string, token: string);
+    init(): Promise<void>;
+    private request;
+    search(query: string, limit?: number, filter?: {
+        agent_id?: string;
+    }): Promise<SearchResult[]>;
+    ingest(chunks: Chunk[]): Promise<number>;
+    remember(text: string, category?: Memory['category']): Promise<number>;
+    forget(memoryId: number): Promise<boolean>;
+    status(): Promise<CrystalStatus>;
+    chunkText(text: string): string[];
+}
+/** Create the appropriate Crystal instance based on config. */
+declare function createCrystal(config: CrystalConfig): Crystal | RemoteCrystal;
+
+export { type Chunk, Crystal, type CrystalConfig, type CrystalStatus, type ExportedChunk, type Memory, RemoteCrystal, type SearchResult, type SourceCollection, type SourceFile, type SourcesStatus, type StructuredQuery, type SyncResult, createCrystal, resolveConfig };

package/dist/core.js

@@ -0,0 +1,12 @@
+import {
+  Crystal,
+  RemoteCrystal,
+  createCrystal,
+  resolveConfig
+} from "./chunk-2GBYLMEF.js";
+export {
+  Crystal,
+  RemoteCrystal,
+  createCrystal,
+  resolveConfig
+};

package/dist/crypto.d.ts

@@ -0,0 +1,20 @@
+declare function loadRelayKey(): Buffer;
+interface EncryptedPayload {
+    v: 1;
+    nonce: string;
+    ciphertext: string;
+    tag: string;
+    hmac: string;
+}
+declare function encrypt(plaintext: Buffer, masterKey: Buffer): EncryptedPayload;
+declare function decrypt(payload: EncryptedPayload, masterKey: Buffer): Buffer;
+declare function encryptJSON(data: unknown, masterKey: Buffer): EncryptedPayload;
+declare function decryptJSON<T = unknown>(payload: EncryptedPayload, masterKey: Buffer): T;
+declare function encryptFile(filePath: string, masterKey: Buffer): EncryptedPayload;
+declare const RELAY_KEY_PATH: string;
+declare function generateRelayKey(): Buffer;
+declare function encodePairingString(key: Buffer): string;
+declare function decodePairingString(str: string): Buffer;
+declare function hashBuffer(data: Buffer): string;
+
+export { type EncryptedPayload, RELAY_KEY_PATH, decodePairingString, decrypt, decryptJSON, encodePairingString, encrypt, encryptFile, encryptJSON, generateRelayKey, hashBuffer, loadRelayKey };

package/dist/crypto.js

@@ -0,0 +1,27 @@
+import {
+  RELAY_KEY_PATH,
+  decodePairingString,
+  decrypt,
+  decryptJSON,
+  encodePairingString,
+  encrypt,
+  encryptFile,
+  encryptJSON,
+  generateRelayKey,
+  hashBuffer,
+  loadRelayKey
+} from "./chunk-D3MACYZ4.js";
+import "./chunk-DFQ72B7M.js";
+export {
+  RELAY_KEY_PATH,
+  decodePairingString,
+  decrypt,
+  decryptJSON,
+  encodePairingString,
+  encrypt,
+  encryptFile,
+  encryptJSON,
+  generateRelayKey,
+  hashBuffer,
+  loadRelayKey
+};

package/dist/crystal-capture.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Job: crystal-capture
+# Continuous capture for Claude Code sessions.
+# Reads JSONL files on disk, ingests into Crystal, exports MD sessions, writes daily logs.
+# Primary capture path. Runs every minute via cron.
+# The Stop hook (cc-hook.ts) is a redundancy check only.
+#
+# Source of truth: memory-crystal-private/scripts/crystal-capture.sh
+# Deployed to: ~/.ldm/bin/crystal-capture.sh (via crystal init)
+# Cron entry: * * * * * ~/.ldm/bin/crystal-capture.sh >> /tmp/ldm-dev-tools/crystal-capture.log 2>&1
+#
+# The Node poller fetches the OpenAI API key internally via opRead() in core.ts.
+# opRead uses: op read "op://Agent Secrets/OpenAI API/api key" with the SA token from
+# ~/.openclaw/secrets/op-sa-token. Do NOT call op from this shell script... it triggers
+# macOS TCC popups when run from cron.
+
+# Cron provides minimal PATH. Ensure Homebrew binaries (node, op) are findable.
+export PATH="/opt/homebrew/bin:$PATH"
+
+POLLER="$HOME/.ldm/extensions/memory-crystal/dist/cc-poller.js"
+NODE="/opt/homebrew/bin/node"
+
+if [ ! -f "$POLLER" ]; then
+  echo "ERROR: cc-poller not found at $POLLER"
+  exit 1
+fi
+
+# Single run: scan all sessions, ingest new turns, export MD, exit.
+$NODE "$POLLER" 2>&1

package/dist/crystal-serve.d.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+declare function startServer(port?: number): void;
+
+export { startServer };

package/dist/crystal-serve.js

@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+import {
+  ldmPaths
+} from "./chunk-DFQ72B7M.js";
+
+// src/crystal-serve.ts
+import { createServer } from "http";
+import { spawn } from "child_process";
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+var DEFAULT_PORT = 18790;
+var AUTH_TOKEN = process.env.CRYSTAL_SERVE_TOKEN || "";
+function checkAuth(req) {
+  if (!AUTH_TOKEN) return true;
+  const authHeader = req.headers.authorization;
+  if (!authHeader) return false;
+  return authHeader === `Bearer ${AUTH_TOKEN}`;
+}
+function sendJson(res, status, body) {
+  res.writeHead(status, { "Content-Type": "application/json" });
+  res.end(JSON.stringify(body));
+}
+function send401(res) {
+  sendJson(res, 401, { error: "Unauthorized" });
+}
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    let body = "";
+    req.on("data", (chunk) => {
+      body += chunk;
+    });
+    req.on("end", () => resolve(body));
+    req.on("error", reject);
+    req.on("data", () => {
+      if (body.length > 1e6) {
+        req.destroy();
+        reject(new Error("Request body too large"));
+      }
+    });
+  });
+}
+async function handleChatCompletions(req, res) {
+  const raw = await readBody(req);
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    sendJson(res, 400, { error: "Invalid JSON" });
+    return;
+  }
+  const messages = parsed.messages;
+  if (!messages || !Array.isArray(messages) || messages.length === 0) {
+    sendJson(res, 400, { error: "messages array required" });
+    return;
+  }
+  const lastUser = [...messages].reverse().find((m) => m.role === "user");
+  if (!lastUser) {
+    sendJson(res, 400, { error: "No user message found" });
+    return;
+  }
+  const prompt = typeof lastUser.content === "string" ? lastUser.content : Array.isArray(lastUser.content) ? lastUser.content.filter((b) => b.type === "text").map((b) => b.text).join("\n") : "";
+  if (!prompt) {
+    sendJson(res, 400, { error: "Empty prompt" });
+    return;
+  }
+  const systemMsg = messages.find((m) => m.role === "system");
+  const systemPrompt = systemMsg ? typeof systemMsg.content === "string" ? systemMsg.content : "" : void 0;
+  try {
+    const args2 = ["-p", prompt];
+    if (systemPrompt) args2.push("--system", systemPrompt);
+    args2.push("--output-format", "text");
+    const result = await new Promise((resolve, reject) => {
+      const proc = spawn("claude", args2, {
+        stdio: ["ignore", "pipe", "pipe"],
+        env: { ...process.env },
+        timeout: 3e5
+      });
+      let stdout = "";
+      let stderr = "";
+      proc.stdout.on("data", (data) => {
+        stdout += data.toString();
+      });
+      proc.stderr.on("data", (data) => {
+        stderr += data.toString();
+      });
+      proc.on("close", (code) => {
+        if (code === 0) resolve(stdout.trim());
+        else reject(new Error(`claude exited ${code}: ${stderr.slice(0, 300)}`));
+      });
+      proc.on("error", reject);
+    });
+    sendJson(res, 200, {
+      id: `crystal-${Date.now()}`,
+      object: "chat.completion",
+      created: Math.floor(Date.now() / 1e3),
+      model: "claude-opus-4-6",
+      choices: [{
+        index: 0,
+        message: { role: "assistant", content: result },
+        finish_reason: "stop"
+      }]
+    });
+  } catch (err) {
+    sendJson(res, 500, { error: err.message });
+  }
+}
+async function handleProcess(req, res) {
+  const raw = await readBody(req);
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    sendJson(res, 400, { error: "Invalid JSON" });
+    return;
+  }
+  const agentId = parsed.agent_id || parsed.agentId;
+  const action = parsed.action || "dream-weave";
+  if (!agentId) {
+    sendJson(res, 400, { error: "agent_id required" });
+    return;
+  }
+  const args2 = [];
+  if (action === "backfill") {
+    args2.push("backfill", "--agent", agentId);
+  } else if (action === "dream-weave") {
+    args2.push("dream-weave", "--agent", agentId, "--mode", parsed.mode || "incremental");
+  } else {
+    sendJson(res, 400, { error: `Unknown action: ${action}` });
+    return;
+  }
+  try {
+    const proc = spawn("crystal", args2, {
+      stdio: "ignore",
+      detached: true,
+      env: { ...process.env }
+    });
+    proc.unref();
+    sendJson(res, 202, {
+      status: "accepted",
+      action,
+      agent_id: agentId,
+      message: `${action} started for ${agentId}`
+    });
+  } catch (err) {
+    sendJson(res, 500, { error: err.message });
+  }
+}
+function handleStatus(_req, res) {
+  const paths = ldmPaths();
+  const hasDb = existsSync(paths.crystalDb);
+  let lastDreamWeaver = null;
+  const agentId = process.env.CRYSTAL_AGENT_ID || "cc-mini";
+  const wmPath = join(paths.state, `dream-weaver-${agentId}.json`);
+  if (existsSync(wmPath)) {
+    try {
+      const wm = JSON.parse(readFileSync(wmPath, "utf-8"));
+      lastDreamWeaver = wm.lastRunAt || null;
+    } catch {
+    }
+  }
+  let role = "core";
+  const rolePath = join(paths.state, "role.json");
+  if (existsSync(rolePath)) {
+    try {
+      const r = JSON.parse(readFileSync(rolePath, "utf-8"));
+      role = r.role || "core";
+    } catch {
+    }
+  }
+  sendJson(res, 200, {
+    status: "ok",
+    role,
+    agentId,
+    hasDatabase: hasDb,
+    lastDreamWeaver,
+    uptime: process.uptime(),
+    version: "0.6.0"
+  });
+}
+function startServer(port2 = DEFAULT_PORT) {
+  const server = createServer(async (req, res) => {
+    const url = req.url || "";
+    const method = req.method || "GET";
+    res.setHeader("Access-Control-Allow-Origin", "http://localhost:*");
+    res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+    res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization");
+    if (method === "OPTIONS") {
+      res.writeHead(204);
+      res.end();
+      return;
+    }
+    if (url !== "/status" && !checkAuth(req)) {
+      send401(res);
+      return;
+    }
+    try {
+      if (method === "POST" && url === "/v1/chat/completions") {
+        await handleChatCompletions(req, res);
+      } else if (method === "POST" && url === "/process") {
+        await handleProcess(req, res);
+      } else if (method === "GET" && url === "/status") {
+        handleStatus(req, res);
+      } else {
+        sendJson(res, 404, { error: "Not found" });
+      }
+    } catch (err) {
+      sendJson(res, 500, { error: err.message });
+    }
+  });
+  server.listen(port2, "127.0.0.1", () => {
+    console.log(`Crystal Core gateway listening on http://127.0.0.1:${port2}`);
+    console.log(`  POST /v1/chat/completions   (OpenAI-compatible)`);
+    console.log(`  POST /process               (trigger backfill/dream-weaver)`);
+    console.log(`  GET  /status                (health check)`);
+    if (AUTH_TOKEN) {
+      console.log(`  Auth: bearer token required`);
+    } else {
+      console.log(`  Auth: none (set CRYSTAL_SERVE_TOKEN to enable)`);
+    }
+  });
+  process.on("SIGTERM", () => {
+    console.log("Crystal Core gateway shutting down...");
+    server.close();
+    process.exit(0);
+  });
+  process.on("SIGINT", () => {
+    console.log("Crystal Core gateway shutting down...");
+    server.close();
+    process.exit(0);
+  });
+}
+var args = process.argv.slice(2);
+if (args.includes("--help") || args.includes("-h")) {
+  console.log(`crystal serve [--port <port>]
+
+Crystal Core gateway. Localhost-only HTTP server.
+
+Options:
+  --port <port>    Port to listen on (default: ${DEFAULT_PORT})
+
+Environment:
+  CRYSTAL_SERVE_TOKEN    Bearer token for auth (optional)
+  CRYSTAL_AGENT_ID       Agent identifier (default: cc-mini)
+`);
+  process.exit(0);
+}
+var portFlag = args.indexOf("--port");
+var port = portFlag >= 0 ? parseInt(args[portFlag + 1], 10) : DEFAULT_PORT;
+startServer(port);
+export {
+  startServer
+};

package/dist/dev-update-SZ2Z4WCQ.js

@@ -0,0 +1,6 @@
+import {
+  runDevUpdate
+} from "./chunk-MK42FMEG.js";
+export {
+  runDevUpdate
+};

package/dist/discover.d.ts

@@ -0,0 +1,30 @@
+interface HarnessInfo {
+    platform: 'claude-code' | 'openclaw';
+    sessionDir: string;
+    workspaceDir?: string;
+    filePattern: string;
+    agentIdDefault: string;
+}
+interface DiscoveryBreakdown {
+    platform: string;
+    files: number;
+    sizeBytes: number;
+    sessionPaths: string[];
+}
+interface DiscoveryResult {
+    harnesses: HarnessInfo[];
+    totalFiles: number;
+    totalSizeBytes: number;
+    breakdown: DiscoveryBreakdown[];
+}
+/** Detect which agent platforms are installed on this machine. */
+declare function discoverHarnesses(): HarnessInfo[];
+/** Discover all session JSONL files for a given harness. */
+declare function discoverSessionFiles(harness: HarnessInfo): string[];
+/** Discover workspace .md files for OpenClaw harness. */
+declare function discoverWorkspaceFiles(harness: HarnessInfo): string[];
+/** Run full discovery: detect harnesses, find all files, compute sizes. */
+declare function discoverAll(): DiscoveryResult;
+declare function formatBytes(bytes: number): string;
+
+export { type DiscoveryBreakdown, type DiscoveryResult, type HarnessInfo, discoverAll, discoverHarnesses, discoverSessionFiles, discoverWorkspaceFiles, formatBytes };