@wipcomputer/memory-crystal 0.7.29 → 0.7.32

package/dist/chunk-CCYI5O3D.js

@@ -1,148 +0,0 @@
-import {
-  ensureLdm,
-  init_ldm,
-  ldmPaths
-} from "./chunk-PSQZURHO.js";
-
-// src/staging.ts
-init_ldm();
-import {
-  existsSync,
-  mkdirSync,
-  readdirSync,
-  renameSync,
-  unlinkSync,
-  writeFileSync
-} from "fs";
-import { join } from "path";
-import { execSync } from "child_process";
-var HOME = process.env.HOME || "";
-var STAGING_ROOT = join(HOME, ".ldm", "staging");
-function stagingPaths(agentId) {
-  const root = join(STAGING_ROOT, agentId);
-  return {
-    root,
-    transcripts: join(root, "transcripts"),
-    readyFile: join(root, "READY")
-  };
-}
-function ensureStaging(agentId) {
-  const paths = stagingPaths(agentId);
-  mkdirSync(paths.transcripts, { recursive: true });
-  return paths;
-}
-function markReady(agentId) {
-  const paths = stagingPaths(agentId);
-  writeFileSync(paths.readyFile, JSON.stringify({
-    markedAt: (/* @__PURE__ */ new Date()).toISOString(),
-    agentId
-  }));
-}
-function isNewAgent(agentId) {
-  const paths = ldmPaths(agentId);
-  return !existsSync(paths.agentRoot);
-}
-function hasStagedData(agentId) {
-  const paths = stagingPaths(agentId);
-  return existsSync(paths.readyFile);
-}
-function listStagedAgents() {
-  if (!existsSync(STAGING_ROOT)) return [];
-  const agents = [];
-  for (const entry of readdirSync(STAGING_ROOT, { withFileTypes: true })) {
-    if (!entry.isDirectory()) continue;
-    if (existsSync(join(STAGING_ROOT, entry.name, "READY"))) {
-      agents.push(entry.name);
-    }
-  }
-  return agents;
-}
-async function processStagedAgent(agentId) {
-  const startTime = Date.now();
-  const staging = stagingPaths(agentId);
-  if (!existsSync(staging.readyFile)) {
-    throw new Error(`No READY file for agent ${agentId}`);
-  }
-  const agentPaths = ensureLdm(agentId);
-  const stagedFiles = existsSync(staging.transcripts) ? readdirSync(staging.transcripts).filter((f) => f.endsWith(".jsonl")) : [];
-  if (stagedFiles.length === 0) {
-    unlinkSync(staging.readyFile);
-    return {
-      agentId,
-      transcriptsProcessed: 0,
-      backfillChunks: 0,
-      dreamWeaverRan: false,
-      durationMs: Date.now() - startTime
-    };
-  }
-  for (const file of stagedFiles) {
-    const src = join(staging.transcripts, file);
-    const dest = join(agentPaths.transcripts, file);
-    try {
-      renameSync(src, dest);
-    } catch {
-      const { copyFileSync } = await import("fs");
-      copyFileSync(src, dest);
-      unlinkSync(src);
-    }
-  }
-  let backfillChunks = 0;
-  let dreamWeaverRan = false;
-  try {
-    const output = execSync(
-      `crystal backfill --agent ${agentId}`,
-      { encoding: "utf-8", timeout: 6e5 }
-    );
-    const match = output.match(/(\d+) chunks embedded/);
-    if (match) backfillChunks = parseInt(match[1], 10);
-  } catch (err) {
-    process.stderr.write(`[staging] backfill failed for ${agentId}: ${err.message}
-`);
-  }
-  try {
-    execSync(
-      `crystal dream-weave --agent ${agentId} --mode full`,
-      { encoding: "utf-8", timeout: 6e5 }
-    );
-    dreamWeaverRan = true;
-  } catch (err) {
-    process.stderr.write(`[staging] dream-weave failed for ${agentId}: ${err.message}
-`);
-  }
-  try {
-    unlinkSync(staging.readyFile);
-  } catch {
-  }
-  return {
-    agentId,
-    transcriptsProcessed: stagedFiles.length,
-    backfillChunks,
-    dreamWeaverRan,
-    durationMs: Date.now() - startTime
-  };
-}
-async function processAllStaged() {
-  const agents = listStagedAgents();
-  const results = [];
-  for (const agentId of agents) {
-    try {
-      const result = await processStagedAgent(agentId);
-      results.push(result);
-    } catch (err) {
-      process.stderr.write(`[staging] failed to process ${agentId}: ${err.message}
-`);
-    }
-  }
-  return results;
-}
-
-export {
-  stagingPaths,
-  ensureStaging,
-  markReady,
-  isNewAgent,
-  hasStagedData,
-  listStagedAgents,
-  processStagedAgent,
-  processAllStaged
-};

package/dist/chunk-D3I3ZSE2.js

@@ -1,411 +0,0 @@
-// src/core.ts
-import * as lancedb from "@lancedb/lancedb";
-import Database from "better-sqlite3";
-import { readFileSync, existsSync, mkdirSync } from "fs";
-import { execSync } from "child_process";
-import { join } from "path";
-import http from "http";
-import https from "https";
-async function embedOpenAI(texts, apiKey, model) {
-  return new Promise((resolve, reject) => {
-    const body = JSON.stringify({ input: texts, model });
-    const req = https.request({
-      hostname: "api.openai.com",
-      path: "/v1/embeddings",
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "Authorization": `Bearer ${apiKey}`,
-        "Content-Length": Buffer.byteLength(body)
-      },
-      timeout: 3e4
-    }, (res) => {
-      let data = "";
-      res.on("data", (chunk) => data += chunk);
-      res.on("end", () => {
-        if (res.statusCode !== 200) {
-          reject(new Error(`OpenAI API error ${res.statusCode}: ${data.slice(0, 200)}`));
-          return;
-        }
-        const parsed = JSON.parse(data);
-        resolve(parsed.data.map((d) => d.embedding));
-      });
-    });
-    req.on("error", reject);
-    req.on("timeout", () => {
-      req.destroy();
-      reject(new Error("OpenAI timeout"));
-    });
-    req.write(body);
-    req.end();
-  });
-}
-async function embedOllama(texts, host, model) {
-  const results = [];
-  for (const text of texts) {
-    const result = await new Promise((resolve, reject) => {
-      const url = new URL("/api/embeddings", host);
-      const body = JSON.stringify({ model, prompt: text });
-      const req = http.request({
-        hostname: url.hostname,
-        port: url.port,
-        path: url.pathname,
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "Content-Length": Buffer.byteLength(body)
-        },
-        timeout: 15e3
-      }, (res) => {
-        let data = "";
-        res.on("data", (chunk) => data += chunk);
-        res.on("end", () => {
-          if (res.statusCode !== 200) {
-            reject(new Error(`Ollama error ${res.statusCode}: ${data.slice(0, 200)}`));
-            return;
-          }
-          resolve(JSON.parse(data).embedding);
-        });
-      });
-      req.on("error", reject);
-      req.on("timeout", () => {
-        req.destroy();
-        reject(new Error("Ollama timeout"));
-      });
-      req.write(body);
-      req.end();
-    });
-    results.push(result);
-  }
-  return results;
-}
-async function embedGoogle(texts, apiKey, model) {
-  return new Promise((resolve, reject) => {
-    const body = JSON.stringify({
-      requests: texts.map((text) => ({ model: `models/${model}`, content: { parts: [{ text }] } }))
-    });
-    const req = https.request({
-      hostname: "generativelanguage.googleapis.com",
-      path: `/v1beta/models/${model}:batchEmbedContents?key=${apiKey}`,
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "Content-Length": Buffer.byteLength(body)
-      },
-      timeout: 3e4
-    }, (res) => {
-      let data = "";
-      res.on("data", (chunk) => data += chunk);
-      res.on("end", () => {
-        if (res.statusCode !== 200) {
-          reject(new Error(`Google API error ${res.statusCode}: ${data.slice(0, 200)}`));
-          return;
-        }
-        const parsed = JSON.parse(data);
-        resolve(parsed.embeddings.map((e) => e.values));
-      });
-    });
-    req.on("error", reject);
-    req.on("timeout", () => {
-      req.destroy();
-      reject(new Error("Google timeout"));
-    });
-    req.write(body);
-    req.end();
-  });
-}
-var Crystal = class {
-  config;
-  lanceDb = null;
-  sqliteDb = null;
-  chunksTable = null;
-  constructor(config) {
-    this.config = config;
-    if (!existsSync(config.dataDir)) {
-      mkdirSync(config.dataDir, { recursive: true });
-    }
-  }
-  // ── Initialization ──
-  async init() {
-    const lanceDir = join(this.config.dataDir, "lance");
-    const sqlitePath = join(this.config.dataDir, "crystal.db");
-    if (!existsSync(lanceDir)) mkdirSync(lanceDir, { recursive: true });
-    this.lanceDb = await lancedb.connect(lanceDir);
-    this.sqliteDb = new Database(sqlitePath);
-    this.sqliteDb.pragma("journal_mode = WAL");
-    this.initSqliteTables();
-    await this.initLanceTables();
-  }
-  initSqliteTables() {
-    const db = this.sqliteDb;
-    db.exec(`
-      CREATE TABLE IF NOT EXISTS sources (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        type TEXT NOT NULL,
-        uri TEXT NOT NULL,
-        title TEXT,
-        agent_id TEXT NOT NULL,
-        metadata TEXT DEFAULT '{}',
-        ingested_at TEXT NOT NULL,
-        chunk_count INTEGER DEFAULT 0
-      );
-
-      CREATE TABLE IF NOT EXISTS capture_state (
-        agent_id TEXT NOT NULL,
-        source_id TEXT NOT NULL,
-        last_message_count INTEGER DEFAULT 0,
-        capture_count INTEGER DEFAULT 0,
-        last_capture_at TEXT,
-        PRIMARY KEY (agent_id, source_id)
-      );
-
-      CREATE TABLE IF NOT EXISTS memories (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        text TEXT NOT NULL,
-        category TEXT NOT NULL DEFAULT 'fact',
-        confidence REAL NOT NULL DEFAULT 1.0,
-        source_ids TEXT DEFAULT '[]',
-        status TEXT NOT NULL DEFAULT 'active',
-        created_at TEXT NOT NULL,
-        updated_at TEXT NOT NULL
-      );
-
-      CREATE TABLE IF NOT EXISTS entities (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        name TEXT NOT NULL UNIQUE,
-        type TEXT NOT NULL DEFAULT 'concept',
-        description TEXT,
-        properties TEXT DEFAULT '{}',
-        created_at TEXT NOT NULL,
-        updated_at TEXT NOT NULL
-      );
-
-      CREATE TABLE IF NOT EXISTS relationships (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        source_id INTEGER NOT NULL REFERENCES entities(id),
-        target_id INTEGER NOT NULL REFERENCES entities(id),
-        type TEXT NOT NULL,
-        description TEXT,
-        weight REAL DEFAULT 1.0,
-        valid_from TEXT NOT NULL,
-        valid_until TEXT,
-        created_at TEXT NOT NULL
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_sources_agent ON sources(agent_id);
-      CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
-      CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
-      CREATE INDEX IF NOT EXISTS idx_relationships_source ON relationships(source_id);
-      CREATE INDEX IF NOT EXISTS idx_relationships_target ON relationships(target_id);
-    `);
-  }
-  async initLanceTables() {
-    const db = this.lanceDb;
-    const tableNames = await db.tableNames();
-    if (tableNames.includes("chunks")) {
-      this.chunksTable = await db.openTable("chunks");
-    }
-  }
-  // ── Embedding ──
-  async embed(texts) {
-    if (texts.length === 0) return [];
-    const cfg = this.config;
-    switch (cfg.embeddingProvider) {
-      case "openai":
-        if (!cfg.openaiApiKey) throw new Error("OpenAI API key required");
-        return embedOpenAI(texts, cfg.openaiApiKey, cfg.openaiModel || "text-embedding-3-small");
-      case "ollama":
-        return embedOllama(texts, cfg.ollamaHost || "http://localhost:11434", cfg.ollamaModel || "nomic-embed-text");
-      case "google":
-        if (!cfg.googleApiKey) throw new Error("Google API key required");
-        return embedGoogle(texts, cfg.googleApiKey, cfg.googleModel || "text-embedding-004");
-      default:
-        throw new Error(`Unknown embedding provider: ${cfg.embeddingProvider}`);
-    }
-  }
-  // ── Chunking ──
-  chunkText(text, targetTokens = 400, overlapTokens = 80) {
-    const targetChars = targetTokens * 4;
-    const overlapChars = overlapTokens * 4;
-    const chunks = [];
-    let start = 0;
-    while (start < text.length) {
-      let end = Math.min(start + targetChars, text.length);
-      if (end < text.length) {
-        const minBreak = start + Math.floor(targetChars * 0.5);
-        const paraBreak = text.lastIndexOf("\n\n", end);
-        if (paraBreak > minBreak) {
-          end = paraBreak;
-        } else {
-          const sentBreak = text.lastIndexOf(". ", end);
-          if (sentBreak > minBreak) {
-            end = sentBreak + 1;
-          }
-        }
-      }
-      const chunk = text.slice(start, end).trim();
-      if (chunk.length > 0) chunks.push(chunk);
-      if (end >= text.length) break;
-      start = end - overlapChars;
-      if (start <= (chunks.length > 0 ? end - targetChars : 0)) {
-        start = end;
-      }
-    }
-    return chunks;
-  }
-  // ── Ingest ──
-  async ingest(chunks) {
-    if (chunks.length === 0) return 0;
-    const texts = chunks.map((c) => c.text);
-    const embeddings = await this.embed(texts);
-    const records = chunks.map((chunk, i) => ({
-      text: chunk.text,
-      vector: embeddings[i],
-      role: chunk.role,
-      source_type: chunk.source_type,
-      source_id: chunk.source_id,
-      agent_id: chunk.agent_id,
-      token_count: chunk.token_count,
-      created_at: chunk.created_at || (/* @__PURE__ */ new Date()).toISOString()
-    }));
-    if (!this.chunksTable) {
-      this.chunksTable = await this.lanceDb.createTable("chunks", records);
-    } else {
-      await this.chunksTable.add(records);
-    }
-    return records.length;
-  }
-  // ── Search ──
-  async search(query, limit = 5, filter) {
-    if (!this.chunksTable) return [];
-    const [embedding] = await this.embed([query]);
-    let queryBuilder = this.chunksTable.search(embedding).distanceType("cosine").limit(limit);
-    if (filter?.agent_id) {
-      queryBuilder = queryBuilder.where(`agent_id = '${filter.agent_id}'`);
-    }
-    if (filter?.source_type) {
-      queryBuilder = queryBuilder.where(`source_type = '${filter.source_type}'`);
-    }
-    const results = await queryBuilder.toArray();
-    return results.map((row) => ({
-      text: row.text,
-      role: row.role,
-      score: row._distance != null ? 1 - row._distance : 0,
-      source_type: row.source_type,
-      source_id: row.source_id,
-      agent_id: row.agent_id,
-      created_at: row.created_at
-    }));
-  }
-  // ── Remember (explicit fact storage) ──
-  async remember(text, category = "fact") {
-    const db = this.sqliteDb;
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const stmt = db.prepare(`
-      INSERT INTO memories (text, category, confidence, source_ids, status, created_at, updated_at)
-      VALUES (?, ?, 1.0, '[]', 'active', ?, ?)
-    `);
-    const result = stmt.run(text, category, now, now);
-    await this.ingest([{
-      text,
-      role: "system",
-      source_type: "manual",
-      source_id: `memory:${result.lastInsertRowid}`,
-      agent_id: "system",
-      token_count: Math.ceil(text.length / 4),
-      created_at: now
-    }]);
-    return result.lastInsertRowid;
-  }
-  // ── Forget (deprecate a memory) ──
-  forget(memoryId) {
-    const db = this.sqliteDb;
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const result = db.prepare(`
-      UPDATE memories SET status = 'deprecated', updated_at = ? WHERE id = ? AND status = 'active'
-    `).run(now, memoryId);
-    return result.changes > 0;
-  }
-  // ── Status ──
-  async status() {
-    const db = this.sqliteDb;
-    let chunks = 0;
-    let oldestChunk = null;
-    let newestChunk = null;
-    const agents = [];
-    if (this.chunksTable) {
-      chunks = await this.chunksTable.countRows();
-      try {
-        const sample = await this.chunksTable.search([]).limit(1).toArray();
-      } catch {
-      }
-    }
-    const memories = db.prepare("SELECT COUNT(*) as count FROM memories WHERE status = ?").get("active")?.count || 0;
-    const sources = db.prepare("SELECT COUNT(*) as count FROM sources").get()?.count || 0;
-    const agentRows = db.prepare("SELECT DISTINCT agent_id FROM sources").all();
-    agents.push(...agentRows.map((r) => r.agent_id));
-    return {
-      chunks,
-      memories,
-      sources,
-      agents,
-      oldestChunk,
-      newestChunk,
-      embeddingProvider: this.config.embeddingProvider,
-      dataDir: this.config.dataDir
-    };
-  }
-  // ── Capture State (for incremental ingestion) ──
-  getCaptureState(agentId, sourceId) {
-    const db = this.sqliteDb;
-    const row = db.prepare("SELECT last_message_count, capture_count FROM capture_state WHERE agent_id = ? AND source_id = ?").get(agentId, sourceId);
-    return row || { lastMessageCount: 0, captureCount: 0 };
-  }
-  setCaptureState(agentId, sourceId, messageCount, captureCount) {
-    const db = this.sqliteDb;
-    db.prepare(`
-      INSERT OR REPLACE INTO capture_state (agent_id, source_id, last_message_count, capture_count, last_capture_at)
-      VALUES (?, ?, ?, ?, ?)
-    `).run(agentId, sourceId, messageCount, captureCount, (/* @__PURE__ */ new Date()).toISOString());
-  }
-  // ── Cleanup ──
-  close() {
-    this.sqliteDb?.close();
-  }
-};
-function resolveConfig(overrides) {
-  const openclawHome = process.env.OPENCLAW_HOME || join(process.env.HOME || "/Users/lesa", ".openclaw");
-  const dataDir = overrides?.dataDir || join(openclawHome, "memory-crystal");
-  let openaiApiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY;
-  if (!openaiApiKey) {
-    try {
-      const saTokenPath = join(openclawHome, "secrets", "op-sa-token");
-      if (existsSync(saTokenPath)) {
-        const saToken = readFileSync(saTokenPath, "utf8").trim();
-        openaiApiKey = execSync('op read "op://Agent Secrets/OpenAI API/api key"', {
-          encoding: "utf8",
-          env: { ...process.env, OP_SERVICE_ACCOUNT_TOKEN: saToken },
-          timeout: 1e4
-        }).trim();
-      }
-    } catch {
-    }
-  }
-  return {
-    dataDir,
-    embeddingProvider: overrides?.embeddingProvider || process.env.CRYSTAL_EMBEDDING_PROVIDER || "openai",
-    openaiApiKey,
-    openaiModel: overrides?.openaiModel || process.env.CRYSTAL_OPENAI_MODEL || "text-embedding-3-small",
-    ollamaHost: overrides?.ollamaHost || process.env.CRYSTAL_OLLAMA_HOST || "http://localhost:11434",
-    ollamaModel: overrides?.ollamaModel || process.env.CRYSTAL_OLLAMA_MODEL || "nomic-embed-text",
-    googleApiKey: overrides?.googleApiKey || process.env.GOOGLE_API_KEY,
-    googleModel: overrides?.googleModel || process.env.CRYSTAL_GOOGLE_MODEL || "text-embedding-004",
-    remoteUrl: overrides?.remoteUrl || process.env.CRYSTAL_REMOTE_URL,
-    remoteToken: overrides?.remoteToken || process.env.CRYSTAL_REMOTE_TOKEN
-  };
-}
-
-export {
-  Crystal,
-  resolveConfig
-};