memory-crystal 0.2.0

package/dist/chunk-NIJCVN3O.js

@@ -0,0 +1,147 @@
+// src/dev-update.ts
+import { execSync } from "child_process";
+import { existsSync, mkdirSync, writeFileSync, readFileSync } from "fs";
+import { join, basename } from "path";
+var HOME = process.env.HOME || "/Users/lesa";
+var STAFF_DIR = join(HOME, "Documents", "wipcomputer--mac-mini-01", "staff");
+var CC_REPOS = join(STAFF_DIR, "Parker", "Claude Code - Mini", "repos");
+var LESA_REPOS = join(STAFF_DIR, "L\u0113sa", "repos");
+var DEV_UPDATES_DIR = join(CC_REPOS, "wip-dev-updates");
+var LAST_RUN_PATH = join(HOME, ".openclaw", "memory", "dev-update-last-run.json");
+function loadLastRun() {
+  try {
+    if (existsSync(LAST_RUN_PATH)) {
+      return JSON.parse(readFileSync(LAST_RUN_PATH, "utf-8"));
+    }
+  } catch {
+  }
+  return null;
+}
+function saveLastRun(run) {
+  const dir = join(HOME, ".openclaw", "memory");
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  writeFileSync(LAST_RUN_PATH, JSON.stringify(run, null, 2));
+}
+function git(repoPath, cmd) {
+  try {
+    return execSync(`git -C "${repoPath}" ${cmd}`, {
+      encoding: "utf-8",
+      timeout: 1e4,
+      stdio: ["pipe", "pipe", "pipe"]
+    }).trim();
+  } catch {
+    return "";
+  }
+}
+function scanRepo(repoPath, since) {
+  if (!existsSync(join(repoPath, ".git"))) return null;
+  const name = basename(repoPath);
+  if (name === "_third-party-repos" || name === "wip-dev-updates") return null;
+  const recentCommits = git(repoPath, `log --oneline --since="${since}"`);
+  const uncommitted = git(repoPath, "status --porcelain");
+  if (!recentCommits && !uncommitted) return null;
+  const lines = [];
+  lines.push(`# ${name}`);
+  lines.push("");
+  if (recentCommits) {
+    lines.push("## Recent Commits");
+    lines.push("");
+    lines.push("```");
+    lines.push(...recentCommits.split("\n").slice(0, 10));
+    lines.push("```");
+    lines.push("");
+  }
+  if (uncommitted) {
+    lines.push("## Uncommitted Changes");
+    lines.push("");
+    lines.push("```");
+    lines.push(...uncommitted.split("\n").slice(0, 20));
+    lines.push("```");
+    lines.push("");
+  }
+  if (recentCommits) {
+    const diffStat = git(repoPath, `diff --stat "HEAD@{${since}}" HEAD`);
+    if (diffStat) {
+      lines.push("## Files Changed");
+      lines.push("");
+      lines.push("```");
+      lines.push(...diffStat.split("\n").slice(-15));
+      lines.push("```");
+      lines.push("");
+    }
+  }
+  const branch = git(repoPath, "branch --show-current") || "unknown";
+  lines.push(`**Branch:** ${branch}`);
+  lines.push("");
+  return lines.join("\n");
+}
+function runDevUpdate(author) {
+  const lastRun = loadLastRun();
+  if (lastRun) {
+    const elapsed = Date.now() - new Date(lastRun.timestamp).getTime();
+    if (elapsed < 60 * 60 * 1e3) {
+      return { reposUpdated: 0, files: [] };
+    }
+  }
+  let since = "6 hours ago";
+  if (lastRun?.timestamp) {
+    const lastDate = new Date(lastRun.timestamp);
+    const hoursAgo = Math.ceil((Date.now() - lastDate.getTime()) / (1e3 * 60 * 60));
+    since = `${Math.max(hoursAgo, 1)} hours ago`;
+  }
+  const now = /* @__PURE__ */ new Date();
+  const ts = [
+    String(now.getMonth() + 1).padStart(2, "0"),
+    String(now.getDate()).padStart(2, "0"),
+    String(now.getFullYear())
+  ].join("-") + "--" + [
+    String(now.getHours()).padStart(2, "0"),
+    String(now.getMinutes()).padStart(2, "0"),
+    String(now.getSeconds()).padStart(2, "0")
+  ].join("-");
+  const files = [];
+  const repoDirs = [CC_REPOS, LESA_REPOS];
+  for (const parentDir of repoDirs) {
+    if (!existsSync(parentDir)) continue;
+    let entries;
+    try {
+      entries = execSync(`ls "${parentDir}"`, { encoding: "utf-8" }).trim().split("\n");
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      const repoPath = join(parentDir, entry);
+      const content = scanRepo(repoPath, since);
+      if (!content) continue;
+      const repoName = basename(repoPath);
+      const outDir = join(DEV_UPDATES_DIR, repoName);
+      const outFile = join(outDir, `${author}-dev-update-${ts}.md`);
+      mkdirSync(outDir, { recursive: true });
+      const header = `*Auto-generated dev update by ${author} at ${now.toISOString().slice(0, 16).replace("T", " ")}*
+
+`;
+      writeFileSync(outFile, content.replace(/^# .+\n/, `$&
+${header}`));
+      files.push(`${repoName}/${author}-dev-update-${ts}.md`);
+    }
+  }
+  if (files.length > 0 && existsSync(join(DEV_UPDATES_DIR, ".git"))) {
+    try {
+      execSync(
+        `cd "${DEV_UPDATES_DIR}" && git add -A && git commit -m "${author} auto-dev-update ${ts}: ${files.length} repo(s)" --no-verify && git push --quiet`,
+        { encoding: "utf-8", timeout: 3e4, stdio: "pipe" }
+      );
+    } catch {
+    }
+  }
+  saveLastRun({
+    timestamp: now.toISOString(),
+    author,
+    reposUpdated: files.length
+  });
+  return { reposUpdated: files.length, files };
+}
+
+export {
+  runDevUpdate
+};

package/dist/chunk-O2UITJGH.js

@@ -0,0 +1,465 @@
+// src/core.ts
+import * as lancedb from "@lancedb/lancedb";
+import Database from "better-sqlite3";
+import { readFileSync, existsSync, mkdirSync } from "fs";
+import { execSync } from "child_process";
+import { join } from "path";
+import http from "http";
+import https from "https";
+async function embedOpenAI(texts, apiKey, model) {
+  return new Promise((resolve, reject) => {
+    const body = JSON.stringify({ input: texts, model });
+    const req = https.request({
+      hostname: "api.openai.com",
+      path: "/v1/embeddings",
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`,
+        "Content-Length": Buffer.byteLength(body)
+      },
+      timeout: 3e4
+    }, (res) => {
+      let data = "";
+      res.on("data", (chunk) => data += chunk);
+      res.on("end", () => {
+        if (res.statusCode !== 200) {
+          reject(new Error(`OpenAI API error ${res.statusCode}: ${data.slice(0, 200)}`));
+          return;
+        }
+        const parsed = JSON.parse(data);
+        resolve(parsed.data.map((d) => d.embedding));
+      });
+    });
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("OpenAI timeout"));
+    });
+    req.write(body);
+    req.end();
+  });
+}
+async function embedOllama(texts, host, model) {
+  const results = [];
+  for (const text of texts) {
+    const result = await new Promise((resolve, reject) => {
+      const url = new URL("/api/embeddings", host);
+      const body = JSON.stringify({ model, prompt: text });
+      const req = http.request({
+        hostname: url.hostname,
+        port: url.port,
+        path: url.pathname,
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Content-Length": Buffer.byteLength(body)
+        },
+        timeout: 15e3
+      }, (res) => {
+        let data = "";
+        res.on("data", (chunk) => data += chunk);
+        res.on("end", () => {
+          if (res.statusCode !== 200) {
+            reject(new Error(`Ollama error ${res.statusCode}: ${data.slice(0, 200)}`));
+            return;
+          }
+          resolve(JSON.parse(data).embedding);
+        });
+      });
+      req.on("error", reject);
+      req.on("timeout", () => {
+        req.destroy();
+        reject(new Error("Ollama timeout"));
+      });
+      req.write(body);
+      req.end();
+    });
+    results.push(result);
+  }
+  return results;
+}
+async function embedGoogle(texts, apiKey, model) {
+  return new Promise((resolve, reject) => {
+    const body = JSON.stringify({
+      requests: texts.map((text) => ({ model: `models/${model}`, content: { parts: [{ text }] } }))
+    });
+    const req = https.request({
+      hostname: "generativelanguage.googleapis.com",
+      path: `/v1beta/models/${model}:batchEmbedContents?key=${apiKey}`,
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Content-Length": Buffer.byteLength(body)
+      },
+      timeout: 3e4
+    }, (res) => {
+      let data = "";
+      res.on("data", (chunk) => data += chunk);
+      res.on("end", () => {
+        if (res.statusCode !== 200) {
+          reject(new Error(`Google API error ${res.statusCode}: ${data.slice(0, 200)}`));
+          return;
+        }
+        const parsed = JSON.parse(data);
+        resolve(parsed.embeddings.map((e) => e.values));
+      });
+    });
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("Google timeout"));
+    });
+    req.write(body);
+    req.end();
+  });
+}
+var Crystal = class {
+  config;
+  lanceDb = null;
+  sqliteDb = null;
+  chunksTable = null;
+  constructor(config) {
+    this.config = config;
+    if (!existsSync(config.dataDir)) {
+      mkdirSync(config.dataDir, { recursive: true });
+    }
+  }
+  // ── Initialization ──
+  async init() {
+    const lanceDir = join(this.config.dataDir, "lance");
+    const sqlitePath = join(this.config.dataDir, "crystal.db");
+    if (!existsSync(lanceDir)) mkdirSync(lanceDir, { recursive: true });
+    this.lanceDb = await lancedb.connect(lanceDir);
+    this.sqliteDb = new Database(sqlitePath);
+    this.sqliteDb.pragma("journal_mode = WAL");
+    this.initSqliteTables();
+    await this.initLanceTables();
+  }
+  initSqliteTables() {
+    const db = this.sqliteDb;
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS sources (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        type TEXT NOT NULL,
+        uri TEXT NOT NULL,
+        title TEXT,
+        agent_id TEXT NOT NULL,
+        metadata TEXT DEFAULT '{}',
+        ingested_at TEXT NOT NULL,
+        chunk_count INTEGER DEFAULT 0
+      );
+
+      CREATE TABLE IF NOT EXISTS capture_state (
+        agent_id TEXT NOT NULL,
+        source_id TEXT NOT NULL,
+        last_message_count INTEGER DEFAULT 0,
+        capture_count INTEGER DEFAULT 0,
+        last_capture_at TEXT,
+        PRIMARY KEY (agent_id, source_id)
+      );
+
+      CREATE TABLE IF NOT EXISTS memories (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        text TEXT NOT NULL,
+        category TEXT NOT NULL DEFAULT 'fact',
+        confidence REAL NOT NULL DEFAULT 1.0,
+        source_ids TEXT DEFAULT '[]',
+        status TEXT NOT NULL DEFAULT 'active',
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS entities (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT NOT NULL UNIQUE,
+        type TEXT NOT NULL DEFAULT 'concept',
+        description TEXT,
+        properties TEXT DEFAULT '{}',
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS relationships (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        source_id INTEGER NOT NULL REFERENCES entities(id),
+        target_id INTEGER NOT NULL REFERENCES entities(id),
+        type TEXT NOT NULL,
+        description TEXT,
+        weight REAL DEFAULT 1.0,
+        valid_from TEXT NOT NULL,
+        valid_until TEXT,
+        created_at TEXT NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_sources_agent ON sources(agent_id);
+      CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
+      CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
+      CREATE INDEX IF NOT EXISTS idx_relationships_source ON relationships(source_id);
+      CREATE INDEX IF NOT EXISTS idx_relationships_target ON relationships(target_id);
+    `);
+  }
+  async initLanceTables() {
+    const db = this.lanceDb;
+    const tableNames = await db.tableNames();
+    if (tableNames.includes("chunks")) {
+      this.chunksTable = await db.openTable("chunks");
+    }
+  }
+  // ── Embedding ──
+  async embed(texts) {
+    if (texts.length === 0) return [];
+    const cfg = this.config;
+    switch (cfg.embeddingProvider) {
+      case "openai":
+        if (!cfg.openaiApiKey) throw new Error("OpenAI API key required");
+        return embedOpenAI(texts, cfg.openaiApiKey, cfg.openaiModel || "text-embedding-3-small");
+      case "ollama":
+        return embedOllama(texts, cfg.ollamaHost || "http://localhost:11434", cfg.ollamaModel || "nomic-embed-text");
+      case "google":
+        if (!cfg.googleApiKey) throw new Error("Google API key required");
+        return embedGoogle(texts, cfg.googleApiKey, cfg.googleModel || "text-embedding-004");
+      default:
+        throw new Error(`Unknown embedding provider: ${cfg.embeddingProvider}`);
+    }
+  }
+  // ── Chunking ──
+  chunkText(text, targetTokens = 400, overlapTokens = 80) {
+    const targetChars = targetTokens * 4;
+    const overlapChars = overlapTokens * 4;
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+      let end = Math.min(start + targetChars, text.length);
+      if (end < text.length) {
+        const minBreak = start + Math.floor(targetChars * 0.5);
+        const paraBreak = text.lastIndexOf("\n\n", end);
+        if (paraBreak > minBreak) {
+          end = paraBreak;
+        } else {
+          const sentBreak = text.lastIndexOf(". ", end);
+          if (sentBreak > minBreak) {
+            end = sentBreak + 1;
+          }
+        }
+      }
+      const chunk = text.slice(start, end).trim();
+      if (chunk.length > 0) chunks.push(chunk);
+      if (end >= text.length) break;
+      start = end - overlapChars;
+      if (start <= (chunks.length > 0 ? end - targetChars : 0)) {
+        start = end;
+      }
+    }
+    return chunks;
+  }
+  // ── Ingest ──
+  async ingest(chunks) {
+    if (chunks.length === 0) return 0;
+    const texts = chunks.map((c) => c.text);
+    const embeddings = await this.embed(texts);
+    const records = chunks.map((chunk, i) => ({
+      text: chunk.text,
+      vector: embeddings[i],
+      role: chunk.role,
+      source_type: chunk.source_type,
+      source_id: chunk.source_id,
+      agent_id: chunk.agent_id,
+      token_count: chunk.token_count,
+      created_at: chunk.created_at || (/* @__PURE__ */ new Date()).toISOString()
+    }));
+    if (!this.chunksTable) {
+      this.chunksTable = await this.lanceDb.createTable("chunks", records);
+    } else {
+      await this.chunksTable.add(records);
+    }
+    return records.length;
+  }
+  // ── Recency helpers ──
+  recencyWeight(ageDays) {
+    return Math.max(0.5, 1 - ageDays * 0.01);
+  }
+  freshnessLabel(ageDays) {
+    if (ageDays < 3) return "fresh";
+    if (ageDays < 7) return "recent";
+    if (ageDays < 14) return "aging";
+    return "stale";
+  }
+  // ── Search ──
+  async search(query, limit = 5, filter) {
+    if (!this.chunksTable) return [];
+    const [embedding] = await this.embed([query]);
+    const fetchLimit = Math.max(limit * 3, 30);
+    let queryBuilder = this.chunksTable.vectorSearch(embedding).distanceType("cosine").limit(fetchLimit);
+    if (filter?.agent_id) {
+      queryBuilder = queryBuilder.where(`agent_id = '${filter.agent_id}'`);
+    }
+    if (filter?.source_type) {
+      queryBuilder = queryBuilder.where(`source_type = '${filter.source_type}'`);
+    }
+    const results = await queryBuilder.toArray();
+    const now = Date.now();
+    return results.map((row) => {
+      const cosine = row._distance != null ? 1 - row._distance : 0;
+      const createdAt = row.created_at || "";
+      const ageDays = createdAt ? (now - new Date(createdAt).getTime()) / (1e3 * 60 * 60 * 24) : 0;
+      const weight = createdAt ? this.recencyWeight(ageDays) : 1;
+      return {
+        text: row.text,
+        role: row.role,
+        score: cosine * weight,
+        source_type: row.source_type,
+        source_id: row.source_id,
+        agent_id: row.agent_id,
+        created_at: createdAt,
+        freshness: createdAt ? this.freshnessLabel(ageDays) : void 0
+      };
+    }).sort((a, b) => b.score - a.score).slice(0, limit);
+  }
+  // ── Remember (explicit fact storage) ──
+  async remember(text, category = "fact") {
+    const db = this.sqliteDb;
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const stmt = db.prepare(`
+      INSERT INTO memories (text, category, confidence, source_ids, status, created_at, updated_at)
+      VALUES (?, ?, 1.0, '[]', 'active', ?, ?)
+    `);
+    const result = stmt.run(text, category, now, now);
+    await this.ingest([{
+      text,
+      role: "system",
+      source_type: "manual",
+      source_id: `memory:${result.lastInsertRowid}`,
+      agent_id: "system",
+      token_count: Math.ceil(text.length / 4),
+      created_at: now
+    }]);
+    return result.lastInsertRowid;
+  }
+  // ── Forget (deprecate a memory) ──
+  forget(memoryId) {
+    const db = this.sqliteDb;
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const result = db.prepare(`
+      UPDATE memories SET status = 'deprecated', updated_at = ? WHERE id = ? AND status = 'active'
+    `).run(now, memoryId);
+    return result.changes > 0;
+  }
+  // ── Status ──
+  async status() {
+    const db = this.sqliteDb;
+    let chunks = 0;
+    let oldestChunk = null;
+    let newestChunk = null;
+    const agents = [];
+    if (this.chunksTable) {
+      chunks = await this.chunksTable.countRows();
+      try {
+        const sample = await this.chunksTable.search([]).limit(1).toArray();
+      } catch {
+      }
+    }
+    const memories = db.prepare("SELECT COUNT(*) as count FROM memories WHERE status = ?").get("active")?.count || 0;
+    const sources = db.prepare("SELECT COUNT(*) as count FROM sources").get()?.count || 0;
+    const sourceAgentRows = db.prepare("SELECT DISTINCT agent_id FROM sources").all();
+    const captureAgentRows = db.prepare("SELECT DISTINCT agent_id FROM capture_state").all();
+    const allAgents = [.../* @__PURE__ */ new Set([
+      ...sourceAgentRows.map((r) => r.agent_id),
+      ...captureAgentRows.map((r) => r.agent_id)
+    ])];
+    agents.push(...allAgents);
+    const captureInfo = db.prepare(
+      "SELECT COUNT(*) as count, MAX(last_capture_at) as latest FROM capture_state"
+    ).get();
+    return {
+      chunks,
+      memories,
+      sources,
+      agents,
+      oldestChunk,
+      newestChunk,
+      embeddingProvider: this.config.embeddingProvider,
+      dataDir: this.config.dataDir,
+      capturedSessions: captureInfo?.count || 0,
+      latestCapture: captureInfo?.latest || null
+    };
+  }
+  // ── Capture State (for incremental ingestion) ──
+  getCaptureState(agentId, sourceId) {
+    const db = this.sqliteDb;
+    const row = db.prepare("SELECT last_message_count, capture_count FROM capture_state WHERE agent_id = ? AND source_id = ?").get(agentId, sourceId);
+    if (!row) return { lastMessageCount: 0, captureCount: 0 };
+    return {
+      lastMessageCount: row.last_message_count,
+      captureCount: row.capture_count
+    };
+  }
+  setCaptureState(agentId, sourceId, messageCount, captureCount) {
+    const db = this.sqliteDb;
+    db.prepare(`
+      INSERT OR REPLACE INTO capture_state (agent_id, source_id, last_message_count, capture_count, last_capture_at)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(agentId, sourceId, messageCount, captureCount, (/* @__PURE__ */ new Date()).toISOString());
+  }
+  // ── Cleanup ──
+  close() {
+    this.sqliteDb?.close();
+  }
+};
+function resolveConfig(overrides) {
+  const openclawHome = process.env.OPENCLAW_HOME || join(process.env.HOME || "/Users/lesa", ".openclaw");
+  const dataDir = overrides?.dataDir || join(openclawHome, "memory-crystal");
+  loadEnvFile(join(dataDir, ".env"));
+  const openaiApiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY || opRead(openclawHome, "OpenAI API", "api key");
+  const googleApiKey = overrides?.googleApiKey || process.env.GOOGLE_API_KEY || opRead(openclawHome, "Google AI", "api key");
+  const remoteToken = overrides?.remoteToken || process.env.CRYSTAL_REMOTE_TOKEN || opRead(openclawHome, "Memory Crystal Remote", "token");
+  return {
+    dataDir,
+    embeddingProvider: overrides?.embeddingProvider || process.env.CRYSTAL_EMBEDDING_PROVIDER || "openai",
+    openaiApiKey,
+    openaiModel: overrides?.openaiModel || process.env.CRYSTAL_OPENAI_MODEL || "text-embedding-3-small",
+    ollamaHost: overrides?.ollamaHost || process.env.CRYSTAL_OLLAMA_HOST || "http://localhost:11434",
+    ollamaModel: overrides?.ollamaModel || process.env.CRYSTAL_OLLAMA_MODEL || "nomic-embed-text",
+    googleApiKey,
+    googleModel: overrides?.googleModel || process.env.CRYSTAL_GOOGLE_MODEL || "text-embedding-004",
+    remoteUrl: overrides?.remoteUrl || process.env.CRYSTAL_REMOTE_URL,
+    remoteToken
+  };
+}
+function loadEnvFile(path) {
+  if (!existsSync(path)) return;
+  const content = readFileSync(path, "utf8");
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eqIdx = trimmed.indexOf("=");
+    if (eqIdx === -1) continue;
+    const key = trimmed.slice(0, eqIdx).trim();
+    let value = trimmed.slice(eqIdx + 1).trim();
+    if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
+      value = value.slice(1, -1);
+    }
+    if (key && !process.env[key]) {
+      process.env[key] = value;
+    }
+  }
+}
+function opRead(openclawHome, item, field) {
+  try {
+    const saTokenPath = join(openclawHome, "secrets", "op-sa-token");
+    if (!existsSync(saTokenPath)) return void 0;
+    const saToken = readFileSync(saTokenPath, "utf8").trim();
+    return execSync(`op read "op://Agent Secrets/${item}/${field}" 2>/dev/null`, {
+      encoding: "utf8",
+      env: { ...process.env, OP_SERVICE_ACCOUNT_TOKEN: saToken },
+      timeout: 1e4
+    }).trim() || void 0;
+  } catch {
+    return void 0;
+  }
+}
+
+export {
+  Crystal,
+  resolveConfig
+};