dsh-trivium 0.2.0

package/lib/index.js

@@ -0,0 +1,388 @@
+import { BlockAssembler, createUserMessage } from "@deepseek-ai/dsh-llm";
+import { cwdOf, registerTools } from "./tools.js";
+import {
+  buildShortMapReport,
+  closeAll,
+  formatHit,
+  namedRecallHits,
+  openWorkspaceDb,
+  searchNodes,
+  setActiveCwd,
+  embedQueryVector,
+} from "./store.js";
+import { clipToTokenBudget } from "./tokens.js";
+import { clip } from "./schema.js";
+import {
+  clearPending,
+  distill,
+  loadPending,
+  savePending,
+  textFromContent,
+} from "./extract.js";
+import { readUiSettings, registerMemoryApi } from "./server.js";
+import { applyRecallMode, recallModeOf } from "./settings.js";
+import { configureEmbedding } from "./embed.js";
+
+export const name = "dsh-trivium";
+export const inject = ["tools"];
+
+const DEFAULTS = {
+  autoRecall: false,
+  anchorRecall: false,
+  recallMode: "off",
+  extractEnabled: true,
+  writeApproval: false,
+  mapTokenBudget: 400,
+  expandDepth: 1,
+  topK: 8,
+  embeddingEnabled: false,
+  embeddingUrl: "",
+  embeddingModel: "text-embedding-3-small",
+};
+
+const buffers = new Map();
+const idleTimers = new Map();
+const status = {
+  lastInjectTokens: 0,
+  lastInjectAt: null,
+  lastExtract: null,
+};
+
+function liveOptions(config) {
+  const merged = { ...DEFAULTS, ...config, ...readUiSettings() };
+  applyRecallMode(merged, recallModeOf(merged));
+  configureEmbedding(merged);
+  return merged;
+}
+
+function sessionKey(session) {
+  return String(session?.id || session?.header?.id || "");
+}
+
+function bufOf(session) {
+  const id = sessionKey(session);
+  if (!buffers.has(id)) {
+    buffers.set(id, {
+      sessionId: id,
+      cwd: session?.header?.cwd || "",
+      turns: [],
+      calls: Object.create(null),
+      dirty: false,
+    });
+  }
+  const buf = buffers.get(id);
+  if (session?.header?.cwd) buf.cwd = session.header.cwd;
+  return buf;
+}
+
+function clearIdle(id) {
+  const timer = idleTimers.get(id);
+  if (timer) clearTimeout(timer);
+  idleTimers.delete(id);
+}
+
+function resolveLlmTarget(session, agent) {
+  const header = typeof session?.requestHeader === "function" ? session.requestHeader() : null;
+  const cfg = header?.config;
+  if (cfg?.provider && cfg?.model) return { provider: cfg.provider, model: cfg.model };
+  if (agent?.options?.provider && agent?.options?.model) {
+    return { provider: agent.options.provider, model: agent.options.model };
+  }
+  return null;
+}
+
+function makeLlmCall(ctx, session, agent) {
+  if (!ctx.llm) return null;
+  const target = resolveLlmTarget(session, agent);
+  if (!target) return null;
+  return async (prompt) => {
+    const assembler = new BlockAssembler();
+    const options = {
+      provider: target.provider,
+      model: target.model,
+      maxTokens: 400,
+      sessionId: session?.id,
+      messages: [
+        createUserMessage({
+          content: [{ type: "text", text: prompt }],
+          source: { kind: "plugin", plugin: "dsh-trivium" },
+        }),
+      ],
+    };
+    for await (const chunk of ctx.llm.stream(options)) assembler.push(chunk);
+    const finish = assembler.finish;
+    if (finish?.kind === "error" || finish?.kind === "aborted") {
+      throw new Error(finish.failure?.message || finish.kind);
+    }
+    return (assembler.blocks() || [])
+      .filter((b) => b.type === "text")
+      .map((b) => b.text)
+      .join("\n");
+  };
+}
+
+async function runExtract(ctx, { cwd, turns, sessionId, session, agent, log, options, reason }) {
+  if (options.extractEnabled === false) return;
+  if (!cwd || !turns?.length) return;
+  try {
+    const db = await openWorkspaceDb(cwd);
+    const result = await distill({
+      db,
+      turns,
+      sessionId,
+          llmCall: makeLlmCall(ctx, session, agent),
+      log,
+    });
+    status.lastExtract = {
+      at: new Date().toISOString(),
+      reason,
+      applied: result.applied.length,
+      llmFailed: result.llmFailed,
+    };
+    if (result.llmFailed) {
+      savePending(cwd, { sessionId, turns, error: "llm failed" });
+    } else {
+      clearPending(cwd);
+    }
+  } catch (err) {
+    log.warn?.(`[dsh-trivium] extract skipped: ${err.message}`);
+    try {
+      savePending(cwd, { sessionId, turns, error: String(err.message || err) });
+    } catch {
+      // ignore
+    }
+  }
+}
+
+function onSessionEvent(session, event) {
+  const buf = bufOf(session);
+  switch (event.type) {
+    case "user/message": {
+      const msg = event.data;
+      if (msg?.source?.kind && msg.source.kind !== "user") break;
+      const text = clip(textFromContent(msg?.content), 500);
+      if (text) {
+        buf.turns.push({ role: "user", text, turn: event.data?.turn });
+        buf.dirty = true;
+      }
+      break;
+    }
+    case "assistant/message": {
+      const msg = event.data?.message || event.data;
+      const text = clip(textFromContent(msg?.content), 400);
+      if (text) {
+        buf.turns.push({ role: "assistant", text, turn: event.data?.turn });
+        buf.dirty = true;
+      }
+      break;
+    }
+    case "tool/call": {
+      buf.calls[event.data.callId] = event.data.name;
+      break;
+    }
+    case "tool/result": {
+      const callId = event.data?.message?.source?.callId || event.data?.callId;
+      const name = buf.calls[callId] || "tool";
+      const msg = event.data?.message || event.data;
+      const content0 = Array.isArray(msg?.content) ? msg.content[0] : null;
+      const err = event.data?.error || msg?.error;
+      const isError =
+        Boolean(err) ||
+        msg?.isError === true ||
+        content0?.isError === true ||
+        event.data?.ok === false ||
+        msg?.ok === false;
+      buf.turns.push({
+        role: "tool",
+        name,
+        ok: !isError,
+        text: clip(textFromContent(event.data?.message?.content), 200),
+        turn: event.data?.turn,
+      });
+      buf.dirty = true;
+      break;
+    }
+    case "turn/end": {
+      if (buf.dirty && buf.cwd && buf.turns.length) {
+        savePending(buf.cwd, { sessionId: buf.sessionId, turns: buf.turns });
+      }
+      break;
+    }
+    default:
+      break;
+  }
+  if (buf.turns.length > 80) buf.turns = buf.turns.slice(-80);
+}
+
+export function apply(ctx, config = {}) {
+  const log = ctx.logger?.("dsh-trivium") ?? console;
+  const optionsOf = () => liveOptions(config);
+
+  let runtime = ctx;
+  ctx.inject(["llm"], (llmCtx) => {
+    runtime = llmCtx;
+  });
+
+  registerTools(ctx, optionsOf);
+
+  ctx.on("agent/session-start", async ({ agent }) => {
+    try {
+      const cwd = cwdOf(agent);
+      setActiveCwd(cwd);
+      const db = await openWorkspaceDb(cwd);
+      const budget = optionsOf().mapTokenBudget || 400;
+      const map = buildShortMapReport(db, budget);
+      status.lastInjectTokens = map.tokens;
+      status.lastInjectAt = new Date().toISOString();
+      agent.inject(
+        createUserMessage({
+          content: [{ type: "text", text: map.text }],
+          source: {
+            kind: "plugin",
+            plugin: "dsh-trivium",
+            form: "snapshot",
+            sections: [{ name: "dsh-trivium", text: map.text }],
+          },
+        }),
+      );
+      const pending = loadPending(cwd);
+      if (pending?.turns?.length && optionsOf().extractEnabled !== false) {
+        runExtract(runtime, {
+          cwd,
+          turns: pending.turns,
+          sessionId: pending.sessionId || agent.session?.id,
+          session: agent.session,
+          agent,
+          log,
+          options: optionsOf(),
+          reason: "pending-replay",
+        }).catch((err) => log.warn?.(`[dsh-trivium] pending replay: ${err.message}`));
+      }
+    } catch (err) {
+      log.warn?.(`[dsh-trivium] session-start skipped: ${err.message}`);
+    }
+  });
+
+  ctx.on("session/event", (session, event) => {
+    try {
+      onSessionEvent(session, event);
+      if (event.type === "compaction/end" && !event.data?.error) {
+        clearIdle(sessionKey(session));
+        const buf = bufOf(session);
+        runExtract(runtime, {
+          cwd: buf.cwd || session.header?.cwd,
+          turns: buf.turns,
+          sessionId: buf.sessionId || session.id,
+          session,
+          log,
+          options: optionsOf(),
+          reason: "compaction/end",
+        }).catch((err) => log.warn?.(`[dsh-trivium] compaction extract: ${err.message}`));
+      }
+      if (event.type === "turn/end") {
+        const id = sessionKey(session);
+        clearIdle(id);
+        const timer = setTimeout(() => {
+          idleTimers.delete(id);
+          const buf = bufOf(session);
+          if (!buf.dirty || !buf.turns.length || optionsOf().extractEnabled === false) return;
+          runExtract(runtime, {
+            cwd: buf.cwd || session.header?.cwd,
+            turns: buf.turns,
+            sessionId: buf.sessionId || session.id,
+            session,
+            log,
+            options: optionsOf(),
+            reason: "idle",
+          })
+            .then(() => {
+              buf.dirty = false;
+            })
+            .catch((err) => log.warn?.(`[dsh-trivium] idle extract: ${err.message}`));
+        }, 12000);
+        idleTimers.set(id, timer);
+      }
+    } catch (err) {
+      log.warn?.(`[dsh-trivium] session/event skipped: ${err.message}`);
+    }
+  });
+
+  ctx.on("agent/pre-step", async ({ agent, messages }, next) => {
+    const decision = await next();
+    try {
+      const options = optionsOf();
+      const mode = recallModeOf(options);
+      if (mode === "off") return decision;
+      if (!decision || decision.kind !== "enter") return decision;
+      const proposed = [...(decision.messages || []), ...(messages || [])];
+      const lastUser = [...proposed].reverse().find((m) => m?.source?.kind === "user");
+      if (!lastUser) return decision;
+      const query = textFromContent(lastUser.content).trim();
+      if (!query) return decision;
+      const db = await openWorkspaceDb(cwdOf(agent));
+      let hits;
+      if (mode === "anchor") {
+        hits = namedRecallHits(db, query, { topK: 8 }).slice(0, 8);
+      } else {
+        const queryVector = await embedQueryVector(query);
+        hits = searchNodes(db, query, {
+          topK: 3,
+          expandDepth: 1,
+          queryVector: queryVector || undefined,
+        }).slice(0, 3);
+      }
+      if (!hits.length) return decision;
+      const lines = hits.map((hit) => {
+        const row = formatHit(db, hit);
+        return `- [${row.type}#${row.id}] ${row.l0} (${row.path.join(" | ") || "no path"})`;
+      });
+      const form = mode === "anchor" ? "anchorRecall" : "autoRecall";
+      const text = clipToTokenBudget(
+        `dsh-trivium ${form} (L0 only; use ctx_read for full text):\n${lines.join("\n")}`,
+        300,
+      );
+      return {
+        kind: "enter",
+        messages: [
+          createUserMessage({
+            content: [{ type: "text", text }],
+            source: {
+              kind: "plugin",
+              plugin: "dsh-trivium",
+              form: "recall",
+            },
+          }),
+          ...decision.messages,
+        ],
+      };
+    } catch (err) {
+      log.warn?.(`[dsh-trivium] pre-step recall skipped: ${err.message}`);
+      return decision;
+    }
+  });
+
+  ctx.on("session/disposed", (session) => {
+    try {
+      clearIdle(sessionKey(session));
+      const buf = bufOf(session);
+      if (buf.dirty && buf.cwd && buf.turns.length) {
+        savePending(buf.cwd, { sessionId: buf.sessionId, turns: buf.turns });
+      }
+      buffers.delete(sessionKey(session));
+    } catch (err) {
+      log.warn?.(`[dsh-trivium] dispose skipped: ${err.message}`);
+    }
+  });
+
+  ctx.inject(["webServer"], (webCtx) => {
+    try {
+      registerMemoryApi(webCtx, { liveOptions: optionsOf, status });
+    } catch (err) {
+      log.warn?.(`[dsh-trivium] settings api skipped: ${err.message}`);
+    }
+  });
+
+  ctx.effect(() => () => {
+    closeAll();
+  });
+}

package/lib/schema.js

@@ -0,0 +1,66 @@
+/** Node and edge conventions. Keep this the single source of payload shape. */
+
+export const DIM = 1536;
+
+export const NODE_TYPES = Object.freeze([
+  "workspace",
+  "entity",
+  "preference",
+  "decision",
+  "experience",
+]);
+
+export const EDGE_LABELS = Object.freeze({
+  inWorkspace: "in_workspace",
+  about: "about",
+  decided: "decided",
+  broke: "broke",
+  fixed: "fixed",
+  sameAs: "same_as",
+  fromSession: "from_session",
+});
+
+export const BUSINESS_EDGES = Object.freeze(["about", "decided", "broke", "fixed"]);
+
+/** Weights written on link(). Engine expand currently ignores labels; we still persist them. */
+export const EDGE_WEIGHTS = Object.freeze({
+  about: 1,
+  decided: 1,
+  broke: 0.9,
+  fixed: 0.9,
+  in_workspace: 0.15,
+  same_as: 0.5,
+  from_session: 0.1,
+});
+
+export function weightOf(label) {
+  return EDGE_WEIGHTS[label] ?? 1;
+}
+
+export const SECRET_RE =
+  /(api[_-]?key|secret|password|token|bearer\s+[a-z0-9]|sk-[a-z0-9])/i;
+
+export function zeroVector() {
+  return new Array(DIM).fill(0);
+}
+
+export function nowIso() {
+  return new Date().toISOString();
+}
+
+export function clip(text, max = 200) {
+  const s = String(text || "").replace(/\s+/g, " ").trim();
+  if (s.length <= max) return s;
+  return `${s.slice(0, max - 1)}…`;
+}
+
+export function looksSecret(text) {
+  return SECRET_RE.test(String(text || ""));
+}
+
+export function indexTextFor(payload) {
+  const parts = [payload.type, payload.name, payload.text, payload.uri, payload.fail, payload.fix, payload.until]
+    .filter(Boolean)
+    .map(String);
+  return parts.join(" ");
+}

package/lib/server.js

@@ -0,0 +1,222 @@
+import { existsSync } from "node:fs";
+import {
+  archiveNode,
+  backfillEmbeddings,
+  dbPathFor,
+  deleteNode,
+  exportGraph,
+  getActiveCwd,
+  importGraph,
+  listNodes,
+  listOpenWorkspaces,
+  mergeNodes,
+  nodeCountOf,
+  openWorkspaceDb,
+  updateNode,
+} from "./store.js";
+import { embeddingPublicStatus } from "./embed.js";
+import { recallModeOf, writeUiSettings } from "./settings.js";
+
+export { readUiSettings, writeUiSettings } from "./settings.js";
+
+function isLoopback(req) {
+  const ra = req.socket && req.socket.remoteAddress;
+  return ra === "127.0.0.1" || ra === "::1" || ra === "::ffff:127.0.0.1";
+}
+
+function sendJson(res, status, body) {
+  const data = Buffer.from(JSON.stringify(body), "utf8");
+  res.writeHead(status, {
+    "content-type": "application/json; charset=utf-8",
+    "cache-control": "no-store",
+    "content-length": String(data.length),
+  });
+  res.end(data);
+}
+
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    req.on("data", (c) => chunks.push(c));
+    req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
+    req.on("error", reject);
+  });
+}
+
+async function dbForRequest(url) {
+  const cwd = url.searchParams.get("cwd") || getActiveCwd();
+  if (!cwd) return { cwd: null, db: null };
+  return { cwd, db: await openWorkspaceDb(cwd) };
+}
+
+export function registerMemoryApi(ctx, { liveOptions, status }) {
+  const dispose = ctx.webServer.register({
+    kind: "prefix",
+    path: "/api/dsh-trivium",
+    handler: async (req, res) => {
+      if (!isLoopback(req)) {
+        res.writeHead(403);
+        res.end("forbidden");
+        return;
+      }
+      try {
+        await handle(req, res, { liveOptions, status });
+      } catch (err) {
+        sendJson(res, 500, { ok: false, message: String(err.message || err) });
+      }
+    },
+  });
+  return dispose;
+}
+
+async function handle(req, res, { liveOptions, status }) {
+  const url = new URL(req.url || "/", "http://127.0.0.1");
+  const path = url.pathname.replace(/\/$/, "") || "/";
+  const method = req.method || "GET";
+
+  if (path === "/api/dsh-trivium/status" && method === "GET") {
+    const opts = liveOptions();
+    const cwd = url.searchParams.get("cwd") || getActiveCwd();
+    let nodeCount = 0;
+    let dbPath = cwd ? dbPathFor(cwd) : "";
+    if (cwd) {
+      const db = await openWorkspaceDb(cwd);
+      nodeCount = nodeCountOf(db);
+      dbPath = dbPathFor(cwd);
+    }
+    const mode = recallModeOf(opts);
+    sendJson(res, 200, {
+      ok: true,
+      cwd: cwd || "",
+      dbPath,
+      dbExists: dbPath ? existsSync(dbPath) : false,
+      nodeCount,
+      lastInjectTokens: status.lastInjectTokens || 0,
+      lastInjectAt: status.lastInjectAt || null,
+      lastExtract: status.lastExtract || null,
+      recallMode: mode,
+      autoRecall: mode === "auto",
+      anchorRecall: mode === "anchor",
+      extractEnabled: opts.extractEnabled !== false,
+      writeApproval: !!opts.writeApproval,
+      mapTokenBudget: opts.mapTokenBudget || 400,
+      workspaces: listOpenWorkspaces(),
+      ...embeddingPublicStatus(),
+    });
+    return;
+  }
+
+  if (path === "/api/dsh-trivium/settings" && method === "POST") {
+    const body = JSON.parse((await readBody(req)) || "{}");
+    const saved = writeUiSettings(body);
+    const opts = liveOptions();
+    sendJson(res, 200, {
+      ok: true,
+      ...opts,
+      recallMode: recallModeOf(opts),
+      ...embeddingPublicStatus(),
+      embeddingApiKey: undefined,
+    });
+    return;
+  }
+
+  if (path === "/api/dsh-trivium/export" && method === "GET") {
+    const { cwd, db } = await dbForRequest(url);
+    if (!db) {
+      sendJson(res, 400, { ok: false, message: "no workspace" });
+      return;
+    }
+    sendJson(res, 200, { ok: true, cwd, graph: exportGraph(db, cwd) });
+    return;
+  }
+
+  if (path === "/api/dsh-trivium/import" && method === "POST") {
+    const { db } = await dbForRequest(url);
+    if (!db) {
+      sendJson(res, 400, { ok: false, message: "还没有打开工作区" });
+      return;
+    }
+    const body = JSON.parse((await readBody(req)) || "{}");
+    const result = importGraph(db, body.graph || body);
+    sendJson(res, result.ok ? 200 : 400, result);
+    return;
+  }
+
+  if (path === "/api/dsh-trivium/embed-backfill" && method === "POST") {
+    const { db } = await dbForRequest(url);
+    if (!db) {
+      sendJson(res, 400, { ok: false, message: "no workspace" });
+      return;
+    }
+    liveOptions();
+    const result = await backfillEmbeddings(db);
+    sendJson(res, result.ok ? 200 : 400, result);
+    return;
+  }
+
+  if (path === "/api/dsh-trivium/nodes" && method === "GET") {
+    const { cwd, db } = await dbForRequest(url);
+    if (!db) {
+      sendJson(res, 200, { ok: true, cwd: "", nodes: [] });
+      return;
+    }
+    const type = url.searchParams.get("type") || "";
+    const q = url.searchParams.get("q") || "";
+    const includeArchived = url.searchParams.get("archived") === "1";
+    const includeStale = url.searchParams.get("stale") === "1";
+    const about = url.searchParams.get("about") || "";
+    sendJson(res, 200, {
+      ok: true,
+      cwd,
+      nodes: listNodes(db, {
+        type: type || undefined,
+        q: q || undefined,
+        includeArchived,
+        includeStale,
+        aboutId: about || undefined,
+      }),
+    });
+    return;
+  }
+
+  const mergeMatch = path.match(/^\/api\/dsh-trivium\/nodes\/(\d+)\/merge$/);
+  if (mergeMatch && method === "POST") {
+    const { db } = await dbForRequest(url);
+    if (!db) {
+      sendJson(res, 400, { ok: false, message: "no workspace" });
+      return;
+    }
+    const body = JSON.parse((await readBody(req)) || "{}");
+    const result = mergeNodes(db, mergeMatch[1], body.dropId);
+    sendJson(res, result.ok ? 200 : 400, result);
+    return;
+  }
+
+  const patchMatch = path.match(/^\/api\/dsh-trivium\/nodes\/(\d+)$/);
+  if (patchMatch && method === "POST") {
+    const { db } = await dbForRequest(url);
+    if (!db) {
+      sendJson(res, 400, { ok: false, message: "no workspace" });
+      return;
+    }
+    const body = JSON.parse((await readBody(req)) || "{}");
+    const payload = updateNode(db, patchMatch[1], body);
+    sendJson(res, payload ? 200 : 404, { ok: Boolean(payload), id: Number(patchMatch[1]), payload });
+    return;
+  }
+
+  const nodeMatch = path.match(/^\/api\/dsh-trivium\/nodes\/(\d+)\/(archive|delete)$/);
+  if (nodeMatch && method === "POST") {
+    const { db } = await dbForRequest(url);
+    if (!db) {
+      sendJson(res, 400, { ok: false, message: "no workspace" });
+      return;
+    }
+    const id = Number(nodeMatch[1]);
+    const ok = nodeMatch[2] === "archive" ? archiveNode(db, id) : deleteNode(db, id);
+    sendJson(res, ok ? 200 : 404, { ok, id });
+    return;
+  }
+
+  sendJson(res, 404, { ok: false, message: "not found" });
+}