nodedex 0.1.0

package/adapters/nodedex-capture-core.mjs

@@ -0,0 +1,129 @@
+// nodedex-capture-core.mjs — shared, host-agnostic capture core.
+//
+// WHY THIS EXISTS
+//   Every host adapter (a Claude Code hook, the Hermes hook, the tee, the next host…) needs
+//   the SAME two things, and only the same two things:
+//     1. find the live Nodedex server — its url + token — WITHOUT the user hardcoding a port,
+//     2. POST a finished turn to /api/reflect/trigger, fire-and-forget, never breaking the agent.
+//   That's this file. A host adapter is then a thin SHIM that knows how ITS host hands over a
+//   turn, parses that, and calls captureTurn() (or resolveNodedexTarget + buildTriggerBody +
+//   postTrigger). Building one host gives every future host this core for free.
+//
+// SELF-LOCATING (the reason the adapter never contains a port)
+//   The server's port can change — switch DB in the TUI and it relaunches on a new port. So we
+//   read the LIVE port + token from ~/.nodedex/tui-session.json, which the TUI rewrites on every
+//   launch / stop / DB-switch. The adapter follows the running server automatically; the user
+//   never types or re-edits a port. An explicit NODEDEX_CAPTURE_URL / NODEDEX_CAPTURE_TOKEN
+//   override wins when you want to pin a specific server (e.g. several running at once).
+//
+// Dependency-free: runs on Node 18+ (global fetch), Bun, or Deno.
+
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+const TRIGGER_MIN_CHARS = 10; // mirror the server's combined-content floor (/api/reflect/trigger)
+const CAPS = { response: 16000, reasoning: 8000, user: 2000 };
+
+/**
+ * Resolve the live Nodedex server to capture into → { url, token } or null.
+ * Order: explicit env override → the FOCUSED server in tui-session.json → first managed server.
+ * Returns null (caller skips silently) when no server is known.
+ */
+export function resolveNodedexTarget() {
+  // 1. Explicit override — pin a specific server.
+  const envUrl = process.env.NODEDEX_CAPTURE_URL;
+  if (envUrl && envUrl.trim()) {
+    return {
+      url: envUrl.trim().replace(/\/+$/, ""),
+      token: process.env.NODEDEX_CAPTURE_TOKEN || process.env.NODEDEX_TOKEN || "",
+    };
+  }
+
+  // 2. Read the TUI's own record of running servers.
+  let session;
+  try {
+    session = JSON.parse(readFileSync(join(homedir(), ".nodedex", "tui-session.json"), "utf8"));
+  } catch {
+    return null; // no session file → no server known
+  }
+  const managed = Array.isArray(session?.managed) ? session.managed : [];
+  if (managed.length === 0) return null;
+
+  // Prefer the server the user is FOCUSED on (matched to its managed entry for the token).
+  let entry = null;
+  const focusedUrl = session?.connected?.url;
+  if (focusedUrl) {
+    let port = null;
+    try { port = Number(new URL(focusedUrl).port) || null; } catch { /* not a url → ignore */ }
+    if (port != null) entry = managed.find((m) => Number(m?.port) === port) || null;
+  }
+  if (!entry) entry = managed[0]; // fallback: first managed server
+
+  if (!entry?.port) return null;
+  // The adapter runs on the HOST (where the agent host / Hermes gateway lives), so the server
+  // is on loopback — 127.0.0.1 (not `localhost`, which can resolve to IPv6 ::1 on Windows; and
+  // not host.docker.internal, which is only for code running INSIDE a container).
+  return { url: `http://127.0.0.1:${entry.port}`, token: entry.token || "" };
+}
+
+/**
+ * Build the /api/reflect/trigger body for one turn. PURE + testable.
+ * Returns null when there's nothing worth sending (no response, or under 50 chars).
+ *   turn = { agentResponse, userMessage?, reasoning?, agentId?, turnName?, hint? }
+ */
+export function buildTriggerBody(turn) {
+  const response = String(turn?.agentResponse ?? "").slice(0, CAPS.response);
+  const reasoning = String(turn?.reasoning ?? "").slice(0, CAPS.reasoning);
+  const user = String(turn?.userMessage ?? "").slice(0, CAPS.user);
+  // Need SOME answer, but floor on COMBINED content — a short answer can ride a rich trace.
+  if (!response.trim()) return null;
+  if ((response + reasoning + user).trim().length < TRIGGER_MIN_CHARS) return null;
+  const body = {
+    agent_response: response,
+    user_message: user,
+    agent_thinking: reasoning,
+    agent_id: turn?.agentId || "agent",
+    turn_name: turn?.turnName || "turn",
+    hint: turn?.hint || "discovery",
+    loaded_block_ids: [],
+  };
+  // turn_number enables ARC mode: the server stores this turn in conversation_turns keyed by
+  // (agent_id, turn_number) and later assembles the range. Only sent when the adapter supplies a
+  // monotonic ordinal; adapters that don't (per-turn-only capture) simply omit it → atomic mode.
+  if (Number.isInteger(turn?.turnNumber)) body.turn_number = turn.turnNumber;
+  return body;
+}
+
+/** POST a body to a resolved target's /api/reflect/trigger. Never throws → true on success. */
+export async function postTrigger(target, body, timeoutMs = 5000) {
+  if (!target?.url || !body) return false;
+  try {
+    const res = await fetch(`${target.url}/api/reflect/trigger`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...(target.token ? { "x-nodedex-token": target.token } : {}) },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    return res.ok;
+  } catch {
+    return false; // server down / unreachable → capture is best-effort
+  }
+}
+
+/**
+ * Convenience: build → resolve → post. Returns a short status string (for logging only):
+ *   "captured" | "skipped:short" | "skipped:no-server" | "failed:post"
+ * Never throws — capture must never affect the agent's turn.
+ */
+export async function captureTurn(turn) {
+  try {
+    const body = buildTriggerBody(turn);
+    if (!body) return "skipped:short";
+    const target = resolveNodedexTarget();
+    if (!target) return "skipped:no-server";
+    return (await postTrigger(target, body)) ? "captured" : "failed:post";
+  } catch {
+    return "failed:post";
+  }
+}

package/adapters/nodedex-capture.mjs

@@ -0,0 +1,169 @@
+// nodedex-capture.mjs — non-intrusive TEE capture adapter.
+//
+// JOB
+//   Send a COPY of each finished agent turn to the Nodedex pipeline so it can extract
+//   decisions / dead-ends / chains. The MCP server is PASSIVE — it only sees tool-call
+//   args + its own responses, never the agent's natural-language output. So capture
+//   can't be done BY the server; it must be PUSHED by the host. This is that push.
+//
+// THE RULE THAT MAKES IT SAFE
+//   OUT-of-path and fire-and-forget — the agent's OWN llm call is NEVER touched, never
+//   slowed, never broken. You call this AFTER a turn completes and ignore the result.
+//   The server already does debounce / dedup / pause-gating / async extraction
+//   (POST /api/reflect/trigger), so this client stays dumb: build payload, POST, swallow.
+//
+// UNIVERSAL
+//   Dependency-free; runs on Node / Bun / Deno. Probes every known reasoning shape
+//   (OpenAI / OpenRouter / Anthropic) so it captures chain-of-thought when present.
+//
+// CONFIGURABLE — choose which fields to capture (env, or a per-call `capture` override):
+//   NODEDEX_CAPTURE_RESPONSE  (default on)  the agent's emitted answer — the SUBSTRATE.
+//                                           off ⇒ nothing is sent (the pipeline needs it).
+//   NODEDEX_CAPTURE_USER      (default on)  the user's message.
+//   NODEDEX_CAPTURE_REASONING (default on)  the agent's reasoning / thinking, if available.
+//   NODEDEX_URL               (default http://localhost:3001)  where the server lives.
+//   NODEDEX_CAPTURE_BUFFER    (default off) on ⇒ buffer to ~/.nodedex/capture-buffer.jsonl
+//                                           when the server is down, flush on next success.
+
+const env = (typeof process !== "undefined" && process.env) ? process.env : {};
+
+const CAPS = { response: 16000, reasoning: 8000, user: 2000 };
+export const DEFAULT_CAPTURE = { response: true, user: true, reasoning: true };
+
+function flag(name, dflt) {
+  const v = env[name];
+  if (v == null || v === "") return dflt;
+  return !/^(0|off|false|no)$/i.test(String(v).trim());
+}
+
+/** Resolve the capture config: per-call override wins over env, env over default. */
+export function resolveCaptureConfig(override) {
+  return {
+    response:  override?.response  ?? flag("NODEDEX_CAPTURE_RESPONSE",  DEFAULT_CAPTURE.response),
+    user:      override?.user      ?? flag("NODEDEX_CAPTURE_USER",      DEFAULT_CAPTURE.user),
+    reasoning: override?.reasoning ?? flag("NODEDEX_CAPTURE_REASONING", DEFAULT_CAPTURE.reasoning),
+  };
+}
+
+/** Probe every known provider shape for chain-of-thought. "" if none present. */
+export function extractReasoning(message) {
+  if (!message || typeof message !== "object") return "";
+  let r = "";
+  for (const f of ["reasoning", "reasoning_content"])                 // OpenAI / OpenRouter / DeepSeek
+    if (typeof message[f] === "string") r += message[f];
+  if (Array.isArray(message.reasoning_details))                       // OpenRouter detailed
+    for (const d of message.reasoning_details) if (d && typeof d.text === "string") r += d.text;
+  if (Array.isArray(message.content))                                 // Anthropic-style thinking parts
+    for (const p of message.content) if (p && p.type === "thinking" && p.thinking) r += p.thinking + "\n";
+  return r.trim();
+}
+
+function slug(s) {
+  return String(s || "").split(/\s+/).slice(0, 5).join("-").toLowerCase()
+    .replace(/[^a-z0-9-]/g, "").slice(0, 60) || "turn";
+}
+
+/**
+ * Build the /api/reflect/trigger payload for one turn under a capture config.
+ * PURE + testable. Returns null when nothing is capturable: response disabled, or the
+ * answer is under 50 chars (the server rejects those anyway).
+ *
+ *   turn = { userMessage, agentResponse, reasoning?, agentId?, turnNumber?, turnName?,
+ *            loadedBlockIds?, hint? }
+ */
+export function buildCapturePayload(turn, override) {
+  const cfg = resolveCaptureConfig(override);
+  if (!cfg.response) return null;                                     // substrate disabled ⇒ skip
+  const response = String(turn?.agentResponse ?? "").slice(0, CAPS.response);
+  if (response.trim().length < 50) return null;                      // server 400s on <50
+  return {
+    agent_response:   response,
+    user_message:     cfg.user      ? String(turn?.userMessage ?? "").slice(0, CAPS.user)      : "",
+    agent_thinking:   cfg.reasoning ? String(turn?.reasoning   ?? "").slice(0, CAPS.reasoning) : "",
+    agent_id:         turn?.agentId,
+    turn_number:      typeof turn?.turnNumber === "number" ? turn.turnNumber : undefined,
+    // Don't leak the user message into the name when user-capture is off.
+    turn_name:        turn?.turnName || (cfg.user ? slug(turn?.userMessage) : "turn"),
+    loaded_block_ids: Array.isArray(turn?.loadedBlockIds) ? turn.loadedBlockIds : [],
+    hint:             turn?.hint || "discovery",
+  };
+}
+
+function baseUrl(url) {
+  return String(url || env.NODEDEX_URL || "http://localhost:3001").replace(/\/+$/, "");
+}
+
+/** Resolve the local buffer handle — only when buffering is enabled AND fs is reachable. */
+async function bufferHandle() {
+  if (!flag("NODEDEX_CAPTURE_BUFFER", false)) return null;
+  try {
+    const [{ default: fs }, { default: os }, { default: path }] = await Promise.all([
+      import("node:fs"), import("node:os"), import("node:path"),
+    ]);
+    const dir = path.join(os.homedir(), ".nodedex");
+    return { fs, dir, file: path.join(dir, "capture-buffer.jsonl") };
+  } catch { return null; }                                            // non-node runtime → no buffering
+}
+
+/** Auth header for a token-gated server (NODEDEX_BIND_HOST=0.0.0.0 + NODEDEX_API_TOKEN).
+ *  /api/reflect/trigger is gated, so set NODEDEX_TOKEN where the adapter runs or captures
+ *  get 401'd. Empty when the server is open (localhost / no token). */
+function authHeaders() {
+  const t = env.NODEDEX_TOKEN || env.NODEDEX_API_TOKEN || "";
+  return t ? { "x-nodedex-token": t } : {};
+}
+
+async function post(url, payload) {
+  const res = await fetch(`${url}/api/reflect/trigger`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", ...authHeaders() },
+    body: JSON.stringify(payload),
+    signal: AbortSignal.timeout(4000),
+  });
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+}
+
+/** Opportunistically drain any payloads buffered by prior failures. No-op if buffering off. */
+export async function flushCaptureBuffer(url) {
+  const b = await bufferHandle();
+  if (!b || !b.fs.existsSync(b.file)) return;
+  let lines;
+  try { lines = b.fs.readFileSync(b.file, "utf8").trim().split("\n").filter(Boolean); } catch { return; }
+  const u = baseUrl(url);
+  let sent = 0;
+  for (; sent < lines.length; sent++) {
+    try { await post(u, JSON.parse(lines[sent])); } catch { break; } // server still down → stop
+  }
+  try {
+    if (sent >= lines.length) b.fs.unlinkSync(b.file);
+    else b.fs.writeFileSync(b.file, lines.slice(sent).join("\n") + "\n");
+  } catch { /* best-effort */ }
+}
+
+async function bufferOnFail(payload) {
+  const b = await bufferHandle();
+  if (!b) return;
+  try {
+    b.fs.mkdirSync(b.dir, { recursive: true });
+    b.fs.appendFileSync(b.file, JSON.stringify({ ...payload, _buffered_at: Date.now() }) + "\n");
+  } catch { /* best-effort */ }
+}
+
+/**
+ * Tee one finished turn into the pipeline. Returns immediately (fire-and-forget),
+ * never throws, never blocks the agent. Call AFTER your turn completes.
+ *
+ *   options = { url?, capture?: { response?, user?, reasoning? } }
+ */
+export function nodedexCapture(turn, options = {}) {
+  const payload = buildCapturePayload(turn, options.capture);
+  if (!payload) return;                                              // nothing capturable this turn
+  const url = baseUrl(options.url);
+  // Out-of-path: drain prior failures, send, buffer-on-fail — all detached from the turn.
+  (async () => {
+    await flushCaptureBuffer(url);
+    try { await post(url, payload); } catch { await bufferOnFail(payload); }
+  })().catch(() => { /* capture must NEVER affect the agent's call */ });
+}
+
+export default nodedexCapture;

package/dist/agent-protocol.d.ts

@@ -0,0 +1,7 @@
+export declare const AGENT_PROTOCOL = "Nodedex is your persistent memory \u2014 a graph of what was decided, tried, ABANDONED, and constrained across sessions. You did NOT write it \u2014 a background pipeline extracts it from your conversation, so DON'T call save/write tools in normal work (they're for fixing graph errors only). READ before you act.\n\nREACH THE GRAPH ONLY THROUGH THESE TOOLS. Don't go hunting the filesystem or shell \u2014 never read Nodedex's database, log, or config files directly, and don't run raw SQL: you'd get rows stripped of the chains that carry the meaning, and you'd miss the dedup/heal the tools apply. NAVIGATE (traverse), don't blindly query. Touch the underlying DB/files ONLY if you genuinely need something no tool can reach \u2014 that's the rare exception, not the default.\n\nTWO REFLEXES \u2014 do these unprompted:\n1. BEFORE proposing ANY approach, check what already failed \u2014 suggesting something abandoned, or that breaks a constraint, is the worst outcome. \u2192 workspace_filter the approach's concepts (or workspace_get a dead_end label); read the dead_ends + constraints FIRST.\n2. When you need context, TRAVERSE \u2014 don't keyword-search. A block is a headline; the causal CHAIN it sits on is the story. Anchor on a block, then walk.\n\nTHE LOOP (traversal-first):\n- New task, no anchor? \u2192 workspace_filter(concepts) \u2014 first-principle terms of your task (technologies, mechanisms, failure-modes; NOT \"fix\"/\"issue\"). Returns relevant project ROOTS + entry blocks.\n- Know it exists but not the label? \u2192 workspace_search (fuzzy \u2014 the FALLBACK; isolated matches).\n- Know exactly what you want? \u2192 CONSTRUCT the label and get it directly: {project}_{entity}_{type}_{concept} ('_' between dimensions, '-' within a concept, entity optional).\n- Have a block? \u2192 workspace_get(label, detail=\"relations\") \u2014 returns the block PLUS the chain(s) it sits on AND the chains they lead to / rest on (the connected story). That is the unit of meaning; read it, open the next block, keep walking.\n\ndetail: surface (scan) \u2192 relations (block + its chain \u2014 the traversal default) \u2192 full. Graph knowledge is established; your own conclusions are hypotheses until the graph agrees.";
+export declare const NODEDEX_BEGIN = "<!-- nodedex:protocol:begin -->";
+export declare const NODEDEX_END = "<!-- nodedex:protocol:end -->";
+/** The protocol as a marked, removable block for the agent to persist into its own
+ *  standing config — only ever with explicit user permission (workspace_onboard). */
+export declare function protocolBlock(): string;
+//# sourceMappingURL=agent-protocol.d.ts.map

package/dist/agent-protocol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-protocol.d.ts","sourceRoot":"","sources":["../src/agent-protocol.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,cAAc,usEAcwJ,CAAC;AAIpL,eAAO,MAAM,aAAa,oCAAoC,CAAC;AAC/D,eAAO,MAAM,WAAW,kCAAkC,CAAC;AAE3D;qFACqF;AACrF,wBAAgB,aAAa,IAAI,MAAM,CAMtC"}

package/dist/agent-protocol.js

@@ -0,0 +1,38 @@
+// agent-protocol.ts — the ONE source of truth for what an agent must know to use
+// Nodedex well. Consumed by BOTH:
+//   • the MCP `instructions` field (server.ts) — the advisory FLOOR, every host,
+//     re-sent each connect, may or may not be surfaced.
+//   • workspace_onboard (tools/system.ts) — the persistent UPGRADE: on a capable
+//     host the agent writes this into its OWN standing config (CLAUDE.md / rules
+//     file) so the reflexes survive and are reliably followed.
+// We can't reprogram the agent (we're a tool), so this is how the reflexes reach it.
+// Keep it tight — it costs context tokens on every connect.
+export const AGENT_PROTOCOL = `Nodedex is your persistent memory — a graph of what was decided, tried, ABANDONED, and constrained across sessions. You did NOT write it — a background pipeline extracts it from your conversation, so DON'T call save/write tools in normal work (they're for fixing graph errors only). READ before you act.
+
+REACH THE GRAPH ONLY THROUGH THESE TOOLS. Don't go hunting the filesystem or shell — never read Nodedex's database, log, or config files directly, and don't run raw SQL: you'd get rows stripped of the chains that carry the meaning, and you'd miss the dedup/heal the tools apply. NAVIGATE (traverse), don't blindly query. Touch the underlying DB/files ONLY if you genuinely need something no tool can reach — that's the rare exception, not the default.
+
+TWO REFLEXES — do these unprompted:
+1. BEFORE proposing ANY approach, check what already failed — suggesting something abandoned, or that breaks a constraint, is the worst outcome. → workspace_filter the approach's concepts (or workspace_get a dead_end label); read the dead_ends + constraints FIRST.
+2. When you need context, TRAVERSE — don't keyword-search. A block is a headline; the causal CHAIN it sits on is the story. Anchor on a block, then walk.
+
+THE LOOP (traversal-first):
+- New task, no anchor? → workspace_filter(concepts) — first-principle terms of your task (technologies, mechanisms, failure-modes; NOT "fix"/"issue"). Returns relevant project ROOTS + entry blocks.
+- Know it exists but not the label? → workspace_search (fuzzy — the FALLBACK; isolated matches).
+- Know exactly what you want? → CONSTRUCT the label and get it directly: {project}_{entity}_{type}_{concept} ('_' between dimensions, '-' within a concept, entity optional).
+- Have a block? → workspace_get(label, detail="relations") — returns the block PLUS the chain(s) it sits on AND the chains they lead to / rest on (the connected story). That is the unit of meaning; read it, open the next block, keep walking.
+
+detail: surface (scan) → relations (block + its chain — the traversal default) → full. Graph knowledge is established; your own conclusions are hypotheses until the graph agrees.`;
+// Markers wrap the persisted block so a re-run can REPLACE it in place (never
+// duplicate) and the user can remove it in one move.
+export const NODEDEX_BEGIN = "<!-- nodedex:protocol:begin -->";
+export const NODEDEX_END = "<!-- nodedex:protocol:end -->";
+/** The protocol as a marked, removable block for the agent to persist into its own
+ *  standing config — only ever with explicit user permission (workspace_onboard). */
+export function protocolBlock() {
+    return `${NODEDEX_BEGIN}
+# Nodedex memory protocol — added with your permission. Delete this whole block to opt out.
+
+${AGENT_PROTOCOL}
+${NODEDEX_END}`;
+}
+//# sourceMappingURL=agent-protocol.js.map

package/dist/agent-protocol.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-protocol.js","sourceRoot":"","sources":["../src/agent-protocol.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,kCAAkC;AAClC,iFAAiF;AACjF,wDAAwD;AACxD,iFAAiF;AACjF,iFAAiF;AACjF,+DAA+D;AAC/D,qFAAqF;AACrF,4DAA4D;AAE5D,MAAM,CAAC,MAAM,cAAc,GAAG;;;;;;;;;;;;;;mLAcqJ,CAAC;AAEpL,8EAA8E;AAC9E,qDAAqD;AACrD,MAAM,CAAC,MAAM,aAAa,GAAG,iCAAiC,CAAC;AAC/D,MAAM,CAAC,MAAM,WAAW,GAAG,+BAA+B,CAAC;AAE3D;qFACqF;AACrF,MAAM,UAAU,aAAa;IAC3B,OAAO,GAAG,aAAa;;;EAGvB,cAAc;EACd,WAAW,EAAE,CAAC;AAChB,CAAC"}

package/dist/api-server.d.ts

@@ -0,0 +1,5 @@
+import { WorkspaceDB } from "./store/database.js";
+import { EmbeddingEngine } from "./engine/embeddings.js";
+import type { SchedulerJobStatus } from "./server.js";
+export declare function startApiServer(db: WorkspaceDB, schedulerHealth?: Record<string, SchedulerJobStatus>, port?: number, embeddings?: EmbeddingEngine): import("http").Server;
+//# sourceMappingURL=api-server.d.ts.map

package/dist/api-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-server.d.ts","sourceRoot":"","sources":["../src/api-server.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AA8BtD,wBAAgB,cAAc,CAC5B,EAAE,EAAE,WAAW,EACf,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,EACpD,IAAI,SAAO,EACX,UAAU,CAAC,EAAE,eAAe,GAC3B,OAAO,MAAM,EAAE,MAAM,CAiUvB"}