nlm-memory 0.4.2 → 0.5.1

package/migrations/005_sources.sql

@@ -1,31 +0,0 @@
--- Migration 005: sources registry.
---
--- Each row represents one transcript source the daemon should scan. The
--- three hardcoded adapters (claude-code / hermes / pi) become seeded rows
--- pointing at the same parse logic; future custom JSONL or webhook sources
--- live alongside them.
---
--- Parse config is a JSON blob whose shape depends on `kind`:
---   - "claude-code" / "hermes" / "pi": preset adapters. parse_config is
---     reserved but unused — paths come from path_or_url.
---   - "jsonl-generic": { sessionIdField, textField, startedAtField,
---     roleField, runtimeLabel, ... }
---   - "webhook": parse_config is empty; ingest is push-based.
---
--- See docs/plans/desktop-product.md (Phase 0).
-
-CREATE TABLE IF NOT EXISTS sources (
-  id            INTEGER PRIMARY KEY AUTOINCREMENT,
-  kind          TEXT    NOT NULL CHECK (kind IN ('claude-code', 'hermes', 'pi', 'jsonl-generic', 'webhook')),
-  name          TEXT    NOT NULL UNIQUE,
-  path_or_url   TEXT,
-  runtime_label TEXT    NOT NULL,
-  parse_config  TEXT    NOT NULL DEFAULT '{}',
-  enabled       INTEGER NOT NULL DEFAULT 1,
-  created_at    TEXT    NOT NULL DEFAULT (datetime('now')),
-  updated_at    TEXT    NOT NULL DEFAULT (datetime('now'))
-);
-
-CREATE INDEX IF NOT EXISTS idx_sources_enabled ON sources(enabled) WHERE enabled = 1;
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (5, '005_sources');

package/migrations/006_providers.sql

@@ -1,33 +0,0 @@
--- Migration 006: providers registry.
---
--- Each row represents one LLM endpoint configured for classification or
--- (future) embedding. The classifier reads this table to choose a
--- provider/model at boot; the UI mutates it through /api/providers.
---
--- API key storage. v0 stores keys in the api_key column. This is fine
--- because the SQLite file already contains the user's transcripts —
--- anyone with disk access already has everything sensitive. Phase 2
--- (Tauri shell) migrates keys to the OS keychain and replaces the column
--- with a keychain reference. The API shape stays the same.
---
--- `kind` is the structural family (openai-compatible, ollama,
--- anthropic-native, deepseek). The classifier+model UI uses it to decide
--- which client class to instantiate.
---
--- See docs/plans/desktop-product.md (Phase 0 task 3).
-
-CREATE TABLE IF NOT EXISTS providers (
-  id            INTEGER PRIMARY KEY AUTOINCREMENT,
-  kind          TEXT    NOT NULL CHECK (kind IN ('deepseek', 'ollama', 'openai', 'anthropic', 'openrouter', 'openai-compatible')),
-  name          TEXT    NOT NULL UNIQUE,
-  base_url      TEXT,
-  api_key       TEXT,
-  default_model TEXT,
-  enabled       INTEGER NOT NULL DEFAULT 1,
-  created_at    TEXT    NOT NULL DEFAULT (datetime('now')),
-  updated_at    TEXT    NOT NULL DEFAULT (datetime('now'))
-);
-
-CREATE INDEX IF NOT EXISTS idx_providers_enabled ON providers(enabled) WHERE enabled = 1;
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (6, '006_providers');

package/migrations/007_source_tokens.sql

@@ -1,17 +0,0 @@
--- Migration 007: webhook token on sources.
---
--- Webhook sources need a bearer token so external tools (anything that
--- pushes sessions via POST /api/ingest) can authenticate. The token is
--- stored alongside the source row so one webhook = one token = one
--- provenance label.
---
--- Storage policy mirrors providers.api_key: column in the canonical
--- SQLite for v0 (the DB file already holds the user's transcripts,
--- adding a token doesn't change the threat model). Phase 2 migrates to
--- OS keychain without changing the API shape.
---
--- See docs/plans/desktop-product.md (Phase 0 task 5).
-
-ALTER TABLE sources ADD COLUMN token TEXT;
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (7, '007_source_tokens');

package/migrations/008_fts_rebuild.sql

@@ -1,9 +0,0 @@
--- One-time safety rebuild of the sessions_fts external-content FTS5 index.
--- The virtual table and its sync triggers (sessions_ai / sessions_au /
--- sessions_ad) were declared in migration 000 and have fired on every write
--- since, so the index is normally already in sync. This rebuild guarantees
--- the index matches every existing sessions row before the recall path
--- starts depending on FTS5 for keyword search. Safe and idempotent.
-INSERT INTO sessions_fts(sessions_fts) VALUES('rebuild');
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (8, 'fts_rebuild');

package/migrations/009_session_embedding_chunks.sql

@@ -1,46 +0,0 @@
--- Migration 009: chunk + max-pool semantic index.
---
--- Replaces session_embeddings (one vector per session, truncated at
--- MAX_EMBED_CHARS=8000) with per-chunk vectors. Recall-time score is
--- max cosine across chunks per session.
---
--- The 2026-05-25 LongMemEval-S baseline showed 98% of gold sessions
--- exceed 8000 chars and were silently truncated. Raising the per-call
--- cap (#172) hit Ollama 500s on >50% of long inputs. Chunking sidesteps
--- both: each chunk is well under the Ollama failure cliff, and the full
--- body becomes searchable. Expected lift: semantic R@5 87.2 → >92,
--- hybrid R@5 94.6 → >96.
---
--- Schema choices:
---   * Auxiliary columns (+session_id, +chunk_idx) so KNN queries return
---     session attribution without a join.
---   * Separate session_chunk_map keyed on session_id supports
---     `DELETE FROM session_embedding_chunks WHERE chunk_id IN
---     (SELECT chunk_id FROM session_chunk_map WHERE session_id = ?)`
---     since vec0 has no documented filtering on aux columns.
---
--- session_embeddings (single-vector) is intentionally left in place:
---   * keeps rollback trivial (revert recall code, old vectors still there)
---   * avoids forcing a multi-hour re-embed at deploy time; backfill
---     populates chunks asynchronously
---   * a future cleanup migration drops it once chunks are validated
-
-CREATE VIRTUAL TABLE IF NOT EXISTS session_embedding_chunks USING vec0(
-  chunk_id   INTEGER PRIMARY KEY,
-  embedding  float[768],
-  +session_id TEXT,
-  +chunk_idx  INTEGER
-);
-
-CREATE TABLE IF NOT EXISTS session_chunk_map (
-  chunk_id   INTEGER PRIMARY KEY,
-  session_id TEXT NOT NULL,
-  chunk_idx  INTEGER NOT NULL,
-  FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE
-);
-
-CREATE INDEX IF NOT EXISTS idx_session_chunk_map_session
-  ON session_chunk_map(session_id);
-
-INSERT OR IGNORE INTO schema_migrations (version, name)
-  VALUES (9, '009_session_embedding_chunks');

package/migrations/010_sources_opencode.sql

@@ -1,30 +0,0 @@
--- Migration 010: add 'opencode' to the sources.kind CHECK constraint.
---
--- SQLite does not support ALTER COLUMN to modify CHECK constraints in place.
--- Standard approach: rename → recreate → copy → drop old.
-
-PRAGMA foreign_keys = OFF;
-
-CREATE TABLE sources_new (
-  id            INTEGER PRIMARY KEY AUTOINCREMENT,
-  kind          TEXT    NOT NULL CHECK (kind IN ('claude-code', 'hermes', 'opencode', 'pi', 'jsonl-generic', 'webhook')),
-  name          TEXT    NOT NULL UNIQUE,
-  path_or_url   TEXT,
-  runtime_label TEXT    NOT NULL,
-  parse_config  TEXT    NOT NULL DEFAULT '{}',
-  enabled       INTEGER NOT NULL DEFAULT 1,
-  token         TEXT,
-  created_at    TEXT    NOT NULL DEFAULT (datetime('now')),
-  updated_at    TEXT    NOT NULL DEFAULT (datetime('now'))
-);
-
-INSERT INTO sources_new SELECT id, kind, name, path_or_url, runtime_label, parse_config, enabled, token, created_at, updated_at FROM sources;
-
-DROP TABLE sources;
-ALTER TABLE sources_new RENAME TO sources;
-
-CREATE INDEX IF NOT EXISTS idx_sources_enabled ON sources(enabled) WHERE enabled = 1;
-
-PRAGMA foreign_keys = ON;
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (10, '010_sources_opencode');

package/migrations/011_sources_hermes_agent.sql

@@ -1,30 +0,0 @@
--- Migration 011: add 'hermes-agent' to the sources.kind CHECK constraint.
---
--- SQLite does not support ALTER COLUMN to modify CHECK constraints in place.
--- Standard approach: rename → recreate → copy → drop old.
-
-PRAGMA foreign_keys = OFF;
-
-CREATE TABLE sources_new (
-  id            INTEGER PRIMARY KEY AUTOINCREMENT,
-  kind          TEXT    NOT NULL CHECK (kind IN ('claude-code', 'hermes', 'hermes-agent', 'opencode', 'pi', 'jsonl-generic', 'webhook')),
-  name          TEXT    NOT NULL UNIQUE,
-  path_or_url   TEXT,
-  runtime_label TEXT    NOT NULL,
-  parse_config  TEXT    NOT NULL DEFAULT '{}',
-  enabled       INTEGER NOT NULL DEFAULT 1,
-  token         TEXT,
-  created_at    TEXT    NOT NULL DEFAULT (datetime('now')),
-  updated_at    TEXT    NOT NULL DEFAULT (datetime('now'))
-);
-
-INSERT INTO sources_new SELECT id, kind, name, path_or_url, runtime_label, parse_config, enabled, token, created_at, updated_at FROM sources;
-
-DROP TABLE sources;
-ALTER TABLE sources_new RENAME TO sources;
-
-CREATE INDEX IF NOT EXISTS idx_sources_enabled ON sources(enabled) WHERE enabled = 1;
-
-PRAGMA foreign_keys = ON;
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (11, '011_sources_hermes_agent');

package/migrations/012_sources_aider.sql

@@ -1,30 +0,0 @@
--- Migration 012: add 'aider' to the sources.kind CHECK constraint.
---
--- SQLite does not support ALTER COLUMN to modify CHECK constraints in place.
--- Standard approach: rename → recreate → copy → drop old.
-
-PRAGMA foreign_keys = OFF;
-
-CREATE TABLE sources_new (
-  id            INTEGER PRIMARY KEY AUTOINCREMENT,
-  kind          TEXT    NOT NULL CHECK (kind IN ('claude-code', 'hermes', 'hermes-agent', 'aider', 'opencode', 'pi', 'jsonl-generic', 'webhook')),
-  name          TEXT    NOT NULL UNIQUE,
-  path_or_url   TEXT,
-  runtime_label TEXT    NOT NULL,
-  parse_config  TEXT    NOT NULL DEFAULT '{}',
-  enabled       INTEGER NOT NULL DEFAULT 1,
-  token         TEXT,
-  created_at    TEXT    NOT NULL DEFAULT (datetime('now')),
-  updated_at    TEXT    NOT NULL DEFAULT (datetime('now'))
-);
-
-INSERT INTO sources_new SELECT id, kind, name, path_or_url, runtime_label, parse_config, enabled, token, created_at, updated_at FROM sources;
-
-DROP TABLE sources;
-ALTER TABLE sources_new RENAME TO sources;
-
-CREATE INDEX IF NOT EXISTS idx_sources_enabled ON sources(enabled) WHERE enabled = 1;
-
-PRAGMA foreign_keys = ON;
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (12, '012_sources_aider');

package/migrations/013_adapter_state_failure_count.sql

@@ -1,12 +0,0 @@
--- Migration 013: add failure_count to adapter_state.
---
--- Tracks consecutive classify/storage failures per source file. The scheduler
--- skips files whose failure_count has reached the backoff ceiling (default 3)
--- and whose file_size has not changed since the last attempt. When the file
--- grows (new content appended), failure_count resets to 0 and the file is
--- retried. This prevents a single permanently-broken or over-large transcript
--- from flooding daemon-err.log on every 30-minute tick.
-
-ALTER TABLE adapter_state ADD COLUMN failure_count INTEGER NOT NULL DEFAULT 0;
-
-INSERT OR IGNORE INTO schema_migrations (version, name) VALUES (13, '013_adapter_state_failure_count');

package/plugin-hermes-agent/README.md

@@ -1,49 +0,0 @@
-# nlm-memory Hermes Agent plugin
-
-This directory is the [NousResearch Hermes Agent](https://github.com/NousResearch/hermes-agent) plugin distribution surface for nlm-memory.
-
-## Install
-
-Prerequisite: `npm install -g nlm-memory` (puts `nlm` on PATH; the MCP server spawns `nlm mcp`).
-
-```bash
-# Recommended — uses nlm connect to copy the plugin and enable it
-nlm connect hermes-agent
-
-# Or manually
-cp -r plugin-hermes-agent ~/.hermes/plugins/nlm-memory
-hermes plugins enable nlm-memory
-nlm connect hermes          # also writes the MCP server entry to ~/.hermes/config.yaml
-```
-
-## What this ships
-
-- **`plugin.yaml`** — Hermes plugin manifest (name: `nlm-memory`, kind: `memory`)
-- **`__init__.py`** — Python shim that registers 6 lifecycle hooks; all delegate to the local nlm daemon at `http://localhost:3940`
-
-## How it works
-
-| Hermes hook | Action |
-|---|---|
-| `pre_llm_call` | Keyword-recalls up to 3 relevant prior sessions; injects them as context above the user message |
-| `post_llm_call` | Scans the assistant response for cited session IDs and logs prose citation events |
-| `on_session_start` | No-op (memo is created lazily on first recall hit) |
-| `on_session_end` | Clears the per-session surfaced-ID memo |
-| `on_session_finalize` | Clears the per-session surfaced-ID memo |
-| `on_session_reset` | Clears the per-session surfaced-ID memo |
-
-## Configuration
-
-| Env var | Default | Purpose |
-|---|---|---|
-| `NLM_DAEMON_PORT` | `3940` | Port the nlm daemon listens on |
-
-## Daemon required
-
-The plugin delegates entirely to the nlm daemon. Start it with `nlm start` or `nlm install` (macOS LaunchAgent). If the daemon is unreachable, every hook silently returns `None` — the agent loop is never blocked.
-
-## Uninstall
-
-```bash
-nlm disconnect hermes-agent
-```

package/plugin-hermes-agent/__init__.py

@@ -1,75 +0,0 @@
-"""
-nlm-memory Hermes Agent plugin.
-
-Wires 6 Hermes lifecycle hooks to the local nlm-memory daemon
-(http://localhost:3940 by default). All calls are fire-and-forget
-except pre_llm_call, which returns a context string for injection.
-
-The daemon must be running for hooks to have effect. If it is unreachable,
-every hook silently returns None so the agent loop is never blocked.
-"""
-
-import json
-import os
-import urllib.error
-import urllib.request
-from typing import Any
-
-
-def _base_url() -> str:
-    port = os.environ.get("NLM_DAEMON_PORT", "3940")
-    return f"http://localhost:{port}"
-
-
-def _post(path: str, payload: dict[str, Any]) -> dict[str, Any]:
-    url = f"{_base_url()}/api/{path}"
-    data = json.dumps(payload).encode()
-    req = urllib.request.Request(
-        url,
-        data=data,
-        headers={"Content-Type": "application/json"},
-        method="POST",
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=5) as resp:
-            return json.loads(resp.read())
-    except (urllib.error.URLError, OSError, json.JSONDecodeError):
-        return {}
-
-
-def register(ctx: Any) -> None:
-    def pre_llm_call(
-        session_id: str,
-        user_message: str,
-        is_first_turn: bool = False,
-        **kwargs: Any,
-    ) -> str | None:
-        result = _post(
-            "hook/hermes-agent/pre-turn",
-            {"session_id": session_id, "user_message": user_message, "is_first_turn": is_first_turn},
-        )
-        context = result.get("context")
-        return context if isinstance(context, str) and context else None
-
-    def post_llm_call(
-        session_id: str,
-        assistant_response: str,
-        **kwargs: Any,
-    ) -> None:
-        _post(
-            "hook/hermes-agent/post-turn",
-            {"session_id": session_id, "assistant_response": assistant_response},
-        )
-
-    def _session_event(event: str, session_id: str | None, **kwargs: Any) -> None:
-        _post(
-            "hook/hermes-agent/session-lifecycle",
-            {"event": event, "session_id": session_id},
-        )
-
-    ctx.register_hook("pre_llm_call", pre_llm_call)
-    ctx.register_hook("post_llm_call", post_llm_call)
-    ctx.register_hook("on_session_start", lambda session_id, **kw: _session_event("start", session_id, **kw))
-    ctx.register_hook("on_session_end", lambda session_id, **kw: _session_event("end", session_id, **kw))
-    ctx.register_hook("on_session_finalize", lambda session_id=None, **kw: _session_event("finalize", session_id, **kw))
-    ctx.register_hook("on_session_reset", lambda session_id, **kw: _session_event("reset", session_id, **kw))

package/plugin-hermes-agent/plugin.yaml

@@ -1,15 +0,0 @@
-name: nlm-memory
-version: 0.1.0
-description: Non-linear memory for Hermes — surfaces relevant prior sessions at each turn via keyword recall
-kind: memory
-author: pbmagnet4
-provides_hooks:
-  - pre_llm_call
-  - post_llm_call
-  - on_session_start
-  - on_session_end
-  - on_session_finalize
-  - on_session_reset
-requires_env:
-  - name: NLM_DAEMON_PORT
-    description: "Port the nlm-memory daemon listens on (default: 3940)"

package/scripts/build-codex-plugin.mjs

@@ -1,61 +0,0 @@
-#!/usr/bin/env node
-/**
- * Bundles each hook entry under src/hook/ as a single-file .mjs into
- * plugin/scripts/, ready for Codex to invoke via ${CLAUDE_PLUGIN_ROOT}.
- *
- * Why single-file bundles: Codex plugins are content-addressed by file hash
- * for the trust system. Shipping a tree of imports would force the user to
- * re-trust every release across dozens of files. One file per hook keeps
- * the trust footprint minimal.
- *
- * Why esbuild not tsc: tsc emits a tree of files with relative imports; the
- * plugin would have to ship `dist/` plus its dependency tree. esbuild flattens
- * to one .mjs per entry with all internal deps inlined. node builtins are
- * left as-is (node:*) and zero runtime deps are bundled because the hook
- * scripts only talk to the daemon over HTTP — no native bindings required.
- */
-
-import { build } from "esbuild";
-import { resolve, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-import { mkdirSync } from "node:fs";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const REPO_ROOT = resolve(__dirname, "..");
-const OUT_DIR = resolve(REPO_ROOT, "plugin/scripts");
-
-const ENTRIES = [
-  "src/hook/prompt-recall-hook.ts",
-  "src/hook/stop-hook.ts",
-];
-
-mkdirSync(OUT_DIR, { recursive: true });
-
-const results = await Promise.all(
-  ENTRIES.map(async (entry) => {
-    const entryPath = resolve(REPO_ROOT, entry);
-    const outName = entry.split("/").pop().replace(/\.ts$/, ".mjs");
-    const outFile = resolve(OUT_DIR, outName);
-    const result = await build({
-      entryPoints: [entryPath],
-      outfile: outFile,
-      bundle: true,
-      platform: "node",
-      format: "esm",
-      target: "node20",
-      minify: false,
-      sourcemap: false,
-      legalComments: "none",
-      logLevel: "warning",
-      banner: { js: "#!/usr/bin/env node" },
-      tsconfig: resolve(REPO_ROOT, "tsconfig.json"),
-    });
-    return { entry, outFile, warnings: result.warnings.length };
-  }),
-);
-
-for (const r of results) {
-  process.stdout.write(`bundled ${r.entry} -> ${r.outFile.replace(REPO_ROOT + "/", "")}`);
-  if (r.warnings > 0) process.stdout.write(` (${r.warnings} warnings)`);
-  process.stdout.write("\n");
-}

package/scripts/deepseek-probe.mjs

@@ -1,67 +0,0 @@
-#!/usr/bin/env node
-// Direct DeepSeek probe — calls the API with the same prompt + a known-failing
-// session body and dumps everything: raw choice content, finish_reason, usage.
-
-import { readFileSync } from "node:fs";
-import Database from "better-sqlite3";
-
-const envContent = readFileSync(`${process.env.HOME}/.nlm/.env`, "utf8");
-const apiKey = envContent.split("\n").find((l) => l.startsWith("DEEPSEEK_API_KEY="))?.split("=")[1]?.trim();
-if (!apiKey) {
-  console.error("no DEEPSEEK_API_KEY in ~/.nlm/.env");
-  process.exit(1);
-}
-
-const db = new Database(`${process.env.HOME}/.nlm/canonical.sqlite`, { readonly: true });
-const targets = process.argv.slice(2);
-if (targets.length === 0) {
-  console.error("usage: deepseek-probe.mjs <session-id> [more ids...]");
-  process.exit(1);
-}
-
-// Pull the actual classifier prompt from the build
-const promptModule = await import("/Users/echalupa/Documents/Coding Projects/nlm-memory-ts/src/core/classifier/prompt.ts");
-
-for (const sid of targets) {
-  const row = db.prepare("SELECT body FROM sessions WHERE id = ?").get(sid);
-  if (!row?.body) {
-    console.log(`\n=== ${sid}: no body ===`);
-    continue;
-  }
-  const userPrompt = promptModule.buildUserPrompt(row.body, "");
-  console.log(`\n=== ${sid} (body ${row.body.length} chars, prompt ${userPrompt.length} chars) ===`);
-
-  const res = await fetch("https://api.deepseek.com/v1/chat/completions", {
-    method: "POST",
-    headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
-    body: JSON.stringify({
-      model: "deepseek-v4-flash",
-      messages: [
-        { role: "system", content: promptModule.CLASSIFIER_SYSTEM_PROMPT },
-        { role: "user", content: userPrompt },
-      ],
-      response_format: { type: "json_object" },
-      temperature: 0.1,
-      max_tokens: 8192,
-      stream: false,
-    }),
-  });
-  if (!res.ok) {
-    console.log(`HTTP ${res.status}: ${(await res.text()).slice(0, 500)}`);
-    continue;
-  }
-  const data = await res.json();
-  const choice = data.choices?.[0];
-  console.log(`finish_reason: ${choice?.finish_reason}`);
-  console.log(`usage: ${JSON.stringify(data.usage)}`);
-  const content = choice?.message?.content ?? "";
-  console.log(`content (${content.length} chars):`);
-  console.log(content);
-  console.log("--- end content ---");
-  try {
-    JSON.parse(content.replace(/^```(?:json)?\s*|\s*```$/gm, "").trim());
-    console.log("JSON.parse: OK");
-  } catch (e) {
-    console.log(`JSON.parse: FAILED — ${e.message}`);
-  }
-}