nlm-memory 0.5.18 → 0.5.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -17
- package/dist/cli/nlm.js +147 -80
- package/dist/cli/nlm.js.map +1 -1
- package/dist/cli/supersede.js +5 -2
- package/dist/cli/supersede.js.map +1 -1
- package/dist/core/actions/actions-log.d.ts +9 -0
- package/dist/core/actions/actions-log.js +86 -0
- package/dist/core/actions/actions-log.js.map +1 -1
- package/dist/core/digest/compose.d.ts +0 -1
- package/dist/core/digest/compose.js +0 -4
- package/dist/core/digest/compose.js.map +1 -1
- package/dist/core/providers/provider-registry.d.ts +15 -0
- package/dist/core/providers/provider-registry.js +85 -0
- package/dist/core/providers/provider-registry.js.map +1 -1
- package/dist/core/recall/query-log.d.ts +1 -2
- package/dist/core/recall/query-log.js +1 -5
- package/dist/core/recall/query-log.js.map +1 -1
- package/dist/core/scheduler/scan-once.d.ts +4 -0
- package/dist/core/scheduler/scan-once.js +64 -0
- package/dist/core/scheduler/scan-once.js.map +1 -1
- package/dist/core/scheduler/scheduler.js +25 -9
- package/dist/core/scheduler/scheduler.js.map +1 -1
- package/dist/core/sources/source-registry.d.ts +16 -0
- package/dist/core/sources/source-registry.js +123 -0
- package/dist/core/sources/source-registry.js.map +1 -1
- package/dist/core/storage/pg-fact-store.d.ts +24 -0
- package/dist/core/storage/pg-fact-store.js +233 -0
- package/dist/core/storage/pg-fact-store.js.map +1 -0
- package/dist/core/storage/pg-session-store.d.ts +29 -0
- package/dist/core/storage/pg-session-store.js +315 -0
- package/dist/core/storage/pg-session-store.js.map +1 -0
- package/dist/core/storage/pg-storage.d.ts +36 -0
- package/dist/core/storage/pg-storage.js +87 -0
- package/dist/core/storage/pg-storage.js.map +1 -0
- package/dist/core/storage/pg-tx-context.d.ts +44 -0
- package/dist/core/storage/pg-tx-context.js +133 -0
- package/dist/core/storage/pg-tx-context.js.map +1 -0
- package/dist/core/storage/sqlite-fact-store.d.ts +13 -7
- package/dist/core/storage/sqlite-fact-store.js +38 -12
- package/dist/core/storage/sqlite-fact-store.js.map +1 -1
- package/dist/core/storage/sqlite-session-store.d.ts +6 -16
- package/dist/core/storage/sqlite-session-store.js +51 -42
- package/dist/core/storage/sqlite-session-store.js.map +1 -1
- package/dist/core/storage/sqlite-storage.d.ts +38 -0
- package/dist/core/storage/sqlite-storage.js +66 -0
- package/dist/core/storage/sqlite-storage.js.map +1 -0
- package/dist/hook/pi-extension.d.ts +43 -0
- package/dist/hook/pi-extension.js +50 -0
- package/dist/hook/pi-extension.js.map +1 -0
- package/dist/hook/prompt-recall-hook.js +1 -36
- package/dist/hook/prompt-recall-hook.js.map +1 -1
- package/dist/hook/recall-over-http.d.ts +10 -0
- package/dist/hook/recall-over-http.js +41 -0
- package/dist/hook/recall-over-http.js.map +1 -0
- package/dist/hook/stop-hook.d.ts +1 -3
- package/dist/hook/stop-hook.js +1 -3
- package/dist/hook/stop-hook.js.map +1 -1
- package/dist/http/app.d.ts +6 -5
- package/dist/http/app.js +15 -4
- package/dist/http/app.js.map +1 -1
- package/dist/install/pi.d.ts +45 -0
- package/dist/install/pi.js +107 -0
- package/dist/install/pi.js.map +1 -0
- package/dist/install/setup.js +26 -10
- package/dist/install/setup.js.map +1 -1
- package/dist/ports/fact-store.d.ts +26 -0
- package/dist/ports/storage.d.ts +45 -0
- package/dist/ports/storage.js +14 -0
- package/dist/ports/storage.js.map +1 -0
- package/nlm/README.md +73 -0
- package/nlm/index.js +280 -0
- package/nlm/package.json +8 -0
- package/package.json +9 -2
- package/plugin/scripts/prompt-recall-hook.mjs +31 -29
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
<a href="https://www.npmjs.com/package/nlm-memory"><img src="https://img.shields.io/npm/v/nlm-memory?color=CB3837&label=npm&logo=npm" alt="npm version" /></a>
|
|
11
11
|
<a href="https://github.com/pbmagnet4/nlm-memory-ts/blob/main/LICENSE"><img src="https://img.shields.io/github/license/pbmagnet4/nlm-memory-ts?color=blue" alt="License: Apache 2.0" /></a>
|
|
12
12
|
<a href="https://nodejs.org"><img src="https://img.shields.io/node/v/nlm-memory?color=brightgreen" alt="Node 20+" /></a>
|
|
13
|
-
<img src="https://img.shields.io/badge/tests-
|
|
13
|
+
<img src="https://img.shields.io/badge/tests-742%20passing-success" alt="742 tests passing" />
|
|
14
14
|
<img src="https://img.shields.io/badge/runtimes-9-8A2BE2" alt="9 runtimes supported" />
|
|
15
15
|
<img src="https://img.shields.io/badge/telemetry-none-informational" alt="Zero telemetry" />
|
|
16
16
|
</p>
|
|
@@ -34,9 +34,9 @@
|
|
|
34
34
|
|
|
35
35
|
1. **Cross-runtime reach.** One index, every adapter.
|
|
36
36
|
2. **Editable timeline.** Sessions can be superseded by newer ones; entities can be retired. Patch history retroactively — no other tool lets you do this. See [docs/supersedence.md](docs/supersedence.md).
|
|
37
|
-
3. **97.2% R@5 baseline.** On a 14-month corpus, keyword recall surfaces the right session in the top 5 on 97.2% of evaluator queries. No fine-tuning
|
|
37
|
+
3. **97.2% R@5 baseline.** On a 14-month corpus, keyword recall surfaces the right session in the top 5 on 97.2% of evaluator queries. No fine-tuning. The labels were generated by DeepSeek V4 — the retrieval algorithm is the same code path you'll run, but expect a lower number with a smaller local classifier. See [docs/methodology-recall-baseline.md](docs/methodology-recall-baseline.md).
|
|
38
38
|
|
|
39
|
-
Everything stays on your machine. No telemetry, no account
|
|
39
|
+
Everything stays on your machine by default. No telemetry, no account. The classifier defaults to local (Ollama); if you opt into a cloud classifier (DeepSeek, OpenAI, Anthropic, OpenRouter, or any OpenAI-compatible endpoint), session transcripts are sent to that provider — see [Security](#security) for the exact data-flow.
|
|
40
40
|
|
|
41
41
|
---
|
|
42
42
|
|
|
@@ -85,7 +85,7 @@ One corpus across every adapter. `nlm connect` wires hooks + MCP for each runtim
|
|
|
85
85
|
| **Windsurf** | `nlm connect windsurf` | Windsurf user dir | MCP only |
|
|
86
86
|
| **OpenCode** | adapter active | `~/.local/share/opencode/` | MCP only |
|
|
87
87
|
| **Aider** | adapter active | `AIDER_CHAT_HISTORY_FILE` | MCP only |
|
|
88
|
-
| **pi.dev** |
|
|
88
|
+
| **pi.dev** | `nlm setup` (auto) or `nlm connect pi` | `~/.pi/agent/sessions/**/*.jsonl` | input (prompt-recall) |
|
|
89
89
|
|
|
90
90
|
`nlm disconnect <runtime>` reverses any of the above.
|
|
91
91
|
|
|
@@ -95,9 +95,11 @@ One corpus across every adapter. `nlm connect` wires hooks + MCP for each runtim
|
|
|
95
95
|
|
|
96
96
|
Two delivery paths. They share the same index.
|
|
97
97
|
|
|
98
|
-
### 1. Hooks
|
|
98
|
+
### 1. Hooks — automatic context injection
|
|
99
99
|
|
|
100
|
-
|
|
100
|
+
Hooks fire on user input and prepend a pointer block of likely-relevant prior sessions to the model's context. Three runtimes ship them today: Claude Code (full five-hook lifecycle), Hermes Agent (parallel set), and pi.dev (one `input` hook via [nlm/](nlm/README.md), wired by `nlm setup` or `nlm connect pi`). Full lifecycle, modes, logging surface, and the load-bearing daily liveness canary documented in [docs/hooks.md](docs/hooks.md).
|
|
101
|
+
|
|
102
|
+
The Claude Code surface (the most complete) installs five hooks into `~/.claude/settings.json`:
|
|
101
103
|
|
|
102
104
|
| Event | What NLM does | Mode |
|
|
103
105
|
|---|---|---|
|
|
@@ -197,7 +199,7 @@ What happens when an AI runtime writes a session and you later recall it:
|
|
|
197
199
|
```
|
|
198
200
|
ingest: runtime transcript (jsonl/sqlite)
|
|
199
201
|
-> adapter parses runtime-specific format
|
|
200
|
-
-> classifier (DeepSeek
|
|
202
|
+
-> classifier (Ollama local by default; DeepSeek / OpenAI / Anthropic / OpenRouter / OpenAI-compatible if you opt in) extracts label + entities + decisions + open questions
|
|
201
203
|
-> embedder (nomic-embed-text via Ollama) computes 768-dim vector
|
|
202
204
|
-> SQLite canonical store + FTS5 keyword index + sqlite-vec ANN index
|
|
203
205
|
|
|
@@ -225,13 +227,18 @@ recall: prompt / query
|
|
|
225
227
|
| `NLM_CITATION_LOG` | `~/.nlm/citation-log.jsonl` | Stop-hook citation events |
|
|
226
228
|
| `NLM_MCP_TOKEN` | auto-generated | 256-bit bearer for `/api/*` (non-browser) and `/mcp` |
|
|
227
229
|
| `NLM_MCP_CONFIG` | `~/.mcp.json` | Path the `connect`/`disconnect` commands modify |
|
|
228
|
-
| `NLM_CLASSIFIER` | `
|
|
229
|
-
| `NLM_CLASSIFIER_MODEL` | `
|
|
230
|
+
| `NLM_CLASSIFIER` | `ollama` | `ollama` (local, default), `deepseek`, `openai`, `anthropic`, `openrouter`, or `openai-compatible` |
|
|
231
|
+
| `NLM_CLASSIFIER_MODEL` | `phi4-mini:latest` | Model id for the chosen provider |
|
|
230
232
|
| `NLM_OLLAMA_URL` | `http://localhost:11434` | Override Ollama endpoint |
|
|
231
233
|
| `NLM_ADAPTERS` | all | Comma-separated allowlist of adapters to enable |
|
|
232
|
-
| `DEEPSEEK_API_KEY` | — | Required when classifier=deepseek |
|
|
234
|
+
| `DEEPSEEK_API_KEY` | — | Required only when classifier=deepseek |
|
|
235
|
+
| `NLM_DISABLE_UPDATE_CHECK` | — | Set to `1` to disable the daily npm-registry update check |
|
|
233
236
|
| `TELEGRAM_BOT_TOKEN` / `TELEGRAM_CHAT_ID` | — | Required for `nlm digest --telegram` |
|
|
234
237
|
|
|
238
|
+
### Changing the classifier from the UI
|
|
239
|
+
|
|
240
|
+
The CLI env-vars above are one path; the running UI is the other. **Settings → Providers** is a full CRUD list of LLM endpoints (Ollama, DeepSeek, OpenAI, Anthropic, OpenRouter, or any OpenAI-compatible endpoint such as LM Studio, llama.cpp, vLLM, text-generation-webui). Click **Add provider**, point it at your local server, hit **Save & test**. Then in **Settings → Classifier**, pick that provider and model. No env-var editing required, and the DeepSeek row can be disabled or deleted from the same screen.
|
|
241
|
+
|
|
235
242
|
Adapter source paths can be overridden individually: `NLM_CLAUDE_PROJECTS_PATH`, `NLM_CODEX_CONFIG`, `NLM_CURSOR_DB_PATH`, `NLM_HERMES_SESSIONS_PATH`, `NLM_HERMES_AGENT_DB_PATH`, `NLM_WINDSURF_USER_DIR`, `OPENCODE_DB_PATH`, `PI_SESSIONS_PATH`, `AIDER_CHAT_HISTORY_FILE`.
|
|
236
243
|
|
|
237
244
|
### Config file
|
|
@@ -249,19 +256,34 @@ Adapter source paths can be overridden individually: `NLM_CLAUDE_PROJECTS_PATH`,
|
|
|
249
256
|
|
|
250
257
|
## Security
|
|
251
258
|
|
|
252
|
-
NLM is local-first by design
|
|
259
|
+
NLM is local-first by design, but "local-first" is not "local-only" — read this section before picking a classifier.
|
|
260
|
+
|
|
261
|
+
**Daemon hardening (always on):**
|
|
253
262
|
|
|
254
263
|
- Binds to `127.0.0.1` only — never `0.0.0.0`
|
|
255
264
|
- Enforces Host + Origin checks on `/api/*` to defeat DNS rebinding and cross-origin drive-by
|
|
256
265
|
- Generates a 256-bit `NLM_MCP_TOKEN` on first run, persists to `~/.nlm/.env` (mode `0600`); non-browser clients authenticate with `Authorization: Bearer ${NLM_MCP_TOKEN}` compared with `timingSafeEqual`
|
|
257
266
|
- Recursively enforces `0700` on `~/.nlm/` and `0600` on its contents on every start
|
|
258
|
-
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
267
|
+
- Optional opt-in UI cookie auth (`NLM_UI_AUTH=cookie`) with HMAC-derived cookie value and nonce-based bootstrap (token never appears in a URL)
|
|
268
|
+
|
|
269
|
+
**Outbound network traffic — exhaustive list:**
|
|
270
|
+
|
|
271
|
+
| Destination | When | What data leaves |
|
|
272
|
+
|---|---|---|
|
|
273
|
+
| Configured classifier endpoint | Every new session is classified | Up to ~30K chars of the session transcript (prompts, responses, code snippets — whatever is in the transcript). If the classifier is **Ollama (default)** the destination is `localhost:11434` and nothing leaves the machine. If you opted into DeepSeek / OpenAI / Anthropic / OpenRouter / any OpenAI-compatible endpoint, the transcript is POSTed to that vendor. |
|
|
274
|
+
| Ollama `localhost:11434` | Every new session | 768-dim embedding request (local) |
|
|
275
|
+
| `registry.npmjs.org` | Once per 24h | Anonymous `GET /nlm-memory/latest` for update notifications. Cached at `~/.nlm/update-check.json`. Disable with `NLM_DISABLE_UPDATE_CHECK=1`. |
|
|
276
|
+
| `api.telegram.org` | Only when `nlm digest --telegram` is invoked | Digest content |
|
|
277
|
+
| AI runtime transcript files | Continuous | Read-only filesystem reads |
|
|
278
|
+
|
|
279
|
+
No analytics SDK. No crash reporter. No vendor ping beyond the four rows above.
|
|
280
|
+
|
|
281
|
+
**Honest caveats — known limitations:**
|
|
263
282
|
|
|
264
|
-
|
|
283
|
+
- **Cloud-classifier data egress.** A "cloud" classifier (DeepSeek, OpenAI, Anthropic, OpenRouter) by definition sees your session content. Anything pasted into a transcript — API keys, client names, internal URLs — is sent to that vendor under their data-use terms. The setup wizard warns you before you pick one. The default is Ollama for this reason.
|
|
284
|
+
- **Provider API keys are stored in plaintext in SQLite today** (`providers.api_key`, in `~/.nlm/canonical.sqlite`). The file is `0600`, in a `0700` directory, owned by your user — so any process running as your user can read it. OS-keychain migration is on the roadmap; until then, treat the SQLite file like a `.env` file.
|
|
285
|
+
- **The classifier is fed untrusted indexed content.** Sessions written by AI runtimes can contain prompt-injection attempts. The classifier output is structured (label, entities, decisions, open questions) and never executed, but if you wire NLM to an agent that *acts* on classifier output, model that as untrusted input.
|
|
286
|
+
- **The hook fails open.** Any error in the recall hook yields a clean exit so it can't block your model. This means a silently-broken hook is possible — the daily digest's `WARN hook silent` canary is the detection path.
|
|
265
287
|
|
|
266
288
|
Report vulnerabilities via [SECURITY.md](SECURITY.md).
|
|
267
289
|
|
package/dist/cli/nlm.js
CHANGED
|
@@ -34,10 +34,12 @@ import { serve } from "@hono/node-server";
|
|
|
34
34
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
35
35
|
import { FactRecallService } from "../core/recall-facts/fact-recall-service.js";
|
|
36
36
|
import { RecallService } from "../core/recall/recall-service.js";
|
|
37
|
-
import { SqliteFactStore } from "../core/storage/sqlite-fact-store.js";
|
|
38
37
|
import { ProviderRegistry } from "../core/providers/provider-registry.js";
|
|
39
38
|
import { SourceRegistry } from "../core/sources/source-registry.js";
|
|
40
|
-
import {
|
|
39
|
+
import { SqliteStorage } from "../core/storage/sqlite-storage.js";
|
|
40
|
+
import { PgStorage } from "../core/storage/pg-storage.js";
|
|
41
|
+
import { PgSourceRegistry } from "../core/sources/source-registry.js";
|
|
42
|
+
import { PgProviderRegistry } from "../core/providers/provider-registry.js";
|
|
41
43
|
import { applyPendingRestore } from "../core/storage/db-restore.js";
|
|
42
44
|
import { createApp } from "../http/app.js";
|
|
43
45
|
import { createMcpServer } from "../mcp/server.js";
|
|
@@ -56,6 +58,7 @@ import { getUpdateStatus } from "../core/update-check/check.js";
|
|
|
56
58
|
import { connectHermes, disconnectHermes, hermesConfigPath } from "../install/hermes.js";
|
|
57
59
|
import { connectHermesAgent, disconnectHermesAgent, hermesAgentPluginDir } from "../install/hermes-agent.js";
|
|
58
60
|
import { connectWindsurf, disconnectWindsurf } from "../install/windsurf.js";
|
|
61
|
+
import { connectPi, disconnectPi, piSettingsPath } from "../install/pi.js";
|
|
59
62
|
import { runSetup } from "../install/setup.js";
|
|
60
63
|
import { runParity } from "./classify-parity.js";
|
|
61
64
|
import { reembedCorpus } from "../core/embedding/embed-backfill.js";
|
|
@@ -67,11 +70,11 @@ import { isAgentLoaded, isBenignBootoutError } from "./launchctl-helpers.js";
|
|
|
67
70
|
import { DAEMON_PKILL_PATTERN, planRestart } from "./restart-helpers.js";
|
|
68
71
|
import { applyEnvAssignment } from "./config-env.js";
|
|
69
72
|
import { adapterFromSource } from "../core/adapters/from-source.js";
|
|
70
|
-
import { scanUsefulHits } from "../core/recall/useful-scan.js";
|
|
71
73
|
import { runDigest } from "./digest.js";
|
|
72
74
|
const __filename = fileURLToPath(import.meta.url);
|
|
73
75
|
const __dirname = dirname(__filename);
|
|
74
76
|
const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
|
|
77
|
+
const PG_MIGRATIONS_DIR = join(fileURLToPath(new URL(".", import.meta.url)), "../../migrations/pg");
|
|
75
78
|
const UI_DIST = resolve(__dirname, "../../dist/ui");
|
|
76
79
|
const DEFAULT_DB_PATH = resolve(homedir(), ".nlm/canonical.sqlite");
|
|
77
80
|
const DEFAULT_PORT = 3940;
|
|
@@ -102,15 +105,16 @@ function buildClassifier() {
|
|
|
102
105
|
?? (provider === "ollama" ? "phi4-mini:latest" : "deepseek-v4-flash");
|
|
103
106
|
return new ClassifierBox({ provider, model, ollamaUrl: ollamaUrl() });
|
|
104
107
|
}
|
|
105
|
-
function buildAdapters(sources) {
|
|
108
|
+
async function buildAdapters(sources) {
|
|
106
109
|
// Sources table is the source of truth. Each enabled row maps to one
|
|
107
110
|
// adapter via adapterFromSource(). Detection still gates registration —
|
|
108
111
|
// a row pointing at a missing dir won't poll. NLM_ADAPTERS keeps working
|
|
109
112
|
// as a name-based filter for forcing a subset during dev.
|
|
110
113
|
const explicit = process.env["NLM_ADAPTERS"];
|
|
111
114
|
const allowed = explicit ? new Set(explicit.split(",").map((s) => s.trim())) : null;
|
|
115
|
+
const rows = await sources.list();
|
|
112
116
|
const out = [];
|
|
113
|
-
for (const row of
|
|
117
|
+
for (const row of rows) {
|
|
114
118
|
if (!row.enabled)
|
|
115
119
|
continue;
|
|
116
120
|
const adapter = adapterFromSource(row);
|
|
@@ -124,7 +128,16 @@ function buildAdapters(sources) {
|
|
|
124
128
|
}
|
|
125
129
|
return out;
|
|
126
130
|
}
|
|
127
|
-
function
|
|
131
|
+
async function buildStorage(path) {
|
|
132
|
+
const pgUrl = process.env["NLM_PG_URL"];
|
|
133
|
+
if (pgUrl) {
|
|
134
|
+
const storage = PgStorage.create({ connectionString: pgUrl, migrationsDir: PG_MIGRATIONS_DIR });
|
|
135
|
+
await storage.init();
|
|
136
|
+
return storage;
|
|
137
|
+
}
|
|
138
|
+
return SqliteStorage.create({ dbPath: path, migrationsDir: MIGRATIONS_DIR });
|
|
139
|
+
}
|
|
140
|
+
async function buildStack() {
|
|
128
141
|
// Load .env before any registry seeds so secrets carried in env vars
|
|
129
142
|
// (DEEPSEEK_API_KEY today; OPENAI_API_KEY etc. tomorrow) bridge into
|
|
130
143
|
// the providers table on first boot under launchd.
|
|
@@ -137,24 +150,29 @@ function buildStack() {
|
|
|
137
150
|
if (restored.archivedTo)
|
|
138
151
|
console.error(` previous db archived at ${restored.archivedTo}`);
|
|
139
152
|
}
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
migrationsDir: MIGRATIONS_DIR,
|
|
143
|
-
});
|
|
153
|
+
const storage = await buildStorage(dbPath());
|
|
154
|
+
const store = storage.sessions;
|
|
144
155
|
// FactStore shares the SessionStore's connection so session+facts ingest
|
|
145
156
|
// can commit in one transaction. Phase B.1 wires it in; no callers yet.
|
|
146
|
-
const facts =
|
|
147
|
-
|
|
148
|
-
sources
|
|
149
|
-
|
|
150
|
-
|
|
157
|
+
const facts = storage.facts;
|
|
158
|
+
// TODO(#215a): replace storage.rawDb() with port methods
|
|
159
|
+
const sources = storage instanceof PgStorage
|
|
160
|
+
? new PgSourceRegistry(storage.pgPool())
|
|
161
|
+
: new SourceRegistry(storage.rawDb());
|
|
162
|
+
await sources.seedDefaults();
|
|
163
|
+
// TODO(#215a): replace storage.rawDb() with port methods
|
|
164
|
+
const providers = storage instanceof PgStorage
|
|
165
|
+
? new PgProviderRegistry(storage.pgPool())
|
|
166
|
+
: new ProviderRegistry(storage.rawDb());
|
|
167
|
+
if (providers instanceof ProviderRegistry)
|
|
168
|
+
providers.seedDefaults();
|
|
151
169
|
// Recall only uses embed(). Embeddings live on Ollama; DeepSeek doesn't
|
|
152
170
|
// expose them. Classifier is wired separately for Phase D ingest.
|
|
153
171
|
const embedder = new OllamaClient({ baseUrl: ollamaUrl() });
|
|
154
172
|
const classifier = buildClassifier();
|
|
155
173
|
const recall = new RecallService({ store, llm: embedder });
|
|
156
174
|
const factRecall = new FactRecallService({ factStore: facts, llm: embedder });
|
|
157
|
-
return { store, facts, sources, providers, recall, factRecall, embedder, classifier };
|
|
175
|
+
return { storage, store, facts, sources, providers, recall, factRecall, embedder, classifier };
|
|
158
176
|
}
|
|
159
177
|
const program = new Command();
|
|
160
178
|
program
|
|
@@ -174,7 +192,7 @@ program
|
|
|
174
192
|
// non-browser callers. Idempotent: re-reads persisted token first.
|
|
175
193
|
autoloadEnv();
|
|
176
194
|
ensureMcpToken();
|
|
177
|
-
const { store, facts, sources, providers, recall, factRecall, embedder, classifier } = buildStack();
|
|
195
|
+
const { storage, store, facts, sources, providers, recall, factRecall, embedder, classifier } = await buildStack();
|
|
178
196
|
const { existsSync } = await import("node:fs");
|
|
179
197
|
const hasMcpToken = Boolean(process.env["NLM_MCP_TOKEN"]);
|
|
180
198
|
const app = createApp({
|
|
@@ -187,7 +205,15 @@ program
|
|
|
187
205
|
classifier,
|
|
188
206
|
sources,
|
|
189
207
|
providers,
|
|
190
|
-
|
|
208
|
+
// TODO(#215a): PgStorage ingest port; cast until then
|
|
209
|
+
...(!(storage instanceof PgStorage) ? {
|
|
210
|
+
ingest: {
|
|
211
|
+
classifier,
|
|
212
|
+
embedder,
|
|
213
|
+
store: store,
|
|
214
|
+
...(facts ? { factStore: facts } : {}),
|
|
215
|
+
},
|
|
216
|
+
} : {}),
|
|
191
217
|
embedderInfo: { provider: "ollama", model: "nomic-embed-text", dims: 768 },
|
|
192
218
|
...(existsSync(UI_DIST) ? { uiDist: UI_DIST } : {}),
|
|
193
219
|
// Wire POST /mcp only when NLM_MCP_TOKEN is present. Absent = route never
|
|
@@ -218,23 +244,29 @@ program
|
|
|
218
244
|
// Keep the SQLite WAL bounded. WAL mode is on but nothing else
|
|
219
245
|
// checkpoints it; under continuous readers it grows without limit
|
|
220
246
|
// (it had reached 38 MB), which slows every read. Drain once at boot,
|
|
221
|
-
// then every 5 minutes.
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
247
|
+
// then every 5 minutes. Skip entirely when using PgStorage (no WAL).
|
|
248
|
+
const checkpointTimer = !(storage instanceof PgStorage)
|
|
249
|
+
? (() => {
|
|
250
|
+
const WAL_CHECKPOINT_INTERVAL_MS = 5 * 60_000;
|
|
251
|
+
const sqliteStore = store;
|
|
252
|
+
try {
|
|
253
|
+
sqliteStore.checkpoint();
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
// Boot checkpoint can lose a race with readers — the interval retries.
|
|
257
|
+
}
|
|
258
|
+
const t = setInterval(() => {
|
|
259
|
+
try {
|
|
260
|
+
sqliteStore.checkpoint();
|
|
261
|
+
}
|
|
262
|
+
catch {
|
|
263
|
+
// Checkpoint contention — the next tick retries.
|
|
264
|
+
}
|
|
265
|
+
}, WAL_CHECKPOINT_INTERVAL_MS);
|
|
266
|
+
t.unref();
|
|
267
|
+
return t;
|
|
268
|
+
})()
|
|
269
|
+
: null;
|
|
238
270
|
// Memo sweep runs independently of the transcript scheduler — it's the
|
|
239
271
|
// backstop for SessionEnd hook unreliability (crashes, kill -9, IDE
|
|
240
272
|
// force-close don't fire SessionEnd, so memo files would otherwise
|
|
@@ -242,27 +274,29 @@ program
|
|
|
242
274
|
const memoSweep = new MemoSweepScheduler();
|
|
243
275
|
memoSweep.start();
|
|
244
276
|
console.error(" memo sweep: dormant cleanup every 5m (threshold 24h)");
|
|
245
|
-
if (opts.scheduler !== false) {
|
|
246
|
-
const adapters = buildAdapters(sources);
|
|
277
|
+
if (opts.scheduler !== false && !(storage instanceof PgStorage)) {
|
|
278
|
+
const adapters = await buildAdapters(sources);
|
|
247
279
|
if (adapters.length === 0) {
|
|
248
280
|
console.error(" scheduler: no adapters detected (set NLM_ADAPTERS to force-enable)");
|
|
249
281
|
}
|
|
250
282
|
else {
|
|
251
283
|
const scheduler = new ScanScheduler({
|
|
252
|
-
|
|
284
|
+
// TODO(#215a): PgStorage scheduler port; SQLite-only until then
|
|
285
|
+
store: store,
|
|
253
286
|
adapters,
|
|
254
287
|
classifier,
|
|
255
288
|
embedder,
|
|
256
|
-
factStore: facts,
|
|
289
|
+
factStore: facts ?? null,
|
|
257
290
|
intervalMs: opts.intervalMin * 60_000,
|
|
258
291
|
});
|
|
259
292
|
scheduler.start();
|
|
260
293
|
console.error(` scheduler: ${adapters.map((a) => a.name).join(", ")} every ${opts.intervalMin}m`);
|
|
261
|
-
const shutdown = () => {
|
|
262
|
-
|
|
294
|
+
const shutdown = async () => {
|
|
295
|
+
if (checkpointTimer)
|
|
296
|
+
clearInterval(checkpointTimer);
|
|
263
297
|
scheduler.stop();
|
|
264
298
|
memoSweep.stop();
|
|
265
|
-
|
|
299
|
+
await storage.close();
|
|
266
300
|
process.exit(0);
|
|
267
301
|
};
|
|
268
302
|
process.on("SIGINT", shutdown);
|
|
@@ -273,14 +307,15 @@ program
|
|
|
273
307
|
program
|
|
274
308
|
.command("migrate")
|
|
275
309
|
.description("Run pending migrations against the canonical SQLite")
|
|
276
|
-
.action(() => {
|
|
310
|
+
.action(async () => {
|
|
277
311
|
// SqliteSessionStore's constructor loads sqlite-vec and runs migrations.
|
|
278
312
|
// Opening + closing is the whole operation.
|
|
279
|
-
const
|
|
313
|
+
const storage = SqliteStorage.create({
|
|
280
314
|
dbPath: dbPath(),
|
|
281
315
|
migrationsDir: MIGRATIONS_DIR,
|
|
282
316
|
});
|
|
283
|
-
|
|
317
|
+
await storage.init();
|
|
318
|
+
await storage.close();
|
|
284
319
|
console.error(`nlm-memory: migrations applied at ${dbPath()}`);
|
|
285
320
|
});
|
|
286
321
|
program
|
|
@@ -292,7 +327,7 @@ program
|
|
|
292
327
|
.option("-m, --mode <mode>", "keyword|semantic|hybrid", "keyword")
|
|
293
328
|
.option("-l, --limit <n>", "max results", (v) => Number.parseInt(v, 10), 10)
|
|
294
329
|
.action(async (query, opts) => {
|
|
295
|
-
const {
|
|
330
|
+
const { storage, recall } = await buildStack();
|
|
296
331
|
try {
|
|
297
332
|
const result = await recall.search({
|
|
298
333
|
query,
|
|
@@ -304,7 +339,7 @@ program
|
|
|
304
339
|
process.stdout.write(JSON.stringify(result, null, 2) + "\n");
|
|
305
340
|
}
|
|
306
341
|
finally {
|
|
307
|
-
|
|
342
|
+
await storage.close();
|
|
308
343
|
}
|
|
309
344
|
});
|
|
310
345
|
program
|
|
@@ -376,10 +411,11 @@ program
|
|
|
376
411
|
.option("--no-embed", "skip per-fact embedding (faster but disables semantic recall)")
|
|
377
412
|
.option("-v, --verbose", "per-session progress on stderr")
|
|
378
413
|
.action(async (opts) => {
|
|
379
|
-
const { store, facts, embedder, classifier } = buildStack();
|
|
414
|
+
const { storage, store, facts, embedder, classifier } = await buildStack();
|
|
380
415
|
try {
|
|
381
416
|
const report = await backfillFacts({
|
|
382
|
-
|
|
417
|
+
// TODO(#215a): PgStorage backfill port; SQLite-only until then
|
|
418
|
+
store: store,
|
|
383
419
|
factStore: facts,
|
|
384
420
|
classifier,
|
|
385
421
|
embedder: opts.embed === false ? null : embedder,
|
|
@@ -400,7 +436,7 @@ program
|
|
|
400
436
|
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
|
|
401
437
|
}
|
|
402
438
|
finally {
|
|
403
|
-
|
|
439
|
+
await storage.close();
|
|
404
440
|
}
|
|
405
441
|
});
|
|
406
442
|
program
|
|
@@ -422,7 +458,7 @@ program
|
|
|
422
458
|
.command("mcp")
|
|
423
459
|
.description("Run as an MCP stdio server (for ~/.mcp.json)")
|
|
424
460
|
.action(async () => {
|
|
425
|
-
const { recall, store, facts, factRecall } = buildStack();
|
|
461
|
+
const { recall, store, facts, factRecall } = await buildStack();
|
|
426
462
|
const server = createMcpServer({ recall, store, factStore: facts, factRecall });
|
|
427
463
|
const transport = new StdioServerTransport();
|
|
428
464
|
await server.connect(transport);
|
|
@@ -968,10 +1004,12 @@ connect
|
|
|
968
1004
|
.description("Register Cursor as an nlm source (reads state.vscdb directly — no files installed)")
|
|
969
1005
|
.option("--db-path <path>", "override path to globalStorage/state.vscdb")
|
|
970
1006
|
.option("--dry-run", "print what would happen without changing files")
|
|
971
|
-
.action((opts) => {
|
|
972
|
-
const
|
|
1007
|
+
.action(async (opts) => {
|
|
1008
|
+
const storage = SqliteStorage.create({ dbPath: dbPath(), migrationsDir: MIGRATIONS_DIR });
|
|
1009
|
+
await storage.init();
|
|
973
1010
|
try {
|
|
974
|
-
|
|
1011
|
+
// TODO(#215a): replace storage.rawDb() with port methods
|
|
1012
|
+
const registry = new SourceRegistry(storage.rawDb());
|
|
975
1013
|
const report = connectCursor(registry, {
|
|
976
1014
|
...(opts.dbPath ? { dbPath: opts.dbPath } : {}),
|
|
977
1015
|
dryRun: Boolean(opts.dryRun),
|
|
@@ -984,7 +1022,7 @@ connect
|
|
|
984
1022
|
console.error(`nlm: Cursor source ${report.action} → ${report.adapterDbPath}${suffix}`);
|
|
985
1023
|
}
|
|
986
1024
|
finally {
|
|
987
|
-
|
|
1025
|
+
await storage.close();
|
|
988
1026
|
}
|
|
989
1027
|
});
|
|
990
1028
|
connect
|
|
@@ -992,10 +1030,12 @@ connect
|
|
|
992
1030
|
.description("Register Windsurf as an nlm source (reads state.vscdb files directly — no files installed)")
|
|
993
1031
|
.option("--user-dir <path>", "override path to Windsurf User directory")
|
|
994
1032
|
.option("--dry-run", "print what would happen without changing files")
|
|
995
|
-
.action((opts) => {
|
|
996
|
-
const
|
|
1033
|
+
.action(async (opts) => {
|
|
1034
|
+
const storage = SqliteStorage.create({ dbPath: dbPath(), migrationsDir: MIGRATIONS_DIR });
|
|
1035
|
+
await storage.init();
|
|
997
1036
|
try {
|
|
998
|
-
|
|
1037
|
+
// TODO(#215a): replace storage.rawDb() with port methods
|
|
1038
|
+
const registry = new SourceRegistry(storage.rawDb());
|
|
999
1039
|
const report = connectWindsurf(registry, {
|
|
1000
1040
|
...(opts.userDir ? { userDir: opts.userDir } : {}),
|
|
1001
1041
|
dryRun: Boolean(opts.dryRun),
|
|
@@ -1008,9 +1048,31 @@ connect
|
|
|
1008
1048
|
console.error(`nlm: Windsurf source ${report.action} → ${report.userDir}${suffix}`);
|
|
1009
1049
|
}
|
|
1010
1050
|
finally {
|
|
1011
|
-
|
|
1051
|
+
await storage.close();
|
|
1012
1052
|
}
|
|
1013
1053
|
});
|
|
1054
|
+
connect
|
|
1055
|
+
.command("pi")
|
|
1056
|
+
.description("Register the nlm-memory prompt-recall extension in ~/.pi/agent/settings.json")
|
|
1057
|
+
.option("--dry-run", "print what would happen without changing files")
|
|
1058
|
+
.action((opts) => {
|
|
1059
|
+
const pluginDir = join(REPO_ROOT, "nlm");
|
|
1060
|
+
const report = connectPi({ pluginDir, dryRun: Boolean(opts.dryRun) });
|
|
1061
|
+
if (opts.dryRun) {
|
|
1062
|
+
const verb = report.alreadyPresent ? "already present in" : "append to";
|
|
1063
|
+
console.error(`nlm connect pi (dry run): ${verb} packages[] in ${report.settingsPath} → ${pluginDir}`);
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
if (report.alreadyPresent) {
|
|
1067
|
+
console.error(`nlm: pi extension already registered → ${report.pluginDir}`);
|
|
1068
|
+
}
|
|
1069
|
+
else {
|
|
1070
|
+
console.error(`nlm: pi extension registered → ${report.settingsPath}`);
|
|
1071
|
+
console.error(` Packages entry: ${report.pluginDir}`);
|
|
1072
|
+
}
|
|
1073
|
+
console.error(" Restart pi to activate the prompt-recall hook.");
|
|
1074
|
+
console.error(" Set NLM_HOOK_MODE=live in ~/.nlm/.env to flip from shadow → live.");
|
|
1075
|
+
});
|
|
1014
1076
|
const disconnect = program
|
|
1015
1077
|
.command("disconnect")
|
|
1016
1078
|
.description("Disconnect nlm-memory from an AI coding runtime");
|
|
@@ -1104,10 +1166,12 @@ disconnect
|
|
|
1104
1166
|
.command("cursor")
|
|
1105
1167
|
.description("Disable the Cursor source in the nlm registry (leaves Cursor untouched)")
|
|
1106
1168
|
.option("--dry-run", "print what would happen without changing files")
|
|
1107
|
-
.action((opts) => {
|
|
1108
|
-
const
|
|
1169
|
+
.action(async (opts) => {
|
|
1170
|
+
const storage = SqliteStorage.create({ dbPath: dbPath(), migrationsDir: MIGRATIONS_DIR });
|
|
1171
|
+
await storage.init();
|
|
1109
1172
|
try {
|
|
1110
|
-
|
|
1173
|
+
// TODO(#215a): replace storage.rawDb() with port methods
|
|
1174
|
+
const registry = new SourceRegistry(storage.rawDb());
|
|
1111
1175
|
const report = disconnectCursor(registry, { dryRun: Boolean(opts.dryRun) });
|
|
1112
1176
|
if (opts.dryRun) {
|
|
1113
1177
|
console.error("nlm disconnect cursor (dry run): disable Cursor source in registry");
|
|
@@ -1118,17 +1182,19 @@ disconnect
|
|
|
1118
1182
|
: "nlm: no Cursor source found in registry");
|
|
1119
1183
|
}
|
|
1120
1184
|
finally {
|
|
1121
|
-
|
|
1185
|
+
await storage.close();
|
|
1122
1186
|
}
|
|
1123
1187
|
});
|
|
1124
1188
|
disconnect
|
|
1125
1189
|
.command("windsurf")
|
|
1126
1190
|
.description("Disable the Windsurf source in the nlm registry (leaves Windsurf untouched)")
|
|
1127
1191
|
.option("--dry-run", "print what would happen without changing files")
|
|
1128
|
-
.action((opts) => {
|
|
1129
|
-
const
|
|
1192
|
+
.action(async (opts) => {
|
|
1193
|
+
const storage = SqliteStorage.create({ dbPath: dbPath(), migrationsDir: MIGRATIONS_DIR });
|
|
1194
|
+
await storage.init();
|
|
1130
1195
|
try {
|
|
1131
|
-
|
|
1196
|
+
// TODO(#215a): replace storage.rawDb() with port methods
|
|
1197
|
+
const registry = new SourceRegistry(storage.rawDb());
|
|
1132
1198
|
const report = disconnectWindsurf(registry, { dryRun: Boolean(opts.dryRun) });
|
|
1133
1199
|
if (opts.dryRun) {
|
|
1134
1200
|
console.error("nlm disconnect windsurf (dry run): disable Windsurf source in registry");
|
|
@@ -1139,8 +1205,22 @@ disconnect
|
|
|
1139
1205
|
: "nlm: no Windsurf source found in registry");
|
|
1140
1206
|
}
|
|
1141
1207
|
finally {
|
|
1142
|
-
|
|
1208
|
+
await storage.close();
|
|
1209
|
+
}
|
|
1210
|
+
});
|
|
1211
|
+
disconnect
|
|
1212
|
+
.command("pi")
|
|
1213
|
+
.description("Remove the nlm-memory pi extension from ~/.pi/agent/settings.json")
|
|
1214
|
+
.option("--dry-run", "print what would happen without changing files")
|
|
1215
|
+
.action((opts) => {
|
|
1216
|
+
const report = disconnectPi({ dryRun: Boolean(opts.dryRun) });
|
|
1217
|
+
if (opts.dryRun) {
|
|
1218
|
+
console.error(`nlm disconnect pi (dry run): strip nlm (and legacy plugin-pi) from packages[] in ${piSettingsPath()}`);
|
|
1219
|
+
return;
|
|
1143
1220
|
}
|
|
1221
|
+
console.error(report.removed
|
|
1222
|
+
? `nlm: pi extension removed → ${report.settingsPath}`
|
|
1223
|
+
: `nlm: no nlm pi extension found in ${report.settingsPath}`);
|
|
1144
1224
|
});
|
|
1145
1225
|
program
|
|
1146
1226
|
.command("setup")
|
|
@@ -1166,19 +1246,6 @@ program
|
|
|
1166
1246
|
buildHookCommand,
|
|
1167
1247
|
});
|
|
1168
1248
|
});
|
|
1169
|
-
program
|
|
1170
|
-
.command("useful-scan")
|
|
1171
|
-
.description("Scan hook log for useful recall hits; writes to ~/.nlm/useful-hit-log.jsonl")
|
|
1172
|
-
.option("-d, --days <n>", "rolling window in days", (v) => Number.parseInt(v, 10), 1)
|
|
1173
|
-
.option("--dry-run", "compute without writing to disk")
|
|
1174
|
-
.action(async (opts) => {
|
|
1175
|
-
const result = await scanUsefulHits({ days: opts.days, ...(opts.dryRun ? { dryRun: true } : {}) });
|
|
1176
|
-
const rate = result.measurable === 0
|
|
1177
|
-
? "no measurable entries"
|
|
1178
|
-
: `${result.useful}/${result.measurable} useful (${Math.round((result.useful / result.measurable) * 100)}%)`;
|
|
1179
|
-
console.error(`nlm useful-scan: scanned ${result.total} recalls in the last ${opts.days}d — ${rate}` +
|
|
1180
|
-
(opts.dryRun ? " (dry-run)" : `, ${result.appended} appended`));
|
|
1181
|
-
});
|
|
1182
1249
|
program
|
|
1183
1250
|
.command("digest")
|
|
1184
1251
|
.description("Compose a daily-activity digest from the running daemon (optionally post to Telegram)")
|