@jbarket/loomai 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/CHANGELOG.md +129 -1
  2. package/README.md +115 -0
  3. package/dist/backends/dedup.d.ts +23 -0
  4. package/dist/backends/dedup.js +100 -0
  5. package/dist/backends/embedding-runtime.js +38 -9
  6. package/dist/backends/episodes.js +1 -1
  7. package/dist/backends/fastembed.js +30 -4
  8. package/dist/backends/migrations.js +200 -0
  9. package/dist/backends/mmr.d.ts +8 -0
  10. package/dist/backends/mmr.js +18 -4
  11. package/dist/backends/people.d.ts +192 -0
  12. package/dist/backends/people.js +824 -0
  13. package/dist/backends/proposals.js +1 -1
  14. package/dist/backends/recorder-health.d.ts +34 -0
  15. package/dist/backends/recorder-health.js +117 -0
  16. package/dist/backends/salience.d.ts +39 -0
  17. package/dist/backends/salience.js +52 -4
  18. package/dist/backends/scope.d.ts +56 -0
  19. package/dist/backends/scope.js +75 -0
  20. package/dist/backends/sqlite-knowledge.d.ts +24 -4
  21. package/dist/backends/sqlite-knowledge.js +253 -91
  22. package/dist/backends/sqlite-vec.d.ts +32 -1
  23. package/dist/backends/sqlite-vec.js +394 -75
  24. package/dist/backends/types.d.ts +203 -3
  25. package/dist/categories.d.ts +2 -0
  26. package/dist/categories.js +2 -0
  27. package/dist/cli/args.d.ts +16 -1
  28. package/dist/cli/args.js +45 -1
  29. package/dist/cli/index.d.ts +7 -0
  30. package/dist/cli/index.js +81 -3
  31. package/dist/cli/knowledge.js +4 -1
  32. package/dist/cli/memory.js +2 -2
  33. package/dist/cli/person.d.ts +2 -0
  34. package/dist/cli/person.js +116 -0
  35. package/dist/cli/pieces.d.ts +24 -0
  36. package/dist/cli/pieces.js +139 -0
  37. package/dist/cli/recall.js +2 -1
  38. package/dist/cli/remember.js +10 -0
  39. package/dist/cli/remote.d.ts +36 -0
  40. package/dist/cli/remote.js +173 -0
  41. package/dist/cli/serve.d.ts +7 -0
  42. package/dist/cli/serve.js +94 -5
  43. package/dist/cli/subcommands.d.ts +1 -1
  44. package/dist/cli/subcommands.js +2 -2
  45. package/dist/cli/update.js +30 -11
  46. package/dist/clients.js +7 -4
  47. package/dist/config.d.ts +14 -0
  48. package/dist/config.js +25 -2
  49. package/dist/index.js +28 -8
  50. package/dist/injection/harnesses.js +9 -3
  51. package/dist/install/harnesses.js +7 -1
  52. package/dist/server.d.ts +8 -1
  53. package/dist/server.js +461 -374
  54. package/dist/tools/dossier.js +5 -1
  55. package/dist/tools/eurorack-import.js +3 -0
  56. package/dist/tools/identity.d.ts +2 -0
  57. package/dist/tools/identity.js +35 -2
  58. package/dist/tools/knowledge-history.d.ts +6 -1
  59. package/dist/tools/knowledge-history.js +26 -8
  60. package/dist/tools/knowledge-maintain.d.ts +5 -0
  61. package/dist/tools/knowledge-maintain.js +92 -5
  62. package/dist/tools/knowledge-recall.d.ts +7 -1
  63. package/dist/tools/knowledge-recall.js +53 -3
  64. package/dist/tools/knowledge-verify.d.ts +11 -1
  65. package/dist/tools/knowledge-verify.js +17 -4
  66. package/dist/tools/knowledge-write.d.ts +8 -1
  67. package/dist/tools/knowledge-write.js +27 -9
  68. package/dist/tools/memory-history.d.ts +5 -0
  69. package/dist/tools/memory-history.js +52 -0
  70. package/dist/tools/memory-list.js +2 -1
  71. package/dist/tools/memory-merge.d.ts +2 -0
  72. package/dist/tools/memory-merge.js +40 -0
  73. package/dist/tools/memory-revision-restore.d.ts +5 -0
  74. package/dist/tools/memory-revision-restore.js +33 -0
  75. package/dist/tools/person.d.ts +32 -0
  76. package/dist/tools/person.js +111 -0
  77. package/dist/tools/prune.js +1 -1
  78. package/dist/tools/recall.js +7 -1
  79. package/dist/tools/remember.js +18 -2
  80. package/dist/tools/update.js +5 -1
  81. package/dist/transport/cli-endpoint.d.ts +13 -0
  82. package/dist/transport/cli-endpoint.js +97 -0
  83. package/dist/transport/guards.d.ts +14 -0
  84. package/dist/transport/guards.js +26 -0
  85. package/dist/transport/http-server.d.ts +28 -1
  86. package/dist/transport/http-server.js +156 -18
  87. package/dist/transport/knowledge-server.d.ts +11 -0
  88. package/dist/transport/knowledge-server.js +30 -0
  89. package/dist/transport/knowledge-tools.d.ts +50 -0
  90. package/dist/transport/knowledge-tools.js +290 -0
  91. package/dist/transport/mcp-app-hello.d.ts +1 -1
  92. package/dist/widgets/display-modes.html +62 -56
  93. package/dist/widgets/hello.html +62 -56
  94. package/dist/widgets/proposals.html +62 -56
  95. package/dist/widgets/top-of-mind.html +60 -54
  96. package/package.json +7 -5
package/CHANGELOG.md CHANGED
@@ -12,6 +12,132 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
12
12
 
13
13
  ## [Unreleased]
14
14
 
15
+ ## [0.7.0] - 2026-10-24
16
+
17
+ ### Added
18
+
19
+ - **Pieces: `loom <piece>` dispatches to `loom-<piece>` on PATH, and `loom
20
+ pieces` lists them.** loom's command namespace now reaches separate,
21
+ swappable extensions (tasks, secrets, a wake clock, …) the way `git` reaches
22
+ `git-<cmd>`. loom never links them, and they never write loom's stores. Core
23
+ commands always win, piece names are validated before PATH is searched,
24
+ loom's env passes through unchanged, and the piece's exit code comes back.
25
+ The tiers, the rules (alone, composes without requiring, a contract per
26
+ role, no secrets in argv) and how pieces meet through named seams are in
27
+ `docs/extensions.md`.
28
+
29
+ ## [0.6.0] - 2026-10-24
30
+
31
+ ### Added
32
+
33
+ - **CLI remote mode: `--url` / `LOOM_URL`, and `POST /cli` on loom-http.**
34
+ The store verbs (`wake`, `recall`, `remember`, `update`, `forget`, `memory`,
35
+ `knowledge`, `person`) can run against a daemon instead of a local stack,
36
+ for hooks and scripts on a host or in a container where loom lives
37
+ elsewhere. The CLI ships argv and stdin to `POST /cli`; the daemon runs the
38
+ same CLI on its own stack and returns stdout, stderr and the exit code, so
39
+ output and `--json` match a local run by construction. The bearer token
40
+ comes from `LOOM_TOKEN_FILE`. The endpoint is full-scope only, needs
41
+ `LOOM_BEARER_TOKEN` set, refuses browser requests (an `Origin` header or a
42
+ non-JSON body), and sits behind the same bearer gate and payload cap as
43
+ MCP. It takes only those verbs (minus `memory recompute-salience` and
44
+ `memory recall-stats`, which have no MCP tool), refuses `--context-dir` and
45
+ `--url`, and passes only `LOOM_CLIENT`/`LOOM_MODEL` through from the
46
+ caller's env.
47
+
48
+ - **Person cards: `people` + `person_revisions`, eight `person_*` tools,
49
+ `loom person show|list`, and identity's `people` param.** A card is the
50
+ condensed current state of one human the agent works with: `where` fields
51
+ carrying a freshness marker (`[ok]`, `[unverified]`, `[STALE <n>d]`), `how`
52
+ fields (time zone, mode, who to loop in), descriptive `may` entries each with
53
+ a source, and a short agent-written summary. One value per field,
54
+ overwritten in place; every stored change snapshots the previous row (newest
55
+ 10 kept), and every mutation is one `BEGIN IMMEDIATE` transaction. Cards are
56
+ not an authorization surface: the harness's gating names (`tier`, `field`,
57
+ `projects`, `scope`, `trust`) are rejected anywhere on a card.
58
+ `identity({people: [slug, ...]})` adds an `# In the room` block after the
59
+ tape (≤ 4 cards, 6,500 chars, best-effort). The dossier now says worker
60
+ bodies must not write cards. Design and acceptance criteria:
61
+ `docs/specs/person-cards.md` (criteria 1–12; the misfile heuristic, 13, is
62
+ still to come).
63
+
64
+ - **Knowledge actor attribution: `pages.author`, `page_revisions.actor`, and a
65
+ `verifications` table (page_id, verifier, verified_at, outcome,
66
+ citation_checked).** Previously the only attribution field was
67
+ `created_by` — free text, meaning "who owns this `ours/` artifact," not
68
+ "who actually wrote or verified it" — so there was no way to tell who wrote
69
+ a page's first version, who performed a given body revision, or who has
70
+ verified a page and found what. `pages.author` is stamped once at creation
71
+ and never changed by later upserts; `page_revisions.actor` is stamped on
72
+ every body-replacing write and every history-restore; `verifications` is an
73
+ attributed audit trail additive to the existing `pages.verified_at` stamp
74
+ (which stays as the fast single-column freshness field everything else
75
+ sorts/filters on). All three are resolved server-side, never a free-text
76
+ tool argument, so no caller can claim another identity's authorship or
77
+ verification: MCP calls use the connection's identity from
78
+ `registerKnowledgeTools` ('art' on the main server, the bearer-resolved
79
+ identity on the knowledge-only service); the CLI and eurorack import use
80
+ `resolveIdentityName()` (`LOOM_IDENTITY` env override, else the context
81
+ dir's real basename).
82
+ `knowledge_verify` gains `outcome` ('confirmed' | 'stale' | 'corrected') and
83
+ `citation_checked` parameters; `knowledge_history`'s listing now shows who
84
+ performed each revision and surfaces the page's verification history. (t-675)
85
+ An outcome of 'stale' is a failed verification: it is recorded in
86
+ `verifications` but does NOT advance `pages.verified_at`, so a known-wrong
87
+ page keeps its place at the front of the stale-first queue; the result's
88
+ new `stamped` flag (and the tool output) says whether the stamp moved.
89
+
90
+ - **`MemoryMatch` carries `similarity` alongside `relevance`, and `scope`
91
+ alongside `project`.** `relevance` is now the post-MMR score that actually
92
+ decided a result's rank — equal to `similarity` (raw cosine) unless MMR
93
+ re-ranking traded a pick off against ones already selected, in which case
94
+ the two now diverge and both are visible (modeled on MemoriLabs/Memori's
95
+ `FactSearchResult.{similarity, rank_score}` split). Separately, every
96
+ memory now has an effective `scope` ('project' or 'global'): `recall` and
97
+ `memory similar` exclude a project-scoped memory tagged to a *different*
98
+ project than the one passed via `--project`/`project`, while global-scoped
99
+ memories (category default for `user`/`reference`, any untagged memory, or
100
+ an explicit `scope: "global"` at write time) still come through regardless.
101
+ Closes the gap where a preference learned on one project could surface
102
+ while working on an unrelated one. `remember` (MCP tool and `loom remember
103
+ --scope`) takes an explicit `project | global` override for the categories
104
+ where the category default can't tell (`self`, `feedback`) — e.g. a
105
+ security/git/tool-workflow lesson learned in one project but true
106
+ everywhere. (t-337)
107
+
108
+ ### Fixed
109
+
110
+ - **All 34 MCP tools now declare `ToolAnnotations`** (`readOnlyHint`,
111
+ `destructiveHint`, `idempotentHint`, `openWorldHint`), so a client gating
112
+ approval on these hints (e.g. Codex under `approval: never`) can tell a
113
+ read from a write instead of refusing every call uniformly. Hints follow
114
+ actual behavior, not tool-name conventions: `recall`/`knowledge_recall`
115
+ are NOT read-only (they stamp `last_accessed`/`hit_count`, which
116
+ `knowledge_maintain` reads to find cold pages); `knowledge_history` and
117
+ `memory_revision_restore` are destructive in their restore mode even
118
+ though the displaced body is snapshotted first. `update`,
119
+ `knowledge_write` and `knowledge_move` are NOT idempotent: a repeated
120
+ `observation` bumps the evidence count again, a repeated append stores
121
+ the body twice, and a repeated prefix move into a child of itself
122
+ (`music` → `music/gear`) nests again. `openWorldHint` is
123
+ `false` everywhere — loom has no outbound network calls. This does not
124
+ by itself prove any client relaxes its approval gate on these hints; that
125
+ needs a live `tools/list` exchange against an updated server (tracked
126
+ separately). (t-415)
127
+
128
+ - **`--context-dir` (and the other global flags) now work in front of the
129
+ subcommand, as the help text has always claimed.** `loom --context-dir DIR
130
+ knowledge write …` used to route to the stdio MCP server instead of the CLI:
131
+ the entry point dispatched on `argv[2]`, which was `--context-dir`, not a known
132
+ verb. The server read an empty stdin and exited 0, so a *write* verb reported
133
+ success having never entered the write path — and never even reached argument
134
+ validation, exiting 0 on an invocation that the working flag order rejects with
135
+ exit 1. Measured against a disposable context dir: no `knowledge.db` was created
136
+ and no row was written. Both entry points now resolve the subcommand past any
137
+ leading global flags, skipping flag values in pairs so `--context-dir recall`
138
+ still names a directory rather than invoking `recall`. Bare
139
+ `loom --context-dir DIR` with no subcommand remains the MCP path. (t-665)
140
+
15
141
  ## [0.5.0] - 2026-09-04
16
142
 
17
143
  ### Changed
@@ -462,7 +588,9 @@ Initial public release.
462
588
  stack and all added external-service dependencies or operational
463
589
  overhead.
464
590
 
465
- [Unreleased]: https://github.com/sleepunit-agents/loom/compare/v0.5.0...HEAD
591
+ [Unreleased]: https://github.com/sleepunit-agents/loom/compare/v0.7.0...HEAD
592
+ [0.7.0]: https://github.com/sleepunit-agents/loom/compare/v0.6.0...v0.7.0
593
+ [0.6.0]: https://github.com/sleepunit-agents/loom/compare/v0.5.0...v0.6.0
466
594
  [0.5.0]: https://github.com/sleepunit-agents/loom/compare/v0.4.1...v0.5.0
467
595
  [0.4.1]: https://github.com/sleepunit-agents/loom/compare/v0.3.1...v0.4.1
468
596
  [0.4.0-alpha.7]: https://github.com/sleepunit-agents/loom/compare/v0.4.0-alpha.6...v0.4.0-alpha.7
package/README.md CHANGED
@@ -16,6 +16,10 @@ which model or client it runs in.
16
16
 
17
17
  > When the harness changes, the agent persists.
18
18
 
19
+ **See it whole:** [`docs/architecture.md`](docs/architecture.md) maps every
20
+ moving part (entry points, tool wings, backends, stores) and how they
21
+ touch, as diagrams GitHub renders in place.
22
+
19
23
  ## Demo
20
24
 
21
25
  [![asciicast](https://asciinema.org/a/5bUMXCLmCRc8lg51.svg)](https://asciinema.org/a/5bUMXCLmCRc8lg51)
@@ -55,6 +59,20 @@ an agent's persistent state:
55
59
  - **`memory_archive` / `memory_restore`** — soft-retire a memory with a
56
60
  tombstone (who/when/why + original body preserved) instead of deleting it.
57
61
  Archived memories are excluded from recall and audit but remain recoverable.
62
+ - **`memory_merge`** — consolidate 2+ memories into one canonical row. Acts on
63
+ a `memory_audit` duplicate-finding: sources are archived with a tombstone
64
+ and a supersession pointer to the target, the target's times-seen counter
65
+ sums in each source's count, and bodies can optionally be concatenated.
66
+ Unlike `forget`/`memory_archive` on a duplicate pair, this consolidates
67
+ both sides into one row instead of only discarding one.
68
+ - **`memory_history`** — body-revision history for a memory. `update()` snapshots
69
+ the displaced body into `memory_revisions` before overwriting it. List a
70
+ memory's snapshots (id, op, replaced_at, char count), or read one snapshot's
71
+ full content by `revision_id`. To put a snapshot back, use
72
+ `memory_revision_restore`.
73
+ - **`memory_revision_restore`** — restore a snapshotted memory body back onto
74
+ the memory. Snapshots the current body first, so restore is never destructive.
75
+ Use `memory_history` to list revision IDs before calling this.
58
76
  - **`memory_propose` / `memory_proposals` / `memory_ratify` / `memory_reject`** —
59
77
  the capture-propose queue: a staging area a background lane drafts memory
60
78
  writes into, that Art ratifies before they become canon. A proposal is **not**
@@ -130,6 +148,18 @@ an agent's persistent state:
130
148
  itself a destructive overwrite. Revisions follow the page across renames and
131
149
  are purged with it.
132
150
 
151
+ - **Person cards** — `person_write`, `person_verify`, `person_archive`,
152
+ `person_restore`, `person`, `person_list`, `person_history`,
153
+ `person_revision_restore`. A card is the condensed current state of one human
154
+ the agent works with (where they are in each system, with a freshness marker;
155
+ how to work with them; what they were said to be allowed to do, each with a
156
+ source). One value per field, overwritten; the past lives in revisions. Cards
157
+ are descriptive, never a grant: the harness's gating names (`tier`, `scope`,
158
+ …) are rejected on write. `identity({people: [slug, ...]})` adds an
159
+ `# In the room` block with the cards of the people a turn is addressed by.
160
+ CLI: `loom person show <slug>` / `loom person list`. Design:
161
+ `docs/specs/person-cards.md`.
162
+
133
163
  Everything lives on disk as plain markdown plus a single SQLite file.
134
164
  No daemon, no external service, no GPU.
135
165
 
@@ -276,6 +306,34 @@ must carry a matching `Authorization: Bearer …` header (compared in
276
306
  constant time). When unset, the network boundary alone gates access —
277
307
  appropriate for a token-less tailnet-only daemon.
278
308
 
309
+ ### Knowledge-only scope — a second identity without memory tools
310
+
311
+ `loom serve --http --scope knowledge` starts a second, standalone daemon
312
+ that registers **only** the `knowledge_*` tools — no `identity`,
313
+ `remember`/`recall`, or any other memory-wing tool is on its surface.
314
+ This is how a second identity (e.g. a teammate's agent) can read/write a
315
+ shared knowledge store without ever being handed access to memory. Same
316
+ bind-safety as above; auth is a **per-identity bearer** instead of one
317
+ shared token, so the acting identity is derived from which token was
318
+ presented — never from a request field — and a session is pinned to
319
+ whichever identity opened it.
320
+
321
+ ```bash
322
+ LOOM_KNOWLEDGE_BEARER_TOKENS="art:abc123,mark:def456" \
323
+ loom serve --http --scope knowledge --port 8788
324
+ ```
325
+
326
+ | Variable | Flag | Default | Description |
327
+ |---|---|---|---|
328
+ | `LOOM_KNOWLEDGE_HTTP_PORT` | `--port` | `8788` | Port to listen on |
329
+ | `LOOM_KNOWLEDGE_BEARER_TOKENS` | — | *(required)* | `"identity:token,identity:token"` — refuses to start if empty |
330
+
331
+ Both scopes share the same backing store (`knowledge.db` under the
332
+ context dir) and the same tool implementation — the full server (identity
333
+ hardcoded to its own name) and the scoped service go through the exact
334
+ same registration code, so there's no separate "local" code path that
335
+ could drift from the one the scoped service uses.
336
+
279
337
  ### Session and stream behavior
280
338
 
281
339
  - **One server per session.** Each MCP session (keyed by the
@@ -297,6 +355,34 @@ The connecting client picks its own harness from the MCP handshake
297
355
  `clientInfo.name` (see [harness self-describe](#harness-manifests-and-self-describe)),
298
356
  so a single daemon serves multiple harnesses correctly.
299
357
 
358
+ ### The CLI against a remote loom
359
+
360
+ The store verbs (`wake`, `recall`, `remember`, `update`, `forget`,
361
+ `memory`, `knowledge`, `person`) can run against a daemon instead of a
362
+ local stack, so hooks and scripts that shell out to `loom` keep working
363
+ when loom lives on another host or the agent runs in a container:
364
+
365
+ ```bash
366
+ export LOOM_URL=http://100.94.0.12:8787/mcp # or --url per call
367
+ export LOOM_TOKEN_FILE=/run/secrets/loom-token # the bearer token, read from a file
368
+ loom recall "meeting preferences"
369
+ echo "Prefers async updates" | loom remember "working style" --category user --json
370
+ ```
371
+
372
+ The CLI posts the argv (and a `remember`/`update` body from stdin, or from
373
+ your local `$EDITOR`) to `POST /cli` next to the MCP URL. The daemon runs
374
+ the same CLI against its own stack and sends back stdout, stderr and the
375
+ exit code, so output, `--json` and exit codes match a local run. The
376
+ endpoint sits behind the same bearer gate and payload cap as MCP, takes
377
+ only those verbs (each one already has an MCP tool; `memory
378
+ recompute-salience` and `memory recall-stats` have none, so they stay
379
+ local), and refuses
380
+ `--context-dir`: it only serves its own stack. It only exists when
381
+ `LOOM_BEARER_TOKEN` is set, and it refuses browser requests (any `Origin`
382
+ header, or a body that isn't `application/json`). The knowledge-only daemon
383
+ has no `/cli`. Every other command (`doctor`, `inject`, `serve`, …) stays
384
+ local, and `--url` on one of them is an error.
385
+
300
386
  ### Behind a reverse proxy
301
387
 
302
388
  Terminate TLS at a reverse proxy in front of the daemon and point the
@@ -442,6 +528,10 @@ npx @jbarket/loomai memory proposals --context-dir ~/.config/loom/my-agent
442
528
  npx @jbarket/loomai memory ratify 3 --context-dir ~/.config/loom/my-agent
443
529
  npx @jbarket/loomai memory reject 4 --context-dir ~/.config/loom/my-agent
444
530
 
531
+ # Read a person card (exit 3 when there is none) / list cards and their freshness
532
+ npx @jbarket/loomai person show patrick-kells --context-dir ~/.config/loom/my-agent
533
+ npx @jbarket/loomai person list --context-dir ~/.config/loom/my-agent
534
+
445
535
  # Run loom as an HTTP MCP daemon (mesh-reachable; default stays stdio)
446
536
  npx @jbarket/loomai serve --http --host 127.0.0.1 --port 8787
447
537
 
@@ -503,6 +593,24 @@ loom migrate --dry-run # check without applying
503
593
  loom migrate --json # machine-readable output
504
594
  ```
505
595
 
596
+ ### Pieces — `loom tasks`, `loom secrets`, … as separate programs
597
+
598
+ loom stays identity and memory. Everything else an agent leans on (tasks,
599
+ secrets, a wake clock, an approval page) is a **piece**: its own program,
600
+ reached through loom's command namespace the way `git` reaches `git-<cmd>`.
601
+ `loom tasks list` runs the first `loom-tasks` on `PATH` with `list`, and
602
+ passes back its exit code. Core commands can't be shadowed. Pieces never
603
+ write loom's stores, work alone, and compose when others are installed. Each
604
+ piece role has a contract, so an implementation can be swapped for another.
605
+
606
+ ```bash
607
+ loom pieces # what's installed (name → path)
608
+ loom tasks ready # → loom-tasks ready
609
+ ```
610
+
611
+ The rules, the dispatch details and how pieces compose:
612
+ [`docs/extensions.md`](docs/extensions.md).
613
+
506
614
  ### Harness manifests and self-describe
507
615
 
508
616
  A **harness** is the MCP-capable runtime the agent runs in (Claude Code,
@@ -564,6 +672,10 @@ All configuration is through environment variables:
564
672
  | `LOOM_HTTP_HOST` | `127.0.0.1` | Bind host for `loom serve --http` (bind-safety enforced) |
565
673
  | `LOOM_HTTP_PORT` | `8787` | Bind port for `loom serve --http` |
566
674
  | `LOOM_BEARER_TOKEN` | *(unset)* | Bearer token required on every HTTP request when set |
675
+ | `LOOM_URL` | *(unset)* | Run the CLI's store verbs against this loom-http (its MCP URL) instead of a local stack |
676
+ | `LOOM_TOKEN_FILE` | *(unset)* | File holding the bearer token for `LOOM_URL` (never passed in argv) |
677
+ | `LOOM_KNOWLEDGE_HTTP_PORT` | `8788` | Bind port for `loom serve --http --scope knowledge` |
678
+ | `LOOM_KNOWLEDGE_BEARER_TOKENS` | *(unset)* | `"identity:token,identity:token"` — required for `--scope knowledge` |
567
679
 
568
680
  `--context-dir <path>` works as a CLI alternative to
569
681
  `LOOM_CONTEXT_DIR`.
@@ -618,6 +730,9 @@ and `docs/archive/plans/` — implementation history, frozen after merge.
618
730
 
619
731
  ## Docs
620
732
 
733
+ - [`docs/architecture.md`](docs/architecture.md) — loom at a glance:
734
+ every moving part and how they touch. A contract test fails CI when a
735
+ new tool, subcommand, backend module or table isn't on it.
621
736
  - [`docs/troubleshooting.md`](docs/troubleshooting.md) — install
622
737
  failures, MCP tools not appearing, fastembed download issues, and
623
738
  what each `loom doctor` field means.
@@ -0,0 +1,23 @@
1
+ import type { Database } from 'better-sqlite3';
2
+ export interface UniqKeyInput {
3
+ category: string;
4
+ project?: string | null;
5
+ title: string;
6
+ content: string;
7
+ /** Explicit scope override, if any — folded to the effective scope. */
8
+ scope?: string | null;
9
+ }
10
+ export declare function computeUniqKey(input: UniqKeyInput): string;
11
+ /**
12
+ * Recompute `uniq` for every row from its current category/project/title/
13
+ * content with the current computeUniqKey(). Used by the migrations that
14
+ * introduce or re-key the column.
15
+ *
16
+ * Archived rows get NULL — remember() only ever matches active rows, and
17
+ * restore() recomputes the key when a row comes back. Among active rows,
18
+ * oldest (lowest id) wins: if several already share a key (an unresolved
19
+ * historical duplicate), only the first holds it and the rest stay NULL
20
+ * rather than pointing a future reheat at an arbitrary survivor. Those
21
+ * clusters are what memory_audit + memory_merge are for.
22
+ */
23
+ export declare function rebuildUniqKeys(db: Database): void;
@@ -0,0 +1,100 @@
1
+ /**
2
+ * Write-time dedup key for memories (t-335).
3
+ *
4
+ * `remember()` always inserted — audit() found near-duplicates after the
5
+ * fact (embedding similarity) but nothing caught a character-identical
6
+ * restatement before it became a second row. This gives remember() a cheap
7
+ * floor underneath audit/MMR: normalize and hash category+project+title+content,
8
+ * so an exact restatement collides with the existing row's `uniq` value and
9
+ * reheats it instead of inserting.
10
+ *
11
+ * Normalization is deliberately minimal (PR120 review, two rounds). The key
12
+ * must only fold differences that can never change what a memory says, and
13
+ * almost every character can: a sign ("-5" vs "5"), a decimal point ("1.5"
14
+ * vs "15"), a hyphen in an identifier ("foo-bar" vs "foobar"), a space
15
+ * ("foo bar" vs "foobar"), case ("US" vs "us"), a compatibility form ("x²"
16
+ * vs "x2"), indentation in a code snippet. Earlier versions folded case and
17
+ * punctuation (and, first, everything outside [a-z0-9]); each silently merged
18
+ * distinct writes. What is folded now, in title and content only:
19
+ * - canonical Unicode equivalence (NFC — the same text, encoded two ways)
20
+ * - line-ending style (CRLF/CR → LF)
21
+ * - whitespace at the end of each line, and at the start/end of the field
22
+ * category and project are not normalized at all: they are identifiers, so
23
+ * they are compared exactly (missing project = empty string).
24
+ *
25
+ * The key also carries the memory's *effective* scope (backends/scope.ts):
26
+ * restating a project-scoped memory with scope: 'global' is a different
27
+ * write (it changes who can recall it), so it must not reheat the scoped
28
+ * row and drop the promotion. An explicit scope equal to the category
29
+ * default is the same write and still reheats.
30
+ *
31
+ * This only catches character-identical restatements (after normalization).
32
+ * It is not a replacement for audit's similarity-based duplicate detection —
33
+ * paraphrases, reworded titles, or trimmed content still need audit + merge.
34
+ */
35
+ import { createHash } from 'node:crypto';
36
+ import { effectiveScope } from './scope.js';
37
+ function normalize(text) {
38
+ return text
39
+ .normalize('NFC')
40
+ .replace(/\r\n?/g, '\n')
41
+ .replace(/[^\S\n]+$/gm, '')
42
+ .trim();
43
+ }
44
+ export function computeUniqKey(input) {
45
+ // JSON array, not a delimiter join: no field content can alias a
46
+ // different split of the same characters across fields.
47
+ const basis = JSON.stringify([
48
+ input.category,
49
+ input.project ?? '',
50
+ effectiveScope(input),
51
+ normalize(input.title),
52
+ normalize(input.content),
53
+ ]);
54
+ return createHash('sha256').update(basis, 'utf-8').digest('hex');
55
+ }
56
+ /**
57
+ * Recompute `uniq` for every row from its current category/project/title/
58
+ * content with the current computeUniqKey(). Used by the migrations that
59
+ * introduce or re-key the column.
60
+ *
61
+ * Archived rows get NULL — remember() only ever matches active rows, and
62
+ * restore() recomputes the key when a row comes back. Among active rows,
63
+ * oldest (lowest id) wins: if several already share a key (an unresolved
64
+ * historical duplicate), only the first holds it and the rest stay NULL
65
+ * rather than pointing a future reheat at an arbitrary survivor. Those
66
+ * clusters are what memory_audit + memory_merge are for.
67
+ */
68
+ export function rebuildUniqKeys(db) {
69
+ const rows = db
70
+ .prepare(
71
+ // scope only exists from add_scope on; a DB that predates it has no
72
+ // explicit overrides, so NULL (category default) is exact.
73
+ `SELECT id, category, project, title, content, archived,
74
+ ${hasScopeColumn(db) ? 'scope' : 'NULL AS scope'}
75
+ FROM memories ORDER BY id ASC`)
76
+ .all();
77
+ const seen = new Set();
78
+ const update = db.prepare('UPDATE memories SET uniq = ? WHERE id = ?');
79
+ db.transaction(() => {
80
+ // Clear every key first. Rekeying row by row in place could otherwise
81
+ // write a row's new key while a later row still holds it as a stale old
82
+ // key, and trip the unique index on active rows.
83
+ db.prepare('UPDATE memories SET uniq = NULL').run();
84
+ for (const row of rows) {
85
+ let key = null;
86
+ if (row.archived === 0) {
87
+ const k = computeUniqKey(row);
88
+ if (!seen.has(k)) {
89
+ seen.add(k);
90
+ key = k;
91
+ }
92
+ }
93
+ update.run(key, row.id);
94
+ }
95
+ })();
96
+ }
97
+ function hasScopeColumn(db) {
98
+ const info = db.pragma('table_info(memories)');
99
+ return info.some((c) => c.name === 'scope');
100
+ }
@@ -41,12 +41,25 @@ export const EmbeddingModel = {
41
41
  MLE5Large: 'fast-multilingual-e5-large',
42
42
  };
43
43
  /**
44
- * Where model tarballs come from. The upstream default is a Qdrant-owned GCS
45
- * bucket whose access has broken at least twice (fastembed-js issues #18, #33)
46
- * and which nobody maintains any more; the env override exists so a broken
47
- * bucket is a config change, not a release.
44
+ * Where model tarballs come from. The upstream default was a Qdrant-owned
45
+ * GCS bucket; as of 2026-10-16 every object in it 403s (t-1262) and nobody
46
+ * at Qdrant maintains it, so it isn't coming back. The replacement is a
47
+ * GitHub Release asset in this repo (sleepunit-agents/loom, tag
48
+ * models-fast-bge-small-en-v1.5-v1): the exact bytes this box had already
49
+ * downloaded from the GCS bucket before it died, reuploaded rather than
50
+ * re-exported, and confirmed bit-identical against the
51
+ * embedding-runtime.parity.test.ts golden vectors before upload. That
52
+ * matters: the public Hugging Face re-exports of this model are either
53
+ * int8/fp16-quantized or built with a different ONNX optimizer pass, and
54
+ * produce embeddings close to but not bit-identical with every vector
55
+ * already stored in a loom database. Only fast-bge-small-en-v1.5 — the
56
+ * model loom actually defaults to (src/backends/index.ts) — has a release
57
+ * asset; other EmbeddingModel entries will fail download() with a clear
58
+ * "set LOOM_MODEL_BASE_URL" error until someone needs one and mirrors it
59
+ * the same way. The env override exists so a dead host is a config change,
60
+ * not a release.
48
61
  */
49
- const DEFAULT_MODEL_BASE_URL = 'https://storage.googleapis.com/qdrant-fastembed';
62
+ const DEFAULT_MODEL_BASE_URL = 'https://github.com/sleepunit-agents/loom/releases/download/models-fast-bge-small-en-v1.5-v1';
50
63
  /**
51
64
  * Loaded dense embedder: a tokenizer plus an ONNX inference session.
52
65
  *
@@ -238,6 +251,12 @@ function download(outputPath, model) {
238
251
  : model;
239
252
  const base = process.env.LOOM_MODEL_BASE_URL ?? DEFAULT_MODEL_BASE_URL;
240
253
  const url = `${base}/${remoteName}.tar.gz`;
254
+ // GitHub Release assets (the new default host) 302 to a signed, short-lived
255
+ // blob URL; the GCS bucket they replace never redirected, so this following
256
+ // logic didn't previously exist.
257
+ return fetchToFile(url, outputPath, 5);
258
+ }
259
+ function fetchToFile(url, outputPath, redirectsLeft) {
241
260
  return new Promise((resolve, reject) => {
242
261
  const fail = (err) => {
243
262
  // A partial archive on disk would be treated as a complete download on
@@ -246,15 +265,25 @@ function download(outputPath, model) {
246
265
  unlinkSync(outputPath);
247
266
  reject(err);
248
267
  };
249
- const file = createWriteStream(outputPath);
250
- file.on('error', fail);
251
268
  httpsGet(url, { headers: { 'User-Agent': 'loom' } }, (response) => {
252
- if (response.statusCode !== 200) {
269
+ const { statusCode, headers } = response;
270
+ if (statusCode && statusCode >= 300 && statusCode < 400 && headers.location) {
271
+ response.resume();
272
+ if (redirectsLeft <= 0) {
273
+ fail(new Error(`Model download failed: too many redirects fetching ${url.toString()}`));
274
+ return;
275
+ }
276
+ fetchToFile(new URL(headers.location, url), outputPath, redirectsLeft - 1).then(resolve, fail);
277
+ return;
278
+ }
279
+ if (statusCode !== 200) {
253
280
  response.resume();
254
- fail(new Error(`Model download failed: ${url} returned HTTP ${response.statusCode}. ` +
281
+ fail(new Error(`Model download failed: ${url.toString()} returned HTTP ${statusCode}. ` +
255
282
  'Set LOOM_MODEL_BASE_URL to fetch models from a different host.'));
256
283
  return;
257
284
  }
285
+ const file = createWriteStream(outputPath);
286
+ file.on('error', fail);
258
287
  response.on('error', fail);
259
288
  response.pipe(file);
260
289
  file.on('finish', () => {
@@ -93,7 +93,7 @@ function openMemoriesDb(contextDir) {
93
93
  if (!existsSync(dbPath))
94
94
  return null;
95
95
  const db = new BetterSqlite3(dbPath, { readonly: true });
96
- db.pragma('busy_timeout = 0');
96
+ db.pragma('busy_timeout = 5000');
97
97
  return db;
98
98
  }
99
99
  /** Raw episode rows in the window, oldest first. Empty when there is no store. */
@@ -48,11 +48,30 @@ export class FastEmbedProvider {
48
48
  if (texts.length === 0)
49
49
  return [];
50
50
  const embedder = await this.ensureEmbedder();
51
- const vectors = await collectBatches(embedder.embed(texts, 32));
52
- if (vectors.length !== texts.length) {
53
- throw new Error(`fastembed: embedBatch() produced ${vectors.length} vectors for ${texts.length} inputs (model ${this.config.model})`);
51
+ // Staged fallback ladder (t-334): a batch-level failure anywhere in a
52
+ // large request must not take remember()/recall() down with it. Each
53
+ // rung is strictly less efficient but more isolated than the one above
54
+ // it; only descend when the rung above actually failed or came back
55
+ // short.
56
+ const rungs = [
57
+ () => collectBatches(embedder.embed(texts, 32)), // 1. one batched call
58
+ () => collectBatches(embedder.embed(texts, 1)), // 2. sequential, still whole-array
59
+ () => embedOneByOne(texts, (t) => this.embed(t)), // 3. isolated, one call per text
60
+ ];
61
+ let lastErr;
62
+ for (const runRung of rungs) {
63
+ try {
64
+ const vectors = await runRung();
65
+ if (vectors.length === texts.length && vectors.every((v) => v.length > 0)) {
66
+ return vectors;
67
+ }
68
+ lastErr = new Error(`fastembed: embedBatch() produced ${vectors.length} vectors for ${texts.length} inputs (model ${this.config.model})`);
69
+ }
70
+ catch (err) {
71
+ lastErr = err;
72
+ }
54
73
  }
55
- return vectors;
74
+ throw lastErr;
56
75
  }
57
76
  async embedQuery(text) {
58
77
  const embedder = await this.ensureEmbedder();
@@ -93,3 +112,10 @@ async function collectBatches(gen) {
93
112
  }
94
113
  return out;
95
114
  }
115
+ /** Rung 3 of the embedBatch fallback ladder: one isolated call per text. */
116
+ async function embedOneByOne(texts, embedOne) {
117
+ const out = [];
118
+ for (const text of texts)
119
+ out.push(await embedOne(text));
120
+ return out;
121
+ }