@jbarket/loomai 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +114 -1
- package/README.md +97 -0
- package/dist/backends/dedup.d.ts +23 -0
- package/dist/backends/dedup.js +100 -0
- package/dist/backends/embedding-runtime.js +38 -9
- package/dist/backends/episodes.js +1 -1
- package/dist/backends/fastembed.js +30 -4
- package/dist/backends/migrations.js +200 -0
- package/dist/backends/mmr.d.ts +8 -0
- package/dist/backends/mmr.js +18 -4
- package/dist/backends/people.d.ts +192 -0
- package/dist/backends/people.js +824 -0
- package/dist/backends/proposals.js +1 -1
- package/dist/backends/recorder-health.d.ts +34 -0
- package/dist/backends/recorder-health.js +117 -0
- package/dist/backends/salience.d.ts +39 -0
- package/dist/backends/salience.js +52 -4
- package/dist/backends/scope.d.ts +56 -0
- package/dist/backends/scope.js +75 -0
- package/dist/backends/sqlite-knowledge.d.ts +24 -4
- package/dist/backends/sqlite-knowledge.js +253 -91
- package/dist/backends/sqlite-vec.d.ts +32 -1
- package/dist/backends/sqlite-vec.js +394 -75
- package/dist/backends/types.d.ts +203 -3
- package/dist/categories.d.ts +2 -0
- package/dist/categories.js +2 -0
- package/dist/cli/args.d.ts +16 -1
- package/dist/cli/args.js +45 -1
- package/dist/cli/index.d.ts +7 -0
- package/dist/cli/index.js +57 -3
- package/dist/cli/knowledge.js +4 -1
- package/dist/cli/memory.js +2 -2
- package/dist/cli/person.d.ts +2 -0
- package/dist/cli/person.js +116 -0
- package/dist/cli/recall.js +2 -1
- package/dist/cli/remember.js +10 -0
- package/dist/cli/remote.d.ts +36 -0
- package/dist/cli/remote.js +173 -0
- package/dist/cli/serve.d.ts +7 -0
- package/dist/cli/serve.js +94 -5
- package/dist/cli/subcommands.d.ts +1 -1
- package/dist/cli/subcommands.js +1 -1
- package/dist/cli/update.js +30 -11
- package/dist/clients.js +7 -4
- package/dist/config.d.ts +14 -0
- package/dist/config.js +25 -2
- package/dist/index.js +22 -7
- package/dist/injection/harnesses.js +9 -3
- package/dist/install/harnesses.js +7 -1
- package/dist/server.d.ts +8 -1
- package/dist/server.js +461 -374
- package/dist/tools/dossier.js +5 -1
- package/dist/tools/eurorack-import.js +3 -0
- package/dist/tools/identity.d.ts +2 -0
- package/dist/tools/identity.js +35 -2
- package/dist/tools/knowledge-history.d.ts +6 -1
- package/dist/tools/knowledge-history.js +26 -8
- package/dist/tools/knowledge-maintain.d.ts +5 -0
- package/dist/tools/knowledge-maintain.js +92 -5
- package/dist/tools/knowledge-recall.d.ts +7 -1
- package/dist/tools/knowledge-recall.js +53 -3
- package/dist/tools/knowledge-verify.d.ts +11 -1
- package/dist/tools/knowledge-verify.js +17 -4
- package/dist/tools/knowledge-write.d.ts +8 -1
- package/dist/tools/knowledge-write.js +27 -9
- package/dist/tools/memory-history.d.ts +5 -0
- package/dist/tools/memory-history.js +52 -0
- package/dist/tools/memory-list.js +2 -1
- package/dist/tools/memory-merge.d.ts +2 -0
- package/dist/tools/memory-merge.js +40 -0
- package/dist/tools/memory-revision-restore.d.ts +5 -0
- package/dist/tools/memory-revision-restore.js +33 -0
- package/dist/tools/person.d.ts +32 -0
- package/dist/tools/person.js +111 -0
- package/dist/tools/prune.js +1 -1
- package/dist/tools/recall.js +7 -1
- package/dist/tools/remember.js +18 -2
- package/dist/tools/update.js +5 -1
- package/dist/transport/cli-endpoint.d.ts +13 -0
- package/dist/transport/cli-endpoint.js +97 -0
- package/dist/transport/guards.d.ts +14 -0
- package/dist/transport/guards.js +26 -0
- package/dist/transport/http-server.d.ts +28 -1
- package/dist/transport/http-server.js +156 -18
- package/dist/transport/knowledge-server.d.ts +11 -0
- package/dist/transport/knowledge-server.js +30 -0
- package/dist/transport/knowledge-tools.d.ts +50 -0
- package/dist/transport/knowledge-tools.js +290 -0
- package/dist/transport/mcp-app-hello.d.ts +1 -1
- package/dist/widgets/display-modes.html +62 -56
- package/dist/widgets/hello.html +62 -56
- package/dist/widgets/proposals.html +62 -56
- package/dist/widgets/top-of-mind.html +60 -54
- package/package.json +7 -5
package/CHANGELOG.md
CHANGED
|
@@ -12,6 +12,118 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
|
12
12
|
|
|
13
13
|
## [Unreleased]
|
|
14
14
|
|
|
15
|
+
## [0.6.0] - 2026-10-24
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- **CLI remote mode: `--url` / `LOOM_URL`, and `POST /cli` on loom-http.**
|
|
20
|
+
The store verbs (`wake`, `recall`, `remember`, `update`, `forget`, `memory`,
|
|
21
|
+
`knowledge`, `person`) can run against a daemon instead of a local stack,
|
|
22
|
+
for hooks and scripts on a host or in a container where loom lives
|
|
23
|
+
elsewhere. The CLI ships argv and stdin to `POST /cli`; the daemon runs the
|
|
24
|
+
same CLI on its own stack and returns stdout, stderr and the exit code, so
|
|
25
|
+
output and `--json` match a local run by construction. The bearer token
|
|
26
|
+
comes from `LOOM_TOKEN_FILE`. The endpoint is full-scope only, needs
|
|
27
|
+
`LOOM_BEARER_TOKEN` set, refuses browser requests (an `Origin` header or a
|
|
28
|
+
non-JSON body), and sits behind the same bearer gate and payload cap as
|
|
29
|
+
MCP. It takes only those verbs (minus `memory recompute-salience` and
|
|
30
|
+
`memory recall-stats`, which have no MCP tool), refuses `--context-dir` and
|
|
31
|
+
`--url`, and passes only `LOOM_CLIENT`/`LOOM_MODEL` through from the
|
|
32
|
+
caller's env.
|
|
33
|
+
|
|
34
|
+
- **Person cards: `people` + `person_revisions`, eight `person_*` tools,
|
|
35
|
+
`loom person show|list`, and identity's `people` param.** A card is the
|
|
36
|
+
condensed current state of one human the agent works with: `where` fields
|
|
37
|
+
carrying a freshness marker (`[ok]`, `[unverified]`, `[STALE <n>d]`), `how`
|
|
38
|
+
fields (time zone, mode, who to loop in), descriptive `may` entries each with
|
|
39
|
+
a source, and a short agent-written summary. One value per field,
|
|
40
|
+
overwritten in place; every stored change snapshots the previous row (newest
|
|
41
|
+
10 kept), and every mutation is one `BEGIN IMMEDIATE` transaction. Cards are
|
|
42
|
+
not an authorization surface: the harness's gating names (`tier`, `field`,
|
|
43
|
+
`projects`, `scope`, `trust`) are rejected anywhere on a card.
|
|
44
|
+
`identity({people: [slug, ...]})` adds an `# In the room` block after the
|
|
45
|
+
tape (≤ 4 cards, 6,500 chars, best-effort). The dossier now says worker
|
|
46
|
+
bodies must not write cards. Design and acceptance criteria:
|
|
47
|
+
`docs/specs/person-cards.md` (criteria 1–12; the misfile heuristic, 13, is
|
|
48
|
+
still to come).
|
|
49
|
+
|
|
50
|
+
- **Knowledge actor attribution: `pages.author`, `page_revisions.actor`, and a
|
|
51
|
+
`verifications` table (page_id, verifier, verified_at, outcome,
|
|
52
|
+
citation_checked).** Previously the only attribution field was
|
|
53
|
+
`created_by` — free text, meaning "who owns this `ours/` artifact," not
|
|
54
|
+
"who actually wrote or verified it" — so there was no way to tell who wrote
|
|
55
|
+
a page's first version, who performed a given body revision, or who has
|
|
56
|
+
verified a page and found what. `pages.author` is stamped once at creation
|
|
57
|
+
and never changed by later upserts; `page_revisions.actor` is stamped on
|
|
58
|
+
every body-replacing write and every history-restore; `verifications` is an
|
|
59
|
+
attributed audit trail additive to the existing `pages.verified_at` stamp
|
|
60
|
+
(which stays as the fast single-column freshness field everything else
|
|
61
|
+
sorts/filters on). All three are resolved server-side, never a free-text
|
|
62
|
+
tool argument, so no caller can claim another identity's authorship or
|
|
63
|
+
verification: MCP calls use the connection's identity from
|
|
64
|
+
`registerKnowledgeTools` ('art' on the main server, the bearer-resolved
|
|
65
|
+
identity on the knowledge-only service); the CLI and eurorack import use
|
|
66
|
+
`resolveIdentityName()` (`LOOM_IDENTITY` env override, else the context
|
|
67
|
+
dir's real basename).
|
|
68
|
+
`knowledge_verify` gains `outcome` ('confirmed' | 'stale' | 'corrected') and
|
|
69
|
+
`citation_checked` parameters; `knowledge_history`'s listing now shows who
|
|
70
|
+
performed each revision and surfaces the page's verification history. (t-675)
|
|
71
|
+
An outcome of 'stale' is a failed verification: it is recorded in
|
|
72
|
+
`verifications` but does NOT advance `pages.verified_at`, so a known-wrong
|
|
73
|
+
page keeps its place at the front of the stale-first queue; the result's
|
|
74
|
+
new `stamped` flag (and the tool output) says whether the stamp moved.
|
|
75
|
+
|
|
76
|
+
- **`MemoryMatch` carries `similarity` alongside `relevance`, and `scope`
|
|
77
|
+
alongside `project`.** `relevance` is now the post-MMR score that actually
|
|
78
|
+
decided a result's rank — equal to `similarity` (raw cosine) unless MMR
|
|
79
|
+
re-ranking traded a pick off against ones already selected, in which case
|
|
80
|
+
the two now diverge and both are visible (modeled on MemoriLabs/Memori's
|
|
81
|
+
`FactSearchResult.{similarity, rank_score}` split). Separately, every
|
|
82
|
+
memory now has an effective `scope` ('project' or 'global'): `recall` and
|
|
83
|
+
`memory similar` exclude a project-scoped memory tagged to a *different*
|
|
84
|
+
project than the one passed via `--project`/`project`, while global-scoped
|
|
85
|
+
memories (category default for `user`/`reference`, any untagged memory, or
|
|
86
|
+
an explicit `scope: "global"` at write time) still come through regardless.
|
|
87
|
+
Closes the gap where a preference learned on one project could surface
|
|
88
|
+
while working on an unrelated one. `remember` (MCP tool and `loom remember
|
|
89
|
+
--scope`) takes an explicit `project | global` override for the categories
|
|
90
|
+
where the category default can't tell (`self`, `feedback`) — e.g. a
|
|
91
|
+
security/git/tool-workflow lesson learned in one project but true
|
|
92
|
+
everywhere. (t-337)
|
|
93
|
+
|
|
94
|
+
### Fixed
|
|
95
|
+
|
|
96
|
+
- **All 34 MCP tools now declare `ToolAnnotations`** (`readOnlyHint`,
|
|
97
|
+
`destructiveHint`, `idempotentHint`, `openWorldHint`), so a client gating
|
|
98
|
+
approval on these hints (e.g. Codex under `approval: never`) can tell a
|
|
99
|
+
read from a write instead of refusing every call uniformly. Hints follow
|
|
100
|
+
actual behavior, not tool-name conventions: `recall`/`knowledge_recall`
|
|
101
|
+
are NOT read-only (they stamp `last_accessed`/`hit_count`, which
|
|
102
|
+
`knowledge_maintain` reads to find cold pages); `knowledge_history` and
|
|
103
|
+
`memory_revision_restore` are destructive in their restore mode even
|
|
104
|
+
though the displaced body is snapshotted first. `update`,
|
|
105
|
+
`knowledge_write` and `knowledge_move` are NOT idempotent: a repeated
|
|
106
|
+
`observation` bumps the evidence count again, a repeated append stores
|
|
107
|
+
the body twice, and a repeated prefix move into a child of itself
|
|
108
|
+
(`music` → `music/gear`) nests again. `openWorldHint` is
|
|
109
|
+
`false` everywhere — loom has no outbound network calls. This does not
|
|
110
|
+
by itself prove any client relaxes its approval gate on these hints; that
|
|
111
|
+
needs a live `tools/list` exchange against an updated server (tracked
|
|
112
|
+
separately). (t-415)
|
|
113
|
+
|
|
114
|
+
- **`--context-dir` (and the other global flags) now work in front of the
|
|
115
|
+
subcommand, as the help text has always claimed.** `loom --context-dir DIR
|
|
116
|
+
knowledge write …` used to route to the stdio MCP server instead of the CLI:
|
|
117
|
+
the entry point dispatched on `argv[2]`, which was `--context-dir`, not a known
|
|
118
|
+
verb. The server read an empty stdin and exited 0, so a *write* verb reported
|
|
119
|
+
success having never entered the write path — and never even reached argument
|
|
120
|
+
validation, exiting 0 on an invocation that the working flag order rejects with
|
|
121
|
+
exit 1. Measured against a disposable context dir: no `knowledge.db` was created
|
|
122
|
+
and no row was written. Both entry points now resolve the subcommand past any
|
|
123
|
+
leading global flags, skipping flag values in pairs so `--context-dir recall`
|
|
124
|
+
still names a directory rather than invoking `recall`. Bare
|
|
125
|
+
`loom --context-dir DIR` with no subcommand remains the MCP path. (t-665)
|
|
126
|
+
|
|
15
127
|
## [0.5.0] - 2026-09-04
|
|
16
128
|
|
|
17
129
|
### Changed
|
|
@@ -462,7 +574,8 @@ Initial public release.
|
|
|
462
574
|
stack and all added external-service dependencies or operational
|
|
463
575
|
overhead.
|
|
464
576
|
|
|
465
|
-
[Unreleased]: https://github.com/sleepunit-agents/loom/compare/v0.
|
|
577
|
+
[Unreleased]: https://github.com/sleepunit-agents/loom/compare/v0.6.0...HEAD
|
|
578
|
+
[0.6.0]: https://github.com/sleepunit-agents/loom/compare/v0.5.0...v0.6.0
|
|
466
579
|
[0.5.0]: https://github.com/sleepunit-agents/loom/compare/v0.4.1...v0.5.0
|
|
467
580
|
[0.4.1]: https://github.com/sleepunit-agents/loom/compare/v0.3.1...v0.4.1
|
|
468
581
|
[0.4.0-alpha.7]: https://github.com/sleepunit-agents/loom/compare/v0.4.0-alpha.6...v0.4.0-alpha.7
|
package/README.md
CHANGED
|
@@ -16,6 +16,10 @@ which model or client it runs in.
|
|
|
16
16
|
|
|
17
17
|
> When the harness changes, the agent persists.
|
|
18
18
|
|
|
19
|
+
**See it whole:** [`docs/architecture.md`](docs/architecture.md) maps every
|
|
20
|
+
moving part (entry points, tool wings, backends, stores) and how they
|
|
21
|
+
touch, as diagrams GitHub renders in place.
|
|
22
|
+
|
|
19
23
|
## Demo
|
|
20
24
|
|
|
21
25
|
[](https://asciinema.org/a/5bUMXCLmCRc8lg51)
|
|
@@ -55,6 +59,20 @@ an agent's persistent state:
|
|
|
55
59
|
- **`memory_archive` / `memory_restore`** — soft-retire a memory with a
|
|
56
60
|
tombstone (who/when/why + original body preserved) instead of deleting it.
|
|
57
61
|
Archived memories are excluded from recall and audit but remain recoverable.
|
|
62
|
+
- **`memory_merge`** — consolidate 2+ memories into one canonical row. Acts on
|
|
63
|
+
a `memory_audit` duplicate-finding: sources are archived with a tombstone
|
|
64
|
+
and a supersession pointer to the target, the target's times-seen counter
|
|
65
|
+
sums in each source's count, and bodies can optionally be concatenated.
|
|
66
|
+
Unlike `forget`/`memory_archive` on a duplicate pair, this consolidates
|
|
67
|
+
both sides into one row instead of only discarding one.
|
|
68
|
+
- **`memory_history`** — body-revision history for a memory. `update()` snapshots
|
|
69
|
+
the displaced body into `memory_revisions` before overwriting it. List a
|
|
70
|
+
memory's snapshots (id, op, replaced_at, char count), or read one snapshot's
|
|
71
|
+
full content by `revision_id`. To put a snapshot back, use
|
|
72
|
+
`memory_revision_restore`.
|
|
73
|
+
- **`memory_revision_restore`** — restore a snapshotted memory body back onto
|
|
74
|
+
the memory. Snapshots the current body first, so restore is never destructive.
|
|
75
|
+
Use `memory_history` to list revision IDs before calling this.
|
|
58
76
|
- **`memory_propose` / `memory_proposals` / `memory_ratify` / `memory_reject`** —
|
|
59
77
|
the capture-propose queue: a staging area a background lane drafts memory
|
|
60
78
|
writes into, that Art ratifies before they become canon. A proposal is **not**
|
|
@@ -130,6 +148,18 @@ an agent's persistent state:
|
|
|
130
148
|
itself a destructive overwrite. Revisions follow the page across renames and
|
|
131
149
|
are purged with it.
|
|
132
150
|
|
|
151
|
+
- **Person cards** — `person_write`, `person_verify`, `person_archive`,
|
|
152
|
+
`person_restore`, `person`, `person_list`, `person_history`,
|
|
153
|
+
`person_revision_restore`. A card is the condensed current state of one human
|
|
154
|
+
the agent works with (where they are in each system, with a freshness marker;
|
|
155
|
+
how to work with them; what they were said to be allowed to do, each with a
|
|
156
|
+
source). One value per field, overwritten; the past lives in revisions. Cards
|
|
157
|
+
are descriptive, never a grant: the harness's gating names (`tier`, `scope`,
|
|
158
|
+
…) are rejected on write. `identity({people: [slug, ...]})` adds an
|
|
159
|
+
`# In the room` block with the cards of the people a turn is addressed by.
|
|
160
|
+
CLI: `loom person show <slug>` / `loom person list`. Design:
|
|
161
|
+
`docs/specs/person-cards.md`.
|
|
162
|
+
|
|
133
163
|
Everything lives on disk as plain markdown plus a single SQLite file.
|
|
134
164
|
No daemon, no external service, no GPU.
|
|
135
165
|
|
|
@@ -276,6 +306,34 @@ must carry a matching `Authorization: Bearer …` header (compared in
|
|
|
276
306
|
constant time). When unset, the network boundary alone gates access —
|
|
277
307
|
appropriate for a token-less tailnet-only daemon.
|
|
278
308
|
|
|
309
|
+
### Knowledge-only scope — a second identity without memory tools
|
|
310
|
+
|
|
311
|
+
`loom serve --http --scope knowledge` starts a second, standalone daemon
|
|
312
|
+
that registers **only** the `knowledge_*` tools — no `identity`,
|
|
313
|
+
`remember`/`recall`, or any other memory-wing tool is on its surface.
|
|
314
|
+
This is how a second identity (e.g. a teammate's agent) can read/write a
|
|
315
|
+
shared knowledge store without ever being handed access to memory. Same
|
|
316
|
+
bind-safety as above; auth is a **per-identity bearer** instead of one
|
|
317
|
+
shared token, so the acting identity is derived from which token was
|
|
318
|
+
presented — never from a request field — and a session is pinned to
|
|
319
|
+
whichever identity opened it.
|
|
320
|
+
|
|
321
|
+
```bash
|
|
322
|
+
LOOM_KNOWLEDGE_BEARER_TOKENS="art:abc123,mark:def456" \
|
|
323
|
+
loom serve --http --scope knowledge --port 8788
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
| Variable | Flag | Default | Description |
|
|
327
|
+
|---|---|---|---|
|
|
328
|
+
| `LOOM_KNOWLEDGE_HTTP_PORT` | `--port` | `8788` | Port to listen on |
|
|
329
|
+
| `LOOM_KNOWLEDGE_BEARER_TOKENS` | — | *(required)* | `"identity:token,identity:token"` — refuses to start if empty |
|
|
330
|
+
|
|
331
|
+
Both scopes share the same backing store (`knowledge.db` under the
|
|
332
|
+
context dir) and the same tool implementation — the full server (identity
|
|
333
|
+
hardcoded to its own name) and the scoped service go through the exact
|
|
334
|
+
same registration code, so there's no separate "local" code path that
|
|
335
|
+
could drift from the one the scoped service uses.
|
|
336
|
+
|
|
279
337
|
### Session and stream behavior
|
|
280
338
|
|
|
281
339
|
- **One server per session.** Each MCP session (keyed by the
|
|
@@ -297,6 +355,34 @@ The connecting client picks its own harness from the MCP handshake
|
|
|
297
355
|
`clientInfo.name` (see [harness self-describe](#harness-manifests-and-self-describe)),
|
|
298
356
|
so a single daemon serves multiple harnesses correctly.
|
|
299
357
|
|
|
358
|
+
### The CLI against a remote loom
|
|
359
|
+
|
|
360
|
+
The store verbs (`wake`, `recall`, `remember`, `update`, `forget`,
|
|
361
|
+
`memory`, `knowledge`, `person`) can run against a daemon instead of a
|
|
362
|
+
local stack, so hooks and scripts that shell out to `loom` keep working
|
|
363
|
+
when loom lives on another host or the agent runs in a container:
|
|
364
|
+
|
|
365
|
+
```bash
|
|
366
|
+
export LOOM_URL=http://100.94.0.12:8787/mcp # or --url per call
|
|
367
|
+
export LOOM_TOKEN_FILE=/run/secrets/loom-token # the bearer token, read from a file
|
|
368
|
+
loom recall "meeting preferences"
|
|
369
|
+
echo "Prefers async updates" | loom remember "working style" --category user --json
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
The CLI posts the argv (and a `remember`/`update` body from stdin, or from
|
|
373
|
+
your local `$EDITOR`) to `POST /cli` next to the MCP URL. The daemon runs
|
|
374
|
+
the same CLI against its own stack and sends back stdout, stderr and the
|
|
375
|
+
exit code, so output, `--json` and exit codes match a local run. The
|
|
376
|
+
endpoint sits behind the same bearer gate and payload cap as MCP, takes
|
|
377
|
+
only those verbs (each one already has an MCP tool; `memory
|
|
378
|
+
recompute-salience` and `memory recall-stats` have none, so they stay
|
|
379
|
+
local), and refuses
|
|
380
|
+
`--context-dir`: it only serves its own stack. It only exists when
|
|
381
|
+
`LOOM_BEARER_TOKEN` is set, and it refuses browser requests (any `Origin`
|
|
382
|
+
header, or a body that isn't `application/json`). The knowledge-only daemon
|
|
383
|
+
has no `/cli`. Every other command (`doctor`, `inject`, `serve`, …) stays
|
|
384
|
+
local, and `--url` on one of them is an error.
|
|
385
|
+
|
|
300
386
|
### Behind a reverse proxy
|
|
301
387
|
|
|
302
388
|
Terminate TLS at a reverse proxy in front of the daemon and point the
|
|
@@ -442,6 +528,10 @@ npx @jbarket/loomai memory proposals --context-dir ~/.config/loom/my-agent
|
|
|
442
528
|
npx @jbarket/loomai memory ratify 3 --context-dir ~/.config/loom/my-agent
|
|
443
529
|
npx @jbarket/loomai memory reject 4 --context-dir ~/.config/loom/my-agent
|
|
444
530
|
|
|
531
|
+
# Read a person card (exit 3 when there is none) / list cards and their freshness
|
|
532
|
+
npx @jbarket/loomai person show patrick-kells --context-dir ~/.config/loom/my-agent
|
|
533
|
+
npx @jbarket/loomai person list --context-dir ~/.config/loom/my-agent
|
|
534
|
+
|
|
445
535
|
# Run loom as an HTTP MCP daemon (mesh-reachable; default stays stdio)
|
|
446
536
|
npx @jbarket/loomai serve --http --host 127.0.0.1 --port 8787
|
|
447
537
|
|
|
@@ -564,6 +654,10 @@ All configuration is through environment variables:
|
|
|
564
654
|
| `LOOM_HTTP_HOST` | `127.0.0.1` | Bind host for `loom serve --http` (bind-safety enforced) |
|
|
565
655
|
| `LOOM_HTTP_PORT` | `8787` | Bind port for `loom serve --http` |
|
|
566
656
|
| `LOOM_BEARER_TOKEN` | *(unset)* | Bearer token required on every HTTP request when set |
|
|
657
|
+
| `LOOM_URL` | *(unset)* | Run the CLI's store verbs against this loom-http (its MCP URL) instead of a local stack |
|
|
658
|
+
| `LOOM_TOKEN_FILE` | *(unset)* | File holding the bearer token for `LOOM_URL` (never passed in argv) |
|
|
659
|
+
| `LOOM_KNOWLEDGE_HTTP_PORT` | `8788` | Bind port for `loom serve --http --scope knowledge` |
|
|
660
|
+
| `LOOM_KNOWLEDGE_BEARER_TOKENS` | *(unset)* | `"identity:token,identity:token"` — required for `--scope knowledge` |
|
|
567
661
|
|
|
568
662
|
`--context-dir <path>` works as a CLI alternative to
|
|
569
663
|
`LOOM_CONTEXT_DIR`.
|
|
@@ -618,6 +712,9 @@ and `docs/archive/plans/` — implementation history, frozen after merge.
|
|
|
618
712
|
|
|
619
713
|
## Docs
|
|
620
714
|
|
|
715
|
+
- [`docs/architecture.md`](docs/architecture.md) — loom at a glance:
|
|
716
|
+
every moving part and how they touch. A contract test fails CI when a
|
|
717
|
+
new tool, subcommand, backend module or table isn't on it.
|
|
621
718
|
- [`docs/troubleshooting.md`](docs/troubleshooting.md) — install
|
|
622
719
|
failures, MCP tools not appearing, fastembed download issues, and
|
|
623
720
|
what each `loom doctor` field means.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Database } from 'better-sqlite3';
|
|
2
|
+
export interface UniqKeyInput {
|
|
3
|
+
category: string;
|
|
4
|
+
project?: string | null;
|
|
5
|
+
title: string;
|
|
6
|
+
content: string;
|
|
7
|
+
/** Explicit scope override, if any — folded to the effective scope. */
|
|
8
|
+
scope?: string | null;
|
|
9
|
+
}
|
|
10
|
+
export declare function computeUniqKey(input: UniqKeyInput): string;
|
|
11
|
+
/**
|
|
12
|
+
* Recompute `uniq` for every row from its current category/project/title/
|
|
13
|
+
* content with the current computeUniqKey(). Used by the migrations that
|
|
14
|
+
* introduce or re-key the column.
|
|
15
|
+
*
|
|
16
|
+
* Archived rows get NULL — remember() only ever matches active rows, and
|
|
17
|
+
* restore() recomputes the key when a row comes back. Among active rows,
|
|
18
|
+
* oldest (lowest id) wins: if several already share a key (an unresolved
|
|
19
|
+
* historical duplicate), only the first holds it and the rest stay NULL
|
|
20
|
+
* rather than pointing a future reheat at an arbitrary survivor. Those
|
|
21
|
+
* clusters are what memory_audit + memory_merge are for.
|
|
22
|
+
*/
|
|
23
|
+
export declare function rebuildUniqKeys(db: Database): void;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write-time dedup key for memories (t-335).
|
|
3
|
+
*
|
|
4
|
+
* `remember()` always inserted — audit() found near-duplicates after the
|
|
5
|
+
* fact (embedding similarity) but nothing caught a character-identical
|
|
6
|
+
* restatement before it became a second row. This gives remember() a cheap
|
|
7
|
+
* floor underneath audit/MMR: normalize and hash category+project+title+content,
|
|
8
|
+
* so an exact restatement collides with the existing row's `uniq` value and
|
|
9
|
+
* reheats it instead of inserting.
|
|
10
|
+
*
|
|
11
|
+
* Normalization is deliberately minimal (PR120 review, two rounds). The key
|
|
12
|
+
* must only fold differences that can never change what a memory says, and
|
|
13
|
+
* almost every character can: a sign ("-5" vs "5"), a decimal point ("1.5"
|
|
14
|
+
* vs "15"), a hyphen in an identifier ("foo-bar" vs "foobar"), a space
|
|
15
|
+
* ("foo bar" vs "foobar"), case ("US" vs "us"), a compatibility form ("x²"
|
|
16
|
+
* vs "x2"), indentation in a code snippet. Earlier versions folded case and
|
|
17
|
+
* punctuation (and, first, everything outside [a-z0-9]); each silently merged
|
|
18
|
+
* distinct writes. What is folded now, in title and content only:
|
|
19
|
+
* - canonical Unicode equivalence (NFC — the same text, encoded two ways)
|
|
20
|
+
* - line-ending style (CRLF/CR → LF)
|
|
21
|
+
* - whitespace at the end of each line, and at the start/end of the field
|
|
22
|
+
* category and project are not normalized at all: they are identifiers, so
|
|
23
|
+
* they are compared exactly (missing project = empty string).
|
|
24
|
+
*
|
|
25
|
+
* The key also carries the memory's *effective* scope (backends/scope.ts):
|
|
26
|
+
* restating a project-scoped memory with scope: 'global' is a different
|
|
27
|
+
* write (it changes who can recall it), so it must not reheat the scoped
|
|
28
|
+
* row and drop the promotion. An explicit scope equal to the category
|
|
29
|
+
* default is the same write and still reheats.
|
|
30
|
+
*
|
|
31
|
+
* This only catches character-identical restatements (after normalization).
|
|
32
|
+
* It is not a replacement for audit's similarity-based duplicate detection —
|
|
33
|
+
* paraphrases, reworded titles, or trimmed content still need audit + merge.
|
|
34
|
+
*/
|
|
35
|
+
import { createHash } from 'node:crypto';
|
|
36
|
+
import { effectiveScope } from './scope.js';
|
|
37
|
+
function normalize(text) {
|
|
38
|
+
return text
|
|
39
|
+
.normalize('NFC')
|
|
40
|
+
.replace(/\r\n?/g, '\n')
|
|
41
|
+
.replace(/[^\S\n]+$/gm, '')
|
|
42
|
+
.trim();
|
|
43
|
+
}
|
|
44
|
+
export function computeUniqKey(input) {
|
|
45
|
+
// JSON array, not a delimiter join: no field content can alias a
|
|
46
|
+
// different split of the same characters across fields.
|
|
47
|
+
const basis = JSON.stringify([
|
|
48
|
+
input.category,
|
|
49
|
+
input.project ?? '',
|
|
50
|
+
effectiveScope(input),
|
|
51
|
+
normalize(input.title),
|
|
52
|
+
normalize(input.content),
|
|
53
|
+
]);
|
|
54
|
+
return createHash('sha256').update(basis, 'utf-8').digest('hex');
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Recompute `uniq` for every row from its current category/project/title/
|
|
58
|
+
* content with the current computeUniqKey(). Used by the migrations that
|
|
59
|
+
* introduce or re-key the column.
|
|
60
|
+
*
|
|
61
|
+
* Archived rows get NULL — remember() only ever matches active rows, and
|
|
62
|
+
* restore() recomputes the key when a row comes back. Among active rows,
|
|
63
|
+
* oldest (lowest id) wins: if several already share a key (an unresolved
|
|
64
|
+
* historical duplicate), only the first holds it and the rest stay NULL
|
|
65
|
+
* rather than pointing a future reheat at an arbitrary survivor. Those
|
|
66
|
+
* clusters are what memory_audit + memory_merge are for.
|
|
67
|
+
*/
|
|
68
|
+
export function rebuildUniqKeys(db) {
|
|
69
|
+
const rows = db
|
|
70
|
+
.prepare(
|
|
71
|
+
// scope only exists from add_scope on; a DB that predates it has no
|
|
72
|
+
// explicit overrides, so NULL (category default) is exact.
|
|
73
|
+
`SELECT id, category, project, title, content, archived,
|
|
74
|
+
${hasScopeColumn(db) ? 'scope' : 'NULL AS scope'}
|
|
75
|
+
FROM memories ORDER BY id ASC`)
|
|
76
|
+
.all();
|
|
77
|
+
const seen = new Set();
|
|
78
|
+
const update = db.prepare('UPDATE memories SET uniq = ? WHERE id = ?');
|
|
79
|
+
db.transaction(() => {
|
|
80
|
+
// Clear every key first. Rekeying row by row in place could otherwise
|
|
81
|
+
// write a row's new key while a later row still holds it as a stale old
|
|
82
|
+
// key, and trip the unique index on active rows.
|
|
83
|
+
db.prepare('UPDATE memories SET uniq = NULL').run();
|
|
84
|
+
for (const row of rows) {
|
|
85
|
+
let key = null;
|
|
86
|
+
if (row.archived === 0) {
|
|
87
|
+
const k = computeUniqKey(row);
|
|
88
|
+
if (!seen.has(k)) {
|
|
89
|
+
seen.add(k);
|
|
90
|
+
key = k;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
update.run(key, row.id);
|
|
94
|
+
}
|
|
95
|
+
})();
|
|
96
|
+
}
|
|
97
|
+
function hasScopeColumn(db) {
|
|
98
|
+
const info = db.pragma('table_info(memories)');
|
|
99
|
+
return info.some((c) => c.name === 'scope');
|
|
100
|
+
}
|
|
@@ -41,12 +41,25 @@ export const EmbeddingModel = {
|
|
|
41
41
|
MLE5Large: 'fast-multilingual-e5-large',
|
|
42
42
|
};
|
|
43
43
|
/**
|
|
44
|
-
* Where model tarballs come from. The upstream default
|
|
45
|
-
* bucket
|
|
46
|
-
*
|
|
47
|
-
*
|
|
44
|
+
* Where model tarballs come from. The upstream default was a Qdrant-owned
|
|
45
|
+
* GCS bucket; as of 2026-10-16 every object in it 403s (t-1262) and nobody
|
|
46
|
+
* at Qdrant maintains it, so it isn't coming back. The replacement is a
|
|
47
|
+
* GitHub Release asset in this repo (sleepunit-agents/loom, tag
|
|
48
|
+
* models-fast-bge-small-en-v1.5-v1): the exact bytes this box had already
|
|
49
|
+
* downloaded from the GCS bucket before it died, reuploaded rather than
|
|
50
|
+
* re-exported, and confirmed bit-identical against the
|
|
51
|
+
* embedding-runtime.parity.test.ts golden vectors before upload. That
|
|
52
|
+
* matters: the public Hugging Face re-exports of this model are either
|
|
53
|
+
* int8/fp16-quantized or built with a different ONNX optimizer pass, and
|
|
54
|
+
* produce embeddings close to but not bit-identical with every vector
|
|
55
|
+
* already stored in a loom database. Only fast-bge-small-en-v1.5 — the
|
|
56
|
+
* model loom actually defaults to (src/backends/index.ts) — has a release
|
|
57
|
+
* asset; other EmbeddingModel entries will fail download() with a clear
|
|
58
|
+
* "set LOOM_MODEL_BASE_URL" error until someone needs one and mirrors it
|
|
59
|
+
* the same way. The env override exists so a dead host is a config change,
|
|
60
|
+
* not a release.
|
|
48
61
|
*/
|
|
49
|
-
const DEFAULT_MODEL_BASE_URL = 'https://
|
|
62
|
+
const DEFAULT_MODEL_BASE_URL = 'https://github.com/sleepunit-agents/loom/releases/download/models-fast-bge-small-en-v1.5-v1';
|
|
50
63
|
/**
|
|
51
64
|
* Loaded dense embedder: a tokenizer plus an ONNX inference session.
|
|
52
65
|
*
|
|
@@ -238,6 +251,12 @@ function download(outputPath, model) {
|
|
|
238
251
|
: model;
|
|
239
252
|
const base = process.env.LOOM_MODEL_BASE_URL ?? DEFAULT_MODEL_BASE_URL;
|
|
240
253
|
const url = `${base}/${remoteName}.tar.gz`;
|
|
254
|
+
// GitHub Release assets (the new default host) 302 to a signed, short-lived
|
|
255
|
+
// blob URL; the GCS bucket they replace never redirected, so this following
|
|
256
|
+
// logic didn't previously exist.
|
|
257
|
+
return fetchToFile(url, outputPath, 5);
|
|
258
|
+
}
|
|
259
|
+
function fetchToFile(url, outputPath, redirectsLeft) {
|
|
241
260
|
return new Promise((resolve, reject) => {
|
|
242
261
|
const fail = (err) => {
|
|
243
262
|
// A partial archive on disk would be treated as a complete download on
|
|
@@ -246,15 +265,25 @@ function download(outputPath, model) {
|
|
|
246
265
|
unlinkSync(outputPath);
|
|
247
266
|
reject(err);
|
|
248
267
|
};
|
|
249
|
-
const file = createWriteStream(outputPath);
|
|
250
|
-
file.on('error', fail);
|
|
251
268
|
httpsGet(url, { headers: { 'User-Agent': 'loom' } }, (response) => {
|
|
252
|
-
|
|
269
|
+
const { statusCode, headers } = response;
|
|
270
|
+
if (statusCode && statusCode >= 300 && statusCode < 400 && headers.location) {
|
|
271
|
+
response.resume();
|
|
272
|
+
if (redirectsLeft <= 0) {
|
|
273
|
+
fail(new Error(`Model download failed: too many redirects fetching ${url.toString()}`));
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
fetchToFile(new URL(headers.location, url), outputPath, redirectsLeft - 1).then(resolve, fail);
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
if (statusCode !== 200) {
|
|
253
280
|
response.resume();
|
|
254
|
-
fail(new Error(`Model download failed: ${url} returned HTTP ${
|
|
281
|
+
fail(new Error(`Model download failed: ${url.toString()} returned HTTP ${statusCode}. ` +
|
|
255
282
|
'Set LOOM_MODEL_BASE_URL to fetch models from a different host.'));
|
|
256
283
|
return;
|
|
257
284
|
}
|
|
285
|
+
const file = createWriteStream(outputPath);
|
|
286
|
+
file.on('error', fail);
|
|
258
287
|
response.on('error', fail);
|
|
259
288
|
response.pipe(file);
|
|
260
289
|
file.on('finish', () => {
|
|
@@ -93,7 +93,7 @@ function openMemoriesDb(contextDir) {
|
|
|
93
93
|
if (!existsSync(dbPath))
|
|
94
94
|
return null;
|
|
95
95
|
const db = new BetterSqlite3(dbPath, { readonly: true });
|
|
96
|
-
db.pragma('busy_timeout =
|
|
96
|
+
db.pragma('busy_timeout = 5000');
|
|
97
97
|
return db;
|
|
98
98
|
}
|
|
99
99
|
/** Raw episode rows in the window, oldest first. Empty when there is no store. */
|
|
@@ -48,11 +48,30 @@ export class FastEmbedProvider {
|
|
|
48
48
|
if (texts.length === 0)
|
|
49
49
|
return [];
|
|
50
50
|
const embedder = await this.ensureEmbedder();
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
// Staged fallback ladder (t-334): a batch-level failure anywhere in a
|
|
52
|
+
// large request must not take remember()/recall() down with it. Each
|
|
53
|
+
// rung is strictly less efficient but more isolated than the one above
|
|
54
|
+
// it; only descend when the rung above actually failed or came back
|
|
55
|
+
// short.
|
|
56
|
+
const rungs = [
|
|
57
|
+
() => collectBatches(embedder.embed(texts, 32)), // 1. one batched call
|
|
58
|
+
() => collectBatches(embedder.embed(texts, 1)), // 2. sequential, still whole-array
|
|
59
|
+
() => embedOneByOne(texts, (t) => this.embed(t)), // 3. isolated, one call per text
|
|
60
|
+
];
|
|
61
|
+
let lastErr;
|
|
62
|
+
for (const runRung of rungs) {
|
|
63
|
+
try {
|
|
64
|
+
const vectors = await runRung();
|
|
65
|
+
if (vectors.length === texts.length && vectors.every((v) => v.length > 0)) {
|
|
66
|
+
return vectors;
|
|
67
|
+
}
|
|
68
|
+
lastErr = new Error(`fastembed: embedBatch() produced ${vectors.length} vectors for ${texts.length} inputs (model ${this.config.model})`);
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
lastErr = err;
|
|
72
|
+
}
|
|
54
73
|
}
|
|
55
|
-
|
|
74
|
+
throw lastErr;
|
|
56
75
|
}
|
|
57
76
|
async embedQuery(text) {
|
|
58
77
|
const embedder = await this.ensureEmbedder();
|
|
@@ -93,3 +112,10 @@ async function collectBatches(gen) {
|
|
|
93
112
|
}
|
|
94
113
|
return out;
|
|
95
114
|
}
|
|
115
|
+
/** Rung 3 of the embedBatch fallback ladder: one isolated call per text. */
|
|
116
|
+
async function embedOneByOne(texts, embedOne) {
|
|
117
|
+
const out = [];
|
|
118
|
+
for (const text of texts)
|
|
119
|
+
out.push(await embedOne(text));
|
|
120
|
+
return out;
|
|
121
|
+
}
|