memhtml 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -72
- package/agent/instructions.md +34 -10
- package/dist/dist-CBhYV3up.mjs +3 -0
- package/dist/{dist-DgKlozi6.mjs → dist-D1wH0oJ0.mjs} +3353 -901
- package/dist/dist-D1wH0oJ0.mjs.map +1 -0
- package/dist/{dist-BCsav-EP.mjs → dist-DHFdTnlp.mjs} +825 -249
- package/dist/dist-DHFdTnlp.mjs.map +1 -0
- package/dist/memhtml-mcp.mjs +1280 -343
- package/dist/memhtml-mcp.mjs.map +1 -1
- package/dist/memhtml.mjs +1221 -376
- package/dist/memhtml.mjs.map +1 -1
- package/migrations/0007_watermark.sql +4 -2
- package/migrations/0011_edge_indexes.sql +78 -0
- package/migrations/0012_origin_path.sql +21 -0
- package/package.json +10 -10
- package/src/agent-build.ts +293 -21
- package/src/child-stderr.ts +36 -0
- package/src/client.ts +284 -171
- package/src/contract.ts +317 -74
- package/src/mount.ts +31 -7
- package/src/run-auth.ts +18 -15
- package/state-migrations/S0002_entity_corroboration.sql +13 -7
- package/dist/dist-BCsav-EP.mjs.map +0 -1
- package/dist/dist-DgKlozi6.mjs.map +0 -1
- package/dist/dist-DuzGralO.mjs +0 -3
|
@@ -9,8 +9,10 @@ CREATE TABLE index_state (
|
|
|
9
9
|
head_sha TEXT,
|
|
10
10
|
-- The vector space, as `<model-id>@<dim>` (@memhtml/llm's EMBED_WATERMARK). A mismatch against
|
|
11
11
|
-- configuration is a hard refusal, never a silent reindex: a half-migrated vector space degrades
|
|
12
|
-
-- every cosine and is invisible in tests.
|
|
13
|
-
-- truncates `embeddings`
|
|
12
|
+
-- every cosine and is invisible in tests. `rebuild --embed` is the one path past that refusal: it
|
|
13
|
+
-- truncates `embeddings` with the other memory tables and records the configured space before any
|
|
14
|
+
-- vector is written, so it migrates the whole space rather than mixing two. `--no-embed` refuses
|
|
15
|
+
-- before the truncate, so a store that refuses keeps the vectors it has.
|
|
14
16
|
embed_model TEXT NOT NULL,
|
|
15
17
|
embed_dim INTEGER NOT NULL CHECK (embed_dim > 0),
|
|
16
18
|
rebuilt_at TEXT NOT NULL,
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
-- `edges_src` and `edges_dst` carry NO predicate: `(src_path, edge_class)` and `(dst_path, edge_class)`
|
|
2
|
+
-- over every row, authored and derived alike. That is what the memory-graph walk needs, and a partial
|
|
3
|
+
-- index on `derived = 0` cannot serve it at all.
|
|
4
|
+
--
|
|
5
|
+
-- ── Why a predicate makes the index unreachable ───────────────────────────────────────────────────
|
|
6
|
+
--
|
|
7
|
+
-- SQLite may use a partial index only when the query's WHERE clause IMPLIES the index's WHERE clause.
|
|
8
|
+
-- The neighbors walk (`neighborsQuery`, `apps/cli/src/operations.ts`) filters
|
|
9
|
+
-- `src_path = ?1 AND edge_class = 'memory'` and SELECTS `e.derived`, because it wants BOTH kinds of
|
|
10
|
+
-- edge: sleep-mined edges are what lateral retrieval is for, and each node reports which kind reached
|
|
11
|
+
-- it. `edge_class = 'memory'` does not imply `derived = 0`, so an index declared `WHERE derived = 0` is
|
|
12
|
+
-- simply not a candidate.
|
|
13
|
+
--
|
|
14
|
+
-- Measured 2026-08-26 on node 24.19.0's `node:sqlite` with these migrations applied and no `ANALYZE`,
|
|
15
|
+
-- over the walk's own statement at depth 2. With the predicate:
|
|
16
|
+
--
|
|
17
|
+
-- * the two `src_path = ?1` arms probe `sqlite_autoindex_edges_1`, because `src_path` leads
|
|
18
|
+
-- `PRIMARY KEY (src_path, rel, dst_path)`, which binds ONE column;
|
|
19
|
+
-- * the `dst_path = ?1` hop-1 arm is `SCAN e`, a full pass over `edges`;
|
|
20
|
+
-- * the `dst_path`-driven hop-2 arm is `SCAN e2 USING INDEX sqlite_autoindex_edges_1`, a full index
|
|
21
|
+
-- scan of the table per outer row.
|
|
22
|
+
--
|
|
23
|
+
-- Without it, all four arms are `SEARCH … (src_path=? AND edge_class=?)` or
|
|
24
|
+
-- `(dst_path=? AND edge_class=?)`: two bound columns instead of one, in every direction.
|
|
25
|
+
--
|
|
26
|
+
-- ── A REPLACEMENT, and the per-statement census behind that ──────────────────────────────────────
|
|
27
|
+
--
|
|
28
|
+
-- Measured the same way over every statement in the tree that reads `edges` — the walk, both
|
|
29
|
+
-- authored-only anti-joins in `@memhtml/sleep` (`sharedEntityPairs`, `minedPairs`), `retentionEdgeCounts`,
|
|
30
|
+
-- `memoryEdges`, `deepGroupingEdges`, `inboundAuthoredEdges`, `danglingEdges`, retrieval's
|
|
31
|
+
-- `superseded_by` subquery, `doctor`'s stale-blocker join, `task list`'s blockers column, the indexer's
|
|
32
|
+
-- delete-by-source, and `movePath`'s `UPDATE edges SET src_path`. FIVE change plan, and every one of the
|
|
33
|
+
-- five gets a probe it did not have or binds a column more:
|
|
34
|
+
--
|
|
35
|
+
-- * both `dst_path` walk arms, `SCAN e` -> `edges_dst (dst_path=? AND edge_class=?)`;
|
|
36
|
+
-- * both `src_path` walk arms, `sqlite_autoindex_edges_1 (src_path=?)` -> `edges_src (src_path=? AND
|
|
37
|
+
-- edge_class=?)`;
|
|
38
|
+
-- * `retentionEdgeCounts`, `SCAN e` plus a temp b-tree -> `SCAN e USING COVERING INDEX edges_dst`, so
|
|
39
|
+
-- it stops building an `AUTOMATIC PARTIAL COVERING INDEX` per call;
|
|
40
|
+
-- * `task list`'s blockers subquery, `edges_rel (rel, edge_class)` -> `edges_dst (dst_path,
|
|
41
|
+
-- edge_class)`, binding a path rather than the rel `'blocks'`;
|
|
42
|
+
-- * the indexer's delete-by-source and `movePath`'s update, `sqlite_autoindex_edges_1 (src_path=?)` ->
|
|
43
|
+
-- `edges_src (src_path=?)`.
|
|
44
|
+
--
|
|
45
|
+
-- The rest are byte-identical, including retrieval's `superseded_by` subquery, which probes
|
|
46
|
+
-- `edges_dst (dst_path=? AND edge_class=?)` under BOTH shapes — it names the class, so it never depended
|
|
47
|
+
-- on the predicate. The `derived = 0` readers that do not name a path reach for `edges_derived (derived,
|
|
48
|
+
-- rel)`, which binds two columns where either directional pair binds one.
|
|
49
|
+
--
|
|
50
|
+
-- The predicate is not free to keep, and the cost is not the index COUNT: `edges` carries four either
|
|
51
|
+
-- way. It is that a partial index holds only the authored rows. Dropping the predicate puts every mined
|
|
52
|
+
-- edge in both b-trees, and mining writes thousands per run — measured 2026-08-26, 20,000 derived-edge
|
|
53
|
+
-- inserts in one transaction cost 158-162 ms with the predicate and 184-188 ms without it, three
|
|
54
|
+
-- repetitions, so roughly 14-19% more per mined edge. That is the price of the walk's two probes, paid
|
|
55
|
+
-- knowingly: a walk arm was a full pass over `edges` per hop.
|
|
56
|
+
--
|
|
57
|
+
-- ── ADDITIVE, in 0009's sense ────────────────────────────────────────────────────────────────────
|
|
58
|
+
--
|
|
59
|
+
-- `DROP INDEX` plus `CREATE INDEX`, and deliberately NOT 0008's recreate-and-copy. That pattern was
|
|
60
|
+
-- forced by a CHECK-constraint edit, which SQLite cannot `ALTER`, and it carried real risk:
|
|
61
|
+
-- `DROP TABLE edges` would cascade nothing here, but `DROP TABLE files` cascades to `embeddings` and
|
|
62
|
+
-- 0008 had to snapshot six tables to avoid re-paying Bedrock for the whole corpus. An index is derived
|
|
63
|
+
-- data with no rows of its own — dropping one loses nothing and recreating one costs a single scan of
|
|
64
|
+
-- `edges` — so none of that machinery applies.
|
|
65
|
+
--
|
|
66
|
+
-- The NAMES are reused because the name states which column the index leads with, which is what a
|
|
67
|
+
-- reader of a query plan needs from it, and that has not changed.
|
|
68
|
+
--
|
|
69
|
+
-- `IF EXISTS` on the drops, so this file converges from any starting state a real store can be in —
|
|
70
|
+
-- including one where an operator dropped an index by hand in `sqlite3`. A bare `DROP` there would fail
|
|
71
|
+
-- the migration inside its transaction and leave that store unable to open at all. The `CREATE`s carry
|
|
72
|
+
-- no such guard on purpose: if either name is still taken when they run, that is a fact worth failing on.
|
|
73
|
+
|
|
74
|
+
DROP INDEX IF EXISTS edges_src;
|
|
75
|
+
DROP INDEX IF EXISTS edges_dst;
|
|
76
|
+
|
|
77
|
+
CREATE INDEX edges_src ON edges (src_path, edge_class);
|
|
78
|
+
CREATE INDEX edges_dst ON edges (dst_path, edge_class);
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
-- The archive mapping, read backwards. `origin_path` holds the pre-archive path of a file under
|
|
2
|
+
-- `archive/<YYYY>/`, derived from the path itself by the projection, and NULL for an active file.
|
|
3
|
+
--
|
|
4
|
+
-- Without an index, answering "what became of the memory that lived at areas/oncall/x.html" is a
|
|
5
|
+
-- `SCAN files` — the one question an external citation asks, since a correction with a reworded title
|
|
6
|
+
-- moves the memory to a new path and leaves the cited one holding nothing. Measured 2026-08-26 on
|
|
7
|
+
-- node 24's `node:sqlite` with no ANALYZE: `SCAN files` becomes `SEARCH files USING INDEX
|
|
8
|
+
-- files_origin (origin_path=?)`.
|
|
9
|
+
--
|
|
10
|
+
-- PARTIAL on purpose, and the predicate IS reachable from the query: SQLite proves `origin_path = ?`
|
|
11
|
+
-- implies `origin_path IS NOT NULL`, so the probe above is what a partial index yields (probed the
|
|
12
|
+
-- same day, both forms). Most of a corpus is active, so the partial form keeps every live file out of
|
|
13
|
+
-- an index that only archived files can answer from. Contrast `0011_edge_indexes.sql`, where the
|
|
14
|
+
-- predicate had to GO: there the readers filter on a column the predicate does not mention, and an
|
|
15
|
+
-- implication SQLite cannot prove makes the index no candidate at all.
|
|
16
|
+
--
|
|
17
|
+
-- No UNIQUE. One path can be archived more than once — written, evicted, written again at the same
|
|
18
|
+
-- path, evicted again — and each eviction lands in its own `archive/<YYYY>/` partition, so two rows
|
|
19
|
+
-- may legitimately carry one `origin_path`. The reader orders by `archived_at` and takes the newest.
|
|
20
|
+
|
|
21
|
+
CREATE INDEX files_origin ON files (origin_path) WHERE origin_path IS NOT NULL;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "memhtml",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "An agent's long-term memory: one fact per semantic HTML file in git, four-arm retrieval, and a
|
|
3
|
+
"version": "0.7.0",
|
|
4
|
+
"description": "An agent's long-term memory: one fact per semantic HTML file in git, four-arm retrieval, and a curation sleep cycle.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"memory",
|
|
7
7
|
"agent",
|
|
@@ -40,16 +40,16 @@
|
|
|
40
40
|
"LICENSE"
|
|
41
41
|
],
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@ai-sdk/amazon-bedrock": "5.0.
|
|
44
|
-
"@aws-sdk/client-bedrock-runtime": "3.
|
|
43
|
+
"@ai-sdk/amazon-bedrock": "5.0.61",
|
|
44
|
+
"@aws-sdk/client-bedrock-runtime": "3.1116.0",
|
|
45
45
|
"@aws/bedrock-token-generator": "1.1.0",
|
|
46
|
-
"@effect/platform-node": "4.0.0-rc.
|
|
47
|
-
"@effect/platform-node-shared": "4.0.0-rc.
|
|
48
|
-
"ai": "7.0.
|
|
49
|
-
"effect": "4.0.0-rc.
|
|
50
|
-
"eve": "0.
|
|
46
|
+
"@effect/platform-node": "4.0.0-rc.111",
|
|
47
|
+
"@effect/platform-node-shared": "4.0.0-rc.111",
|
|
48
|
+
"ai": "7.0.77",
|
|
49
|
+
"effect": "4.0.0-rc.111",
|
|
50
|
+
"eve": "0.44.1",
|
|
51
51
|
"highlight.js": "11.11.2",
|
|
52
|
-
"just-bash": "3.
|
|
52
|
+
"just-bash": "3.4.2",
|
|
53
53
|
"node-html-parser": "9.0.1",
|
|
54
54
|
"parse5": "8.0.1",
|
|
55
55
|
"zod": "4.4.3"
|
package/src/agent-build.ts
CHANGED
|
@@ -1,11 +1,22 @@
|
|
|
1
1
|
import { spawn } from "node:child_process"
|
|
2
2
|
import { existsSync } from "node:fs"
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
cp,
|
|
5
|
+
mkdir,
|
|
6
|
+
readdir,
|
|
7
|
+
readFile,
|
|
8
|
+
rename,
|
|
9
|
+
rm,
|
|
10
|
+
stat,
|
|
11
|
+
symlink,
|
|
12
|
+
writeFile
|
|
13
|
+
} from "node:fs/promises"
|
|
4
14
|
import { createRequire } from "node:module"
|
|
5
15
|
import { homedir } from "node:os"
|
|
6
16
|
import { dirname, join, resolve } from "node:path"
|
|
7
17
|
import { Effect } from "effect"
|
|
8
18
|
|
|
19
|
+
import { appendStderrTail, stderrMessageTail } from "./child-stderr.js"
|
|
9
20
|
import { ConsolidatorUnavailable } from "./contract.js"
|
|
10
21
|
|
|
11
22
|
/**
|
|
@@ -30,6 +41,33 @@ import { ConsolidatorUnavailable } from "./contract.js"
|
|
|
30
41
|
* (`server/node_modules/node-liblzma/build/Release/node_lzma.node`) and eve says so itself — "Ensure
|
|
31
42
|
* your production environment matches the builder OS and architecture (linux-x64)". A published
|
|
32
43
|
* artifact cannot carry one platform's binaries.
|
|
44
|
+
*
|
|
45
|
+
* ## A finished build belongs to the directory it was built in
|
|
46
|
+
*
|
|
47
|
+
* `eve build` writes the ABSOLUTE path of its build directory into its own output: `appRoot` and
|
|
48
|
+
* `agentRoot` in the `manifest` literal inside `.output/server/index.mjs`, taken from the process cwd
|
|
49
|
+
* (eve offers no root flag — `dist/src/cli/application-root.js` derives the root from
|
|
50
|
+
* `process.cwd()`). And `eve start` does not merely carry those strings: it RE-BUNDLES the authored
|
|
51
|
+
* TypeScript found at `<agentRoot>/agent.ts` on first load
|
|
52
|
+
* (`dist/src/internal/authored-module-loader.js`) and writes the resulting bundle into a cache
|
|
53
|
+
* directory it creates under that same root. Three constraints follow, and the third is the one a
|
|
54
|
+
* reader is likeliest to break:
|
|
55
|
+
*
|
|
56
|
+
* 1. The directory `eve build` ran in is the only directory `eve start` can serve. A finished build
|
|
57
|
+
* that is moved or renamed makes eve's `resolveAuthoredPackageRoot` walk the vanished path looking
|
|
58
|
+
* for a `package.json`, reach `/`, and exit 1 on `Failed to resolve the authored package root for
|
|
59
|
+
* "…/agent/agent.ts"`.
|
|
60
|
+
* 2. That directory must still hold the agent SOURCE, not just `.output/`. A tree published with
|
|
61
|
+
* `.output/` alone fails identically, because the source is what gets re-bundled.
|
|
62
|
+
* 3. That directory must stay WRITABLE for the server's whole life, since the bundle cache is written
|
|
63
|
+
* on first load rather than at build time.
|
|
64
|
+
*
|
|
65
|
+
* Probed live 2026-08-25 against eve 0.38.3: a build that answered `/eve/v1/health` where it was built
|
|
66
|
+
* exited 1 with that message after nothing but a `rename` of its directory, its baked `appRoot` still
|
|
67
|
+
* naming the old path.
|
|
68
|
+
*
|
|
69
|
+
* So the build runs AT the cache root and is never built elsewhere and moved in. What makes an
|
|
70
|
+
* unfinished build detectable without a move is {@link BUILD_COMPLETE_MARKER}, written last.
|
|
33
71
|
*/
|
|
34
72
|
|
|
35
73
|
/**
|
|
@@ -41,8 +79,9 @@ import { ConsolidatorUnavailable } from "./contract.js"
|
|
|
41
79
|
*
|
|
42
80
|
* Resolution goes through the MANIFEST, not the bin. `resolve("eve/bin/eve.js")` raises
|
|
43
81
|
* `ERR_PACKAGE_PATH_NOT_EXPORTED`: eve's `exports` map declares no `./bin/*` subpath, so node refuses
|
|
44
|
-
* the deep path even though the file is there
|
|
45
|
-
*
|
|
82
|
+
* the deep path even though the file is there. `tests/start-port.test.ts` re-proves both halves
|
|
83
|
+
* against the INSTALLED eve on every run — the deep path refused, `./package.json` exported with a
|
|
84
|
+
* real `bin` beside it — so an eve release that changes either fails there.
|
|
46
85
|
*/
|
|
47
86
|
export const eveBinPath = (): string | null => {
|
|
48
87
|
const require = createRequire(import.meta.url)
|
|
@@ -61,6 +100,57 @@ export const eveBinPath = (): string | null => {
|
|
|
61
100
|
const cacheRootFor = (version: string): string =>
|
|
62
101
|
join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "memhtml", "eve", version)
|
|
63
102
|
|
|
103
|
+
/**
|
|
104
|
+
* The file whose PRESENCE says the cache directory holds a COMPLETED build.
|
|
105
|
+
*
|
|
106
|
+
* `.output/` existing cannot say that: a process killed while the tree was being staged or built
|
|
107
|
+
* leaves a partial directory that an existence check reads as complete — forever, because nothing
|
|
108
|
+
* would ever rebuild it, and `eve start` over a partial tree is a server that fails in whatever way
|
|
109
|
+
* the missing half implies. This marker is written LAST, only after `eve build` exits 0 with its
|
|
110
|
+
* {@link BUILT_SERVER_ENTRY} verified on disk, and it is the ONLY thing {@link cacheBuildComplete}
|
|
111
|
+
* trusts. A cache directory without it, whatever else it holds, is a partial to discard and rebuild.
|
|
112
|
+
*
|
|
113
|
+
* Writing it last is what a publishing `rename` would otherwise buy, and it is the shape that is
|
|
114
|
+
* compatible with an output which cannot be relocated (see the note at the top of this file). It is
|
|
115
|
+
* also the finalizer's discriminator: a markerless cache root is this build's own wreckage and gets
|
|
116
|
+
* removed, a marked one is a finished build and never does.
|
|
117
|
+
*/
|
|
118
|
+
const BUILD_COMPLETE_MARKER = ".memhtml-build-complete"
|
|
119
|
+
|
|
120
|
+
/** Where a completed build's marker sits. Exported logic's one source of the path. */
|
|
121
|
+
const buildMarkerPath = (cacheRoot: string): string => join(cacheRoot, BUILD_COMPLETE_MARKER)
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The file `eve start` serves, relative to a built root.
|
|
125
|
+
*
|
|
126
|
+
* A build is verified against THIS PATH rather than against `.output/`, because `eve build` exiting 0
|
|
127
|
+
* is not the same claim as `eve build` having emitted a server. An empty-but-present `.output/` earns
|
|
128
|
+
* the completion marker under a directory check, and the marker is permanent — so the box would serve
|
|
129
|
+
* an app with no entry point for that version's whole life. It is the "a scanner can exit 0 having
|
|
130
|
+
* produced nothing" hazard in build form, and the entry file is the artifact whose absence a boot
|
|
131
|
+
* would discover.
|
|
132
|
+
*/
|
|
133
|
+
const BUILT_SERVER_ENTRY = join(".output", "server", "index.mjs")
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* How old a build lock may be before another process takes it over.
|
|
137
|
+
*
|
|
138
|
+
* The lock (a `mkdir`-ed sibling directory) is held for one stage-plus-build, measured in tens of
|
|
139
|
+
* seconds for the ~17 MB output. Ten minutes says its holder is dead — killed between `mkdir` and
|
|
140
|
+
* the `finally` that removes it — rather than slow, and a dead holder's lock would otherwise block
|
|
141
|
+
* every future run on this box for this version.
|
|
142
|
+
*/
|
|
143
|
+
const BUILD_LOCK_STALE_MS = 10 * 60_000
|
|
144
|
+
|
|
145
|
+
/** How often a waiting process re-checks the marker and the lock. */
|
|
146
|
+
const BUILD_LOCK_POLL_MS = 500
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* How long a process waits on another's build before giving up. Stale takeover happens well before
|
|
150
|
+
* this; the budget only binds when a LIVE holder builds for longer than the stale age plus a poll.
|
|
151
|
+
*/
|
|
152
|
+
const BUILD_WAIT_BUDGET_MS = BUILD_LOCK_STALE_MS + 60_000
|
|
153
|
+
|
|
64
154
|
/** A bare specifier's package name: two segments when scoped, one otherwise. */
|
|
65
155
|
const packageOf = (specifier: string): string => {
|
|
66
156
|
const parts = specifier.split("/")
|
|
@@ -201,10 +291,12 @@ const runEveBuild = (input: {
|
|
|
201
291
|
cwd: input.cwd,
|
|
202
292
|
stdio: ["ignore", "ignore", "pipe"]
|
|
203
293
|
})
|
|
294
|
+
// Only a bounded TAIL is retained, and the failure message below renders the END of it. Both
|
|
295
|
+
// rules are `child-stderr.ts`'s, shared with the `eve start` child in `client.ts`.
|
|
204
296
|
let stderr = ""
|
|
205
297
|
child.stderr.setEncoding("utf8")
|
|
206
298
|
child.stderr.on("data", (chunk: string) => {
|
|
207
|
-
stderr
|
|
299
|
+
stderr = appendStderrTail(stderr, chunk)
|
|
208
300
|
})
|
|
209
301
|
child.once("error", (cause) => {
|
|
210
302
|
resume(
|
|
@@ -219,7 +311,7 @@ const runEveBuild = (input: {
|
|
|
219
311
|
? Effect.void
|
|
220
312
|
: Effect.fail(
|
|
221
313
|
ConsolidatorUnavailable.make({
|
|
222
|
-
reason: `eve build exited with code ${String(code)} in ${input.cwd}. ${stderr
|
|
314
|
+
reason: `eve build exited with code ${String(code)} in ${input.cwd}. ${stderrMessageTail(stderr)}`
|
|
223
315
|
})
|
|
224
316
|
)
|
|
225
317
|
)
|
|
@@ -229,6 +321,118 @@ const runEveBuild = (input: {
|
|
|
229
321
|
})
|
|
230
322
|
})
|
|
231
323
|
|
|
324
|
+
/**
|
|
325
|
+
* Whether a cache directory holds a COMPLETED build. The marker is the answer; `.output/` alone is
|
|
326
|
+
* not, because a killed `eve build` leaves a partial `.output/` behind. See
|
|
327
|
+
* {@link BUILD_COMPLETE_MARKER}, which is written only beside a verified {@link BUILT_SERVER_ENTRY}.
|
|
328
|
+
*/
|
|
329
|
+
export const cacheBuildComplete = (cacheRoot: string): boolean =>
|
|
330
|
+
existsSync(buildMarkerPath(cacheRoot)) && existsSync(join(cacheRoot, ".output"))
|
|
331
|
+
|
|
332
|
+
/** A held build lock: the directory to remove when done. */
|
|
333
|
+
interface BuildLock {
|
|
334
|
+
readonly release: () => Promise<void>
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Move a lock believed stale out of the way, and refuse to move any other lock.
|
|
339
|
+
*
|
|
340
|
+
* ## `rename` is the arbitration; an `rm` is not
|
|
341
|
+
*
|
|
342
|
+
* Two waiters can measure the same stale lock and both decide to take it over. An unconditional
|
|
343
|
+
* `rm(lockDir)` there is not an arbitration at all — it says nothing about WHICH directory it removed,
|
|
344
|
+
* so the ordering `stat(A), stat(B), rm(A), mkdir(A), rm(B), mkdir(B)` leaves A and B both holding: B's
|
|
345
|
+
* `rm` deleted the fresh lock A had just created, and B's `mkdir` then succeeded. `rename` narrows
|
|
346
|
+
* that: for one directory instance exactly one racer's rename can succeed, so the loser gets ENOENT and
|
|
347
|
+
* returns to the `mkdir`, where the winner's fresh lock excludes it.
|
|
348
|
+
*
|
|
349
|
+
* ## The inode is what binds the rename to the lock that was MEASURED
|
|
350
|
+
*
|
|
351
|
+
* `rename` alone still moves whatever sits at the path. A waiter's staleness reading is taken before
|
|
352
|
+
* its rename, and in between the takeover winner can have released and a third process can have created
|
|
353
|
+
* a fresh lock at the same path — renaming THAT aside would delete a live holder's lock and hand this
|
|
354
|
+
* waiter a second, concurrent hold, which is the same defect one step later. So a claim whose renamed
|
|
355
|
+
* directory is not the inode the staleness was read from is put straight back and this waiter acquires
|
|
356
|
+
* nothing; only the measured directory is ever discarded.
|
|
357
|
+
*
|
|
358
|
+
* The residual is the moment between such a mistaken rename and its restore, during which the path is
|
|
359
|
+
* empty and a waiter arriving at the top of the loop can `mkdir` it. That window is microseconds of
|
|
360
|
+
* filesystem calls and it costs at most what the previous shape cost always.
|
|
361
|
+
*
|
|
362
|
+
* Exported for `tests/agent-build.test.ts`, which drives both arms directly: the interleaving above
|
|
363
|
+
* cannot be forced through {@link acquireBuildLock} from one process.
|
|
364
|
+
*/
|
|
365
|
+
export const claimStaleLock = async (lockDir: string, staleIno: number): Promise<void> => {
|
|
366
|
+
const aside = `${lockDir}.stale-${String(process.pid)}`
|
|
367
|
+
await rm(aside, { recursive: true, force: true }).catch(() => {})
|
|
368
|
+
const claimed = await rename(lockDir, aside).then(
|
|
369
|
+
() => true,
|
|
370
|
+
() => false
|
|
371
|
+
)
|
|
372
|
+
if (!claimed) return
|
|
373
|
+
const moved = await stat(aside).then(
|
|
374
|
+
(stats) => stats.ino,
|
|
375
|
+
() => null
|
|
376
|
+
)
|
|
377
|
+
if (moved !== staleIno) {
|
|
378
|
+
await rename(aside, lockDir).catch(() => {})
|
|
379
|
+
return
|
|
380
|
+
}
|
|
381
|
+
await rm(aside, { recursive: true, force: true }).catch(() => {})
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Take the per-version build lock, waiting out or taking over another holder.
|
|
386
|
+
*
|
|
387
|
+
* `mkdir` without `recursive` is the primitive: it either creates the directory (the lock is ours)
|
|
388
|
+
* or throws `EEXIST` (someone holds it), atomically, on every filesystem node runs on. Two runs on
|
|
389
|
+
* one box CAN race here — the sleep cycle and a hand-driven `memhtml` both resolving the same
|
|
390
|
+
* unbuilt version — and without the lock both would build into the shared cache root at once,
|
|
391
|
+
* interleaving two `eve build`s' output.
|
|
392
|
+
*
|
|
393
|
+
* A holder that died between its `mkdir` and its `release` (SIGKILL leaves no `finally`) is detected
|
|
394
|
+
* by the lock directory's AGE: past {@link BUILD_LOCK_STALE_MS} it cannot be a live build, so the
|
|
395
|
+
* waiter claims it through {@link claimStaleLock} and retries the `mkdir`. The claim is a `rename`
|
|
396
|
+
* bound to the inode the staleness was measured on, and that binding is what keeps two waiters from
|
|
397
|
+
* both ending up holding: see that function for the interleaving an unconditional `rm` admits.
|
|
398
|
+
*
|
|
399
|
+
* Exported for `tests/agent-build.test.ts`, which proves the lock excludes and the stale takeover
|
|
400
|
+
* fires; no production caller outside {@link resolveAgentAppRoot} reaches it.
|
|
401
|
+
*/
|
|
402
|
+
export const acquireBuildLock = async (cacheRoot: string): Promise<BuildLock> => {
|
|
403
|
+
const lockDir = `${cacheRoot}.lock`
|
|
404
|
+
// The lock is taken before anything else touches the cache tree, so its parent may not exist yet.
|
|
405
|
+
// Created separately from the lock itself: `recursive: true` on the lock mkdir would report
|
|
406
|
+
// success on an ALREADY-EXISTING directory, which is exactly the case the lock must refuse.
|
|
407
|
+
await mkdir(dirname(lockDir), { recursive: true })
|
|
408
|
+
const deadline = Date.now() + BUILD_WAIT_BUDGET_MS
|
|
409
|
+
for (;;) {
|
|
410
|
+
try {
|
|
411
|
+
await mkdir(lockDir)
|
|
412
|
+
return { release: () => rm(lockDir, { recursive: true, force: true }).catch(() => {}) }
|
|
413
|
+
} catch (cause) {
|
|
414
|
+
if ((cause as { readonly code?: string }).code !== "EEXIST") throw cause
|
|
415
|
+
}
|
|
416
|
+
// The inode travels with the age, because the claim below acts on the directory this reading
|
|
417
|
+
// describes and not merely on the path it sits at.
|
|
418
|
+
const held = await stat(lockDir).then(
|
|
419
|
+
(stats) => ({ age: Date.now() - stats.mtimeMs, ino: stats.ino }),
|
|
420
|
+
() => null
|
|
421
|
+
)
|
|
422
|
+
if (held !== null && held.age > BUILD_LOCK_STALE_MS) {
|
|
423
|
+
await claimStaleLock(lockDir, held.ino)
|
|
424
|
+
continue
|
|
425
|
+
}
|
|
426
|
+
if (Date.now() >= deadline) {
|
|
427
|
+
throw new Error(
|
|
428
|
+
`another process has held the build lock ${lockDir} past the wait budget; ` +
|
|
429
|
+
"remove it if no eve build is running"
|
|
430
|
+
)
|
|
431
|
+
}
|
|
432
|
+
await new Promise((done) => setTimeout(done, BUILD_LOCK_POLL_MS))
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
232
436
|
/**
|
|
233
437
|
* The directory `eve start` will be run in, building the agent first when nothing has.
|
|
234
438
|
*
|
|
@@ -236,6 +440,27 @@ const runEveBuild = (input: {
|
|
|
236
440
|
* package that already holds `.output/` is a checkout where `build:agent` has run, and reusing it keeps
|
|
237
441
|
* development behavior byte-identical. Only the remaining case — an installed package with no output —
|
|
238
442
|
* materializes the cache directory, and it costs one ~17 MB build per version rather than one per run.
|
|
443
|
+
*
|
|
444
|
+
* ## Completion is the MARKER, written last
|
|
445
|
+
*
|
|
446
|
+
* The build runs AT the cache root, because that is the only directory its output works from — a
|
|
447
|
+
* finished build cannot be relocated, and the note at the top of this file is the measurement. So a
|
|
448
|
+
* cache root holding no marker is discarded whole before staging rather than built over, and the
|
|
449
|
+
* marker is written after `eve build` exits 0 and its {@link BUILT_SERVER_ENTRY} is on disk: the file
|
|
450
|
+
* a boot needs, rather than the directory it sits in. Since {@link cacheBuildComplete} consults the
|
|
451
|
+
* marker and nothing else, a process killed anywhere in the middle leaves a markerless root that the
|
|
452
|
+
* next run removes and redoes — which is the property a publishing `rename` would have bought, at a
|
|
453
|
+
* price the artifact cannot pay.
|
|
454
|
+
*
|
|
455
|
+
* A caller might still reach for a temp directory to get atomicity, and `eve build` already provides
|
|
456
|
+
* it where it counts: it compiles in an invocation-owned directory under `.eve/builds/`, publishes the
|
|
457
|
+
* completed output from there, and leaves the last successful `.output/` untouched when it fails (eve
|
|
458
|
+
* 0.38.3, `docs/reference/cli.md`). What eve cannot cover is THIS module's staging copy, which happens
|
|
459
|
+
* before eve is spawned — and that is what the lock and the marker are for.
|
|
460
|
+
*
|
|
461
|
+
* The build runs under a `mkdir`-based lock with stale-age takeover ({@link acquireBuildLock}),
|
|
462
|
+
* because two processes staging into the same version's cache concurrently would interleave their
|
|
463
|
+
* trees; eve's own `.eve/locks` starts too late to cover that copy.
|
|
239
464
|
*/
|
|
240
465
|
export const resolveAgentAppRoot = (input: {
|
|
241
466
|
readonly packageRoot: string
|
|
@@ -255,25 +480,72 @@ export const resolveAgentAppRoot = (input: {
|
|
|
255
480
|
})
|
|
256
481
|
})
|
|
257
482
|
const cacheRoot = cacheRootFor(version)
|
|
258
|
-
if (
|
|
483
|
+
if (cacheBuildComplete(cacheRoot)) return cacheRoot
|
|
259
484
|
|
|
260
|
-
yield* Effect.
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
485
|
+
return yield* Effect.acquireUseRelease(
|
|
486
|
+
Effect.tryPromise({
|
|
487
|
+
try: () => acquireBuildLock(cacheRoot),
|
|
488
|
+
catch: (cause) =>
|
|
489
|
+
ConsolidatorUnavailable.make({
|
|
490
|
+
reason: `could not lock the consolidator agent build: ${String(cause)}`
|
|
491
|
+
})
|
|
492
|
+
}),
|
|
493
|
+
() =>
|
|
494
|
+
Effect.gen(function* () {
|
|
495
|
+
// Another process may have completed the build while this one waited on the lock.
|
|
496
|
+
if (cacheBuildComplete(cacheRoot)) return cacheRoot
|
|
497
|
+
|
|
498
|
+
yield* Effect.logInfo(
|
|
499
|
+
`building the consolidator agent into ${cacheRoot} (once per version)`
|
|
500
|
+
)
|
|
501
|
+
yield* Effect.tryPromise({
|
|
502
|
+
try: async () => {
|
|
503
|
+
// Reaching here means the root carries no marker, so whatever it holds is an
|
|
504
|
+
// unfinished build. Discarded whole rather than staged over: a half-copied tree plus a
|
|
505
|
+
// fresh copy is a tree with no single version's shape.
|
|
506
|
+
await rm(cacheRoot, { recursive: true, force: true })
|
|
507
|
+
await stageAgentTree({ packageRoot, cacheRoot, version })
|
|
508
|
+
},
|
|
509
|
+
catch: (cause) =>
|
|
510
|
+
ConsolidatorUnavailable.make({
|
|
511
|
+
reason: `could not stage the consolidator agent in ${cacheRoot}: ${String(cause)}`
|
|
512
|
+
})
|
|
513
|
+
})
|
|
268
514
|
|
|
269
|
-
|
|
515
|
+
yield* runEveBuild({ eveBin, cwd: cacheRoot })
|
|
270
516
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
517
|
+
if (!existsSync(join(cacheRoot, BUILT_SERVER_ENTRY))) {
|
|
518
|
+
return yield* Effect.fail(
|
|
519
|
+
ConsolidatorUnavailable.make({
|
|
520
|
+
reason: `eve build wrote no ${BUILT_SERVER_ENTRY} in ${cacheRoot}`
|
|
521
|
+
})
|
|
522
|
+
)
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
yield* Effect.tryPromise({
|
|
526
|
+
try: () =>
|
|
527
|
+
writeFile(buildMarkerPath(cacheRoot), `${new Date().toISOString()}\n`, "utf8"),
|
|
528
|
+
catch: (cause) =>
|
|
529
|
+
ConsolidatorUnavailable.make({
|
|
530
|
+
reason: `could not mark the built agent complete in ${cacheRoot}: ${String(cause)}`
|
|
531
|
+
})
|
|
532
|
+
})
|
|
533
|
+
return cacheRoot
|
|
534
|
+
}).pipe(
|
|
535
|
+
// The build's own wreckage, reclaimed while the lock still excludes a concurrent stager: an
|
|
536
|
+
// unfinished build is ~17 MB nothing will ever consult, and the next run would discard it
|
|
537
|
+
// anyway. The MARKER is what makes this safe to run on every exit path, success included —
|
|
538
|
+
// it is written only beside a verified build, so a marked root is a finished one and is
|
|
539
|
+
// never a candidate, while every path that ends without it left a partial.
|
|
540
|
+
Effect.ensuring(
|
|
541
|
+
Effect.promise(async () => {
|
|
542
|
+
if (cacheBuildComplete(cacheRoot)) return
|
|
543
|
+
await rm(cacheRoot, { recursive: true, force: true }).catch(() => {})
|
|
544
|
+
})
|
|
545
|
+
)
|
|
546
|
+
),
|
|
547
|
+
(lock) => Effect.promise(lock.release)
|
|
548
|
+
)
|
|
277
549
|
})
|
|
278
550
|
|
|
279
551
|
/** Exported for the tests that assert the location, which is the part a reader can get wrong. */
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bounded stderr every child spawned from this package keeps, and the slice a failure message
|
|
3
|
+
* renders from it.
|
|
4
|
+
*
|
|
5
|
+
* Two children are spawned here — `eve build` (`agent-build.ts`) and `eve start` (`client.ts`) — and
|
|
6
|
+
* each reads its child's stderr for exactly one purpose: to carry the last thing the child said into a
|
|
7
|
+
* typed failure. Both halves of that are load-bearing, and they are only correct together, which is why
|
|
8
|
+
* they live in one module rather than as a constant per call site:
|
|
9
|
+
*
|
|
10
|
+
* - **Retention is a TAIL.** An unbounded accumulator grows for the child's whole life, and the start
|
|
11
|
+
* child's handle lives for a full turn — ten minutes — so a chatty server would hold every byte it
|
|
12
|
+
* ever logged in this process's heap.
|
|
13
|
+
* - **The message renders that same TAIL.** A message sliced from the HEAD of a capped buffer shows
|
|
14
|
+
* the bytes from just before the cap first bit, which for any child that wrote past the cap is a
|
|
15
|
+
* window ending {@link STDERR_TAIL_CHARS} before the fatal line: a cap that works and a diagnostic
|
|
16
|
+
* that defeats it. What a dying child wrote last is at the END.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* How much of a child's stderr is retained.
|
|
21
|
+
*
|
|
22
|
+
* The stream is read only so a failure can carry the child's last words, and
|
|
23
|
+
* {@link stderrMessageTail} takes 400 characters off it — so retention past that is context, not data.
|
|
24
|
+
* 64 KiB keeps the recent context and bounds the hold regardless of how long the child runs.
|
|
25
|
+
*/
|
|
26
|
+
export const STDERR_TAIL_CHARS = 64 * 1024
|
|
27
|
+
|
|
28
|
+
/** How much of the retained tail rides into a failure message: enough for a stack, not for a log. */
|
|
29
|
+
export const STDERR_MESSAGE_CHARS = 400
|
|
30
|
+
|
|
31
|
+
/** Append a chunk to a retained tail, keeping the LAST {@link STDERR_TAIL_CHARS} characters. */
|
|
32
|
+
export const appendStderrTail = (retained: string, chunk: string): string =>
|
|
33
|
+
(retained + chunk).slice(-STDERR_TAIL_CHARS)
|
|
34
|
+
|
|
35
|
+
/** The END of a retained tail, which is where a dying child's fatal line is. */
|
|
36
|
+
export const stderrMessageTail = (retained: string): string => retained.slice(-STDERR_MESSAGE_CHARS)
|