@dimina-kit/fs-core 0.2.0-dev.20260710085051
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +45 -0
- package/dist/agent-tools.js +52 -0
- package/dist/client.js +234 -0
- package/dist/disk-mirror.js +97 -0
- package/dist/fs-core.worker.js +1001 -0
- package/dist/fs-query.worker.js +115 -0
- package/dist/sync/sync-engine.js +381 -0
- package/dist/sync/truth-port.js +1 -0
- package/dist/worker-lib/engine-shared.js +25 -0
- package/dist/worker-lib/paths.js +17 -0
- package/dist/worker-lib/rpc-types.js +1 -0
- package/dist/worker-lib/wal-codec.js +111 -0
- package/dist/zip.js +60 -0
- package/package.json +75 -0
- package/src/__checks__/types-smoke.ts +23 -0
- package/src/agent-tools.test.ts +151 -0
- package/src/agent-tools.ts +117 -0
- package/src/client.test.ts +110 -0
- package/src/client.ts +284 -0
- package/src/disk-mirror.test.ts +190 -0
- package/src/disk-mirror.ts +119 -0
- package/src/fs-core-recovery.ts +280 -0
- package/src/fs-core-write-ops.ts +402 -0
- package/src/fs-core.worker.ts +182 -0
- package/src/fs-query.worker.ts +152 -0
- package/src/import-smoke.test.ts +40 -0
- package/src/worker-lib/engine-shared.test.ts +30 -0
- package/src/worker-lib/engine-shared.ts +74 -0
- package/src/worker-lib/paths.test.ts +39 -0
- package/src/worker-lib/paths.ts +14 -0
- package/src/worker-lib/rpc-types.ts +67 -0
- package/src/worker-lib/wal-codec.test.ts +83 -0
- package/src/worker-lib/wal-codec.ts +130 -0
- package/src/zip.test.ts +124 -0
- package/src/zip.ts +60 -0
- package/sync/sync-engine.test.ts +695 -0
- package/sync/sync-engine.ts +459 -0
- package/sync/truth-port.ts +72 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// src/fs-query.worker.ts
|
|
2
|
+
var state = {
|
|
3
|
+
mirror: /* @__PURE__ */ new Map(),
|
|
4
|
+
// path -> {content, rev}
|
|
5
|
+
gen: -1,
|
|
6
|
+
// -1 = 尚未收到 core 的首次同步
|
|
7
|
+
waiters: []
|
|
8
|
+
// [{gen, resolve, timer}]
|
|
9
|
+
};
|
|
10
|
+
function applySync(msg) {
|
|
11
|
+
if (msg.full) {
|
|
12
|
+
state.mirror.clear();
|
|
13
|
+
for (const [p, ent] of Object.entries(msg.files || {})) state.mirror.set(p, ent);
|
|
14
|
+
} else {
|
|
15
|
+
for (const [p, ent] of Object.entries(msg.diff || {})) {
|
|
16
|
+
if (ent === null) state.mirror.delete(p);
|
|
17
|
+
else state.mirror.set(p, ent);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
state.gen = msg.gen;
|
|
21
|
+
state.waiters = state.waiters.filter((w) => {
|
|
22
|
+
if (state.gen >= w.gen) {
|
|
23
|
+
clearTimeout(w.timer);
|
|
24
|
+
w.resolve();
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
return true;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
function whenGen(gen) {
|
|
31
|
+
if (gen === void 0 || state.gen >= gen) return Promise.resolve();
|
|
32
|
+
return new Promise((resolve) => {
|
|
33
|
+
const w = { gen, resolve, timer: setTimeout(() => {
|
|
34
|
+
state.waiters = state.waiters.filter((x) => x !== w);
|
|
35
|
+
resolve();
|
|
36
|
+
}, 5e3) };
|
|
37
|
+
state.waiters.push(w);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
var GLOB_SPECIAL_RE = /[.+^${}()|[\]\\]/;
|
|
41
|
+
function translateGlobToken(pattern, i) {
|
|
42
|
+
const c = pattern[i];
|
|
43
|
+
if (c === "*") {
|
|
44
|
+
if (pattern[i + 1] === "*") {
|
|
45
|
+
const slash = pattern[i + 2] === "/";
|
|
46
|
+
return { token: slash ? "(?:.*/)?" : ".*", skip: slash ? 2 : 1 };
|
|
47
|
+
}
|
|
48
|
+
return { token: "[^/]*", skip: 0 };
|
|
49
|
+
}
|
|
50
|
+
if (c === "?") return { token: "[^/]", skip: 0 };
|
|
51
|
+
const escaped = GLOB_SPECIAL_RE.test(c) ? "\\" + c : c;
|
|
52
|
+
return { token: escaped, skip: 0 };
|
|
53
|
+
}
|
|
54
|
+
function globToRegExp(pattern) {
|
|
55
|
+
let re = "";
|
|
56
|
+
for (let i = 0; i < pattern.length; i++) {
|
|
57
|
+
const { token, skip } = translateGlobToken(pattern, i);
|
|
58
|
+
re += token;
|
|
59
|
+
i += skip;
|
|
60
|
+
}
|
|
61
|
+
return new RegExp("^" + re + "$");
|
|
62
|
+
}
|
|
63
|
+
var ops = {
|
|
64
|
+
async snapshot({ gen }) {
|
|
65
|
+
await whenGen(gen);
|
|
66
|
+
const files = {};
|
|
67
|
+
for (const [p, ent] of state.mirror) files[p] = ent.content;
|
|
68
|
+
return { files, gen: state.gen, stale: gen !== void 0 && state.gen < gen };
|
|
69
|
+
},
|
|
70
|
+
async read({ path, gen }) {
|
|
71
|
+
await whenGen(gen);
|
|
72
|
+
const e = state.mirror.get(path);
|
|
73
|
+
if (!e) throw Object.assign(new Error(path), { code: "not-found" });
|
|
74
|
+
return { content: e.content, rev: e.rev, gen: state.gen };
|
|
75
|
+
},
|
|
76
|
+
async glob({ pattern, gen }) {
|
|
77
|
+
await whenGen(gen);
|
|
78
|
+
const re = globToRegExp(pattern);
|
|
79
|
+
return { paths: [...state.mirror.keys()].filter((p) => re.test(p)).sort(), gen: state.gen };
|
|
80
|
+
},
|
|
81
|
+
async grep({ pattern, flags = "", glob, gen, limit = 200 }) {
|
|
82
|
+
await whenGen(gen);
|
|
83
|
+
const re = new RegExp(pattern, flags.replace("g", ""));
|
|
84
|
+
const scope = glob ? globToRegExp(glob) : null;
|
|
85
|
+
const hits = [];
|
|
86
|
+
for (const [path, ent] of state.mirror) {
|
|
87
|
+
if (scope && !scope.test(path)) continue;
|
|
88
|
+
const lines = ent.content.split("\n");
|
|
89
|
+
for (let i = 0; i < lines.length; i++) {
|
|
90
|
+
if (re.test(lines[i])) {
|
|
91
|
+
hits.push({ path, lineNo: i + 1, line: lines[i].slice(0, 500) });
|
|
92
|
+
if (hits.length >= limit) return { hits, gen: state.gen, truncated: true };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return { hits, gen: state.gen, truncated: false };
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
self.onmessage = async (e) => {
|
|
100
|
+
const msg = e.data;
|
|
101
|
+
if (msg.type === "init") {
|
|
102
|
+
msg.corePort.onmessage = (ev) => applySync(ev.data);
|
|
103
|
+
self.postMessage({ type: "ready" });
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (msg.id === void 0) return;
|
|
107
|
+
try {
|
|
108
|
+
const dispatch = ops;
|
|
109
|
+
const result = await dispatch[msg.op](msg.args || {});
|
|
110
|
+
self.postMessage({ id: msg.id, ok: true, result });
|
|
111
|
+
} catch (err) {
|
|
112
|
+
const e2 = err;
|
|
113
|
+
self.postMessage({ id: msg.id, ok: false, code: e2.code || "internal", error: e2.message || String(err) });
|
|
114
|
+
}
|
|
115
|
+
};
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync arbitration engine — the memfs<->disk synchronization core factored
|
|
3
|
+
* out of dimina-kit's workbench `wal-audit.ts` (see
|
|
4
|
+
* devtools-fs-core-feasibility.md §7+§8). A host wires a `client` (the
|
|
5
|
+
* fs-core ledger: write/rm/read/ls) and a `TruthPort` (its external truth
|
|
6
|
+
* source — devtools' `/__fs` bridge + SSE watch, or a future FSA
|
|
7
|
+
* local-directory adapter) and gets back an engine that keeps the ledger,
|
|
8
|
+
* the external truth, and — via `applyToEditor` — the live editor buffer
|
|
9
|
+
* reconciled. `kernel` and turn enforcement never enter this module: a
|
|
10
|
+
* host's own audit-turn surface talks to `client` directly for that (see
|
|
11
|
+
* wal-audit.ts).
|
|
12
|
+
*
|
|
13
|
+
* Echo judgement (both directions):
|
|
14
|
+
* - Outbound (`onHumanSave`): before recording, compare the saved text
|
|
15
|
+
* against the ledger's current record for that path — identical content
|
|
16
|
+
* is a no-op (the ledger already reflects it), skipping a redundant
|
|
17
|
+
* write.
|
|
18
|
+
* - Inbound (`changes` batches, via `handleInboundPath`): first check
|
|
19
|
+
* `pendingWrite` — an entry there was registered by an `onHumanSave` still
|
|
20
|
+
* in flight for the same path, and its presence means this inbound
|
|
21
|
+
* notification is that write's own echo, absorbed with no ledger write
|
|
22
|
+
* and no editor refresh. This check is a structural no-op for a 'push'
|
|
23
|
+
* port (devtools' SSE): registration and clearing both happen inside the
|
|
24
|
+
* SAME `ledgerTurn` FIFO slot as the write they guard (see
|
|
25
|
+
* `onHumanSave`), so by the time any later-queued inbound turn for that
|
|
26
|
+
* path runs, the entry is already gone — the branch exists for a future
|
|
27
|
+
* 'poll' host whose change detection runs OUTSIDE this FIFO (e.g. an
|
|
28
|
+
* mtime/size sweep) and can therefore observe the entry while the write
|
|
29
|
+
* is still in flight. When `pendingWrite` misses, the engine falls back
|
|
30
|
+
* to today's content comparison (truth-source bytes vs. the ledger's own
|
|
31
|
+
* record) — the only judgement a push host ever actually exercises.
|
|
32
|
+
*
|
|
33
|
+
* Inbound-echo consumption (`consumeInboundEcho`, `inboundApplied`): a 'poll'
|
|
34
|
+
* host (e.g. the web local-directory adapter) drives its OWN outbound path
|
|
35
|
+
* OUTSIDE this module — it reads the ledger and writes the truth source
|
|
36
|
+
* directly, not through `onHumanSave` — so a change this engine just applied
|
|
37
|
+
* INBOUND (disk -> ledger, via `handleInboundPath`) can race that host's
|
|
38
|
+
* outbound scan and get echoed straight back to disk before the poll
|
|
39
|
+
* baseline ever settles. `inboundApplied` (`rel -> {kind:'text'|'binary'|
|
|
40
|
+
* 'delete', ...}`) records exactly what `handleInboundPath` just applied,
|
|
41
|
+
* overwriting any prior entry for the same path; `consumeInboundEcho(rel,
|
|
42
|
+
* content)` lets such a host check "is this outbound write about to re-emit
|
|
43
|
+
* an inbound change I haven't published yet?" immediately BEFORE writing to
|
|
44
|
+
* the truth source, consuming (clearing) the record on a match so it is a
|
|
45
|
+
* one-shot suppression, not a standing content cache. A miss (different
|
|
46
|
+
* content, or no record at all) returns false and leaves any existing record
|
|
47
|
+
* untouched — it is not this call's echo to consume.
|
|
48
|
+
*
|
|
49
|
+
* Binary layering (v1): the first 8192 bytes of a file are sniffed for a NUL
|
|
50
|
+
* byte to classify it binary. A binary file never reaches the fs-core
|
|
51
|
+
* ledger (`client.write`/`read`) — it is tracked only in the in-memory
|
|
52
|
+
* `binaryIndex` (`rel -> { size, sha256 }`), and echo judgement for it is
|
|
53
|
+
* size+hash equality instead of a ledger content compare. What/why this is
|
|
54
|
+
* narrower than the text path: binary changes get NO WAL audit and NO
|
|
55
|
+
* rollback (the audit turn surface — `diff`/`restore` — is a string-content
|
|
56
|
+
* contract; v1 does not support an agent writing binary inside a turn), and
|
|
57
|
+
* `binaryIndex` is session-scoped — it is cleared and rebuilt from scratch on
|
|
58
|
+
* every `populateLedger()`, exactly like the ledger's own text reconciliation.
|
|
59
|
+
*/
|
|
60
|
+
const BINARY_SNIFF_BYTES = 8192;
|
|
61
|
+
/** True when the first `BINARY_SNIFF_BYTES` of `bytes` contain a NUL byte. */
|
|
62
|
+
function looksBinary(bytes) {
|
|
63
|
+
const len = Math.min(bytes.length, BINARY_SNIFF_BYTES);
|
|
64
|
+
for (let i = 0; i < len; i++) {
|
|
65
|
+
if (bytes[i] === 0)
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
async function sha256hex(bytes) {
|
|
71
|
+
const digest = await crypto.subtle.digest('SHA-256', bytes);
|
|
72
|
+
return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join('');
|
|
73
|
+
}
|
|
74
|
+
/** Byte-for-byte equality — used by `consumeInboundEcho`'s binary-content match. */
|
|
75
|
+
function bytesEqual(a, b) {
|
|
76
|
+
if (a.length !== b.length)
|
|
77
|
+
return false;
|
|
78
|
+
for (let i = 0; i < a.length; i++)
|
|
79
|
+
if (a[i] !== b[i])
|
|
80
|
+
return false;
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* True when `error` denotes "path does not exist" per the TruthPort error
|
|
85
|
+
* contract (see truth-port.ts): `error.code === 'not-found'` or
|
|
86
|
+
* `error.status === 404`. Every other rejection is "unavailable" (transient
|
|
87
|
+
* I/O failure, permission loss, dead connection, ...).
|
|
88
|
+
*/
|
|
89
|
+
function isNotFoundError(error) {
|
|
90
|
+
const e = error;
|
|
91
|
+
return Boolean(e) && (e.code === 'not-found' || e.status === 404);
|
|
92
|
+
}
|
|
93
|
+
export function createSyncEngine(client, port, opts = {}) {
|
|
94
|
+
const decoder = new TextDecoder();
|
|
95
|
+
const applyToEditor = opts.applyToEditor;
|
|
96
|
+
/**
|
|
97
|
+
* Paths with an `onHumanSave` write in flight — see the module doc's
|
|
98
|
+
* "Echo judgement" section. Registered synchronously when `onHumanSave` is
|
|
99
|
+
* called and cleared unconditionally (`finally`) once that write's own
|
|
100
|
+
* ledgerTurn slot finishes, success or failure.
|
|
101
|
+
*/
|
|
102
|
+
const pendingWrite = new Set();
|
|
103
|
+
/**
|
|
104
|
+
* Binary files never enter the fs-core ledger — see the module doc's
|
|
105
|
+
* "Binary layering" section. Session-scoped: rebuilt from scratch on every
|
|
106
|
+
* `populateLedger()`.
|
|
107
|
+
*/
|
|
108
|
+
const binaryIndex = new Map();
|
|
109
|
+
/**
|
|
110
|
+
* `rel -> { kind: 'text', text } | { kind: 'binary', bytes } | { kind: 'delete' }`
|
|
111
|
+
* — see the module doc's "Inbound-echo consumption" section. Written by
|
|
112
|
+
* `handleInboundPath` immediately after it actually applies an inbound
|
|
113
|
+
* change (ledger write/rm, or a binary/delete index update); consumed
|
|
114
|
+
* (checked + cleared on a match) by `consumeInboundEcho`. Session-scoped,
|
|
115
|
+
* same as `binaryIndex` — cleared on every `populateLedger()`.
|
|
116
|
+
*/
|
|
117
|
+
const inboundApplied = new Map();
|
|
118
|
+
/**
|
|
119
|
+
* FIFO queue serializing every compare-then-record against the ledger
|
|
120
|
+
* (inbound change batches AND the onHumanSave accounting step). Without it
|
|
121
|
+
* the two race: an inbound echo's compare-read can run while an
|
|
122
|
+
* onHumanSave ledger write is still in flight, see stale content, and
|
|
123
|
+
* re-record the identical bytes. Serialized, the loser of the race sees
|
|
124
|
+
* the winner's write and absorbs it as an echo. The chain swallows step
|
|
125
|
+
* rejections (each caller handles its own errors), so one failed step can
|
|
126
|
+
* never wedge the queue.
|
|
127
|
+
*/
|
|
128
|
+
let ledgerTurn = Promise.resolve();
|
|
129
|
+
function enqueueLedgerTurn(fn) {
|
|
130
|
+
const next = ledgerTurn.then(fn, fn);
|
|
131
|
+
ledgerTurn = next.catch(() => { });
|
|
132
|
+
return next;
|
|
133
|
+
}
|
|
134
|
+
/** Walk the port's tree into the ledger, then reconcile: ledger paths the
|
|
135
|
+
* walk did not produce are residue (e.g. from a previous session under the
|
|
136
|
+
* same persisted ledger identity) and are removed so the ledger ends up
|
|
137
|
+
* exactly matching the walked tree. */
|
|
138
|
+
async function seedFromDisk() {
|
|
139
|
+
binaryIndex.clear();
|
|
140
|
+
inboundApplied.clear();
|
|
141
|
+
const seen = new Set();
|
|
142
|
+
await port.walk(async (rel, bytes) => {
|
|
143
|
+
seen.add(rel);
|
|
144
|
+
if (looksBinary(bytes)) {
|
|
145
|
+
binaryIndex.set(rel, { size: bytes.length, sha256: await sha256hex(bytes) });
|
|
146
|
+
return; // binary never enters the ledger
|
|
147
|
+
}
|
|
148
|
+
await client.write(rel, decoder.decode(bytes), { actor: 'human' });
|
|
149
|
+
});
|
|
150
|
+
const { paths } = await client.ls();
|
|
151
|
+
for (const p of paths) {
|
|
152
|
+
// Residue from a previous session, OR a path that used to be
|
|
153
|
+
// ledgered as text but is now classified binary (migration cleanup —
|
|
154
|
+
// binaryIndex.has(p) is authoritative once the walk above ran).
|
|
155
|
+
if (!seen.has(p) || binaryIndex.has(p))
|
|
156
|
+
await client.rm(p, { actor: 'human' });
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Process one inbound path. `pendingWrite` is checked first (see module
|
|
161
|
+
* doc); a miss falls back to content comparison: identical content is the
|
|
162
|
+
* echo of our own last write and is dropped with no ledger write and no
|
|
163
|
+
* `applyToEditor` call. A `port.read` rejection classified as not-found
|
|
164
|
+
* (see truth-port.ts) is a deletion; any other rejection
|
|
165
|
+
* ("unavailable") is transient and skips the path entirely — inferring a
|
|
166
|
+
* deletion from it would rm the ledger record and close the file in the
|
|
167
|
+
* editor while it still exists at the truth source.
|
|
168
|
+
*/
|
|
169
|
+
async function handleInboundPath(rel) {
|
|
170
|
+
if (pendingWrite.has(rel)) {
|
|
171
|
+
pendingWrite.delete(rel);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
let bytes;
|
|
175
|
+
try {
|
|
176
|
+
bytes = await port.read(rel);
|
|
177
|
+
}
|
|
178
|
+
catch (e) {
|
|
179
|
+
if (!isNotFoundError(e)) {
|
|
180
|
+
console.warn('[fs-core/sync] transient port read failure, skipping', rel, e);
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
bytes = null;
|
|
184
|
+
}
|
|
185
|
+
if (bytes === null) {
|
|
186
|
+
// Deletion. A path the binaryIndex knows about was never in the
|
|
187
|
+
// ledger (it never took the client.write path), so removal here is
|
|
188
|
+
// index-only — no client.rm.
|
|
189
|
+
if (binaryIndex.has(rel)) {
|
|
190
|
+
binaryIndex.delete(rel);
|
|
191
|
+
inboundApplied.set(rel, { kind: 'delete' });
|
|
192
|
+
await applyToEditor?.(rel, null);
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
let ledgerContent;
|
|
196
|
+
try {
|
|
197
|
+
ledgerContent = (await client.read(rel)).content;
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
ledgerContent = undefined;
|
|
201
|
+
}
|
|
202
|
+
if (ledgerContent === undefined)
|
|
203
|
+
return; // already absent from both sides
|
|
204
|
+
await client.rm(rel, { actor: 'human' });
|
|
205
|
+
inboundApplied.set(rel, { kind: 'delete' });
|
|
206
|
+
await applyToEditor?.(rel, null);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
if (looksBinary(bytes)) {
|
|
210
|
+
const prior = binaryIndex.get(rel);
|
|
211
|
+
const sha256 = await sha256hex(bytes);
|
|
212
|
+
if (prior && prior.size === bytes.length && prior.sha256 === sha256)
|
|
213
|
+
return; // echo: same bytes already indexed
|
|
214
|
+
binaryIndex.set(rel, { size: bytes.length, sha256 });
|
|
215
|
+
inboundApplied.set(rel, { kind: 'binary', bytes });
|
|
216
|
+
await applyToEditor?.(rel, bytes);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
let ledgerContent;
|
|
220
|
+
try {
|
|
221
|
+
ledgerContent = (await client.read(rel)).content;
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
// Not in the ledger yet (or a transient read failure) — treated the
|
|
225
|
+
// same as "no prior recorded content", so a real file at the truth
|
|
226
|
+
// source always gets recorded/applied below rather than silently
|
|
227
|
+
// dropped.
|
|
228
|
+
ledgerContent = undefined;
|
|
229
|
+
}
|
|
230
|
+
const text = decoder.decode(bytes);
|
|
231
|
+
if (ledgerContent !== undefined && text === ledgerContent)
|
|
232
|
+
return; // echo of our own write
|
|
233
|
+
await client.write(rel, text, { actor: 'human' });
|
|
234
|
+
inboundApplied.set(rel, { kind: 'text', text });
|
|
235
|
+
await applyToEditor?.(rel, bytes);
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* One-shot check for a 'poll' host's own outbound path: "is `content` (the
|
|
239
|
+
* bytes/text about to be written to the truth source for `rel`, or `null`
|
|
240
|
+
* for a delete) exactly what `handleInboundPath` just applied FROM that
|
|
241
|
+
* same truth source?" A match means writing it back out would be a pure
|
|
242
|
+
* echo — the host should skip the write entirely — and the record is
|
|
243
|
+
* cleared (consumed) so it cannot match again. See the module doc's
|
|
244
|
+
* "Inbound-echo consumption" section for why a push host (devtools) never
|
|
245
|
+
* needs this: its outbound path goes through `onHumanSave`/`pendingWrite`
|
|
246
|
+
* instead.
|
|
247
|
+
*/
|
|
248
|
+
function consumeInboundEcho(rel, content) {
|
|
249
|
+
const entry = inboundApplied.get(rel);
|
|
250
|
+
if (!entry)
|
|
251
|
+
return false;
|
|
252
|
+
let matches;
|
|
253
|
+
if (content === null)
|
|
254
|
+
matches = entry.kind === 'delete';
|
|
255
|
+
else if (typeof content === 'string')
|
|
256
|
+
matches = entry.kind === 'text' && entry.text === content;
|
|
257
|
+
else
|
|
258
|
+
matches = entry.kind === 'binary' && bytesEqual(entry.bytes, content);
|
|
259
|
+
if (matches)
|
|
260
|
+
inboundApplied.delete(rel);
|
|
261
|
+
return matches;
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* A batch is trusted as-is: the port adapter (see truth-port.ts's
|
|
265
|
+
* `changes` doc) is responsible for turning a watcher's coalesced/lossy
|
|
266
|
+
* events into the actual set of paths worth re-examining BEFORE calling
|
|
267
|
+
* this engine — devtools' adapter (wal-audit.ts +
|
|
268
|
+
* wal-audit-watch-expand.ts) does that via a stat-level disk compare
|
|
269
|
+
* against a session index, so paths arriving here have already earned
|
|
270
|
+
* their re-examination and no further expansion happens at this layer.
|
|
271
|
+
*/
|
|
272
|
+
async function handleBatch(paths) {
|
|
273
|
+
for (const rel of paths) {
|
|
274
|
+
try {
|
|
275
|
+
// Per-path (not per-batch) queue turns, so a long batch cannot
|
|
276
|
+
// starve an interleaved onHumanSave accounting step of its FIFO
|
|
277
|
+
// position.
|
|
278
|
+
await enqueueLedgerTurn(() => handleInboundPath(rel));
|
|
279
|
+
}
|
|
280
|
+
catch (e) {
|
|
281
|
+
console.warn('[fs-core/sync] disk sync failed for', rel, e);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
let pendingInboundPaths = new Set();
|
|
286
|
+
let inboundBatchRunning = false;
|
|
287
|
+
function enqueueInboundBatch(paths) {
|
|
288
|
+
for (const rel of paths)
|
|
289
|
+
pendingInboundPaths.add(rel);
|
|
290
|
+
if (inboundBatchRunning)
|
|
291
|
+
return;
|
|
292
|
+
inboundBatchRunning = true;
|
|
293
|
+
void (async () => {
|
|
294
|
+
try {
|
|
295
|
+
while (pendingInboundPaths.size) {
|
|
296
|
+
const raw = [...pendingInboundPaths];
|
|
297
|
+
pendingInboundPaths = new Set();
|
|
298
|
+
await handleBatch(raw);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
finally {
|
|
302
|
+
inboundBatchRunning = false;
|
|
303
|
+
if (pendingInboundPaths.size)
|
|
304
|
+
enqueueInboundBatch([]);
|
|
305
|
+
}
|
|
306
|
+
})();
|
|
307
|
+
}
|
|
308
|
+
let stopWatching = () => { };
|
|
309
|
+
/** Subscribe to `port.changes`. `active` gates BOTH callbacks so a
|
|
310
|
+
* late/duplicate event delivered after `onDead` (or after a fresh
|
|
311
|
+
* `start()` superseded this subscription) is a guaranteed no-op, not just
|
|
312
|
+
* best-effort. */
|
|
313
|
+
function start() {
|
|
314
|
+
let active = true;
|
|
315
|
+
const dispose = port.changes((paths) => {
|
|
316
|
+
if (!active)
|
|
317
|
+
return;
|
|
318
|
+
enqueueInboundBatch(paths);
|
|
319
|
+
}, () => {
|
|
320
|
+
if (!active)
|
|
321
|
+
return;
|
|
322
|
+
active = false;
|
|
323
|
+
console.warn('[fs-core/sync] watcher died — reverting to open-time mirror only');
|
|
324
|
+
dispose();
|
|
325
|
+
});
|
|
326
|
+
stopWatching = () => {
|
|
327
|
+
active = false;
|
|
328
|
+
dispose();
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
function stop() {
|
|
332
|
+
stopWatching();
|
|
333
|
+
stopWatching = () => { };
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Outbound accounting for a human save: registers `rel` in `pendingWrite`
|
|
337
|
+
* for the duration of the ledger compare-then-write (see module doc),
|
|
338
|
+
* skips the ledger write when the saved text already matches the ledger's
|
|
339
|
+
* record, and runs inside the same `ledgerTurn` FIFO as inbound batches so
|
|
340
|
+
* the two can never interleave. Errors propagate to the caller — a host's
|
|
341
|
+
* onSave wrapper decides whether a ledger-write failure should be
|
|
342
|
+
* swallowed (best-effort accounting must never unwind a save that already
|
|
343
|
+
* landed at the truth source).
|
|
344
|
+
*
|
|
345
|
+
* `content` may be the decoded text (string, unchanged legacy path) or the
|
|
346
|
+
* raw saved bytes (`Uint8Array`) — passing raw bytes lets this function
|
|
347
|
+
* sniff for binary (see module doc's "Binary layering" section) before
|
|
348
|
+
* deciding whether to decode. A binary save skips the ledger write
|
|
349
|
+
* entirely and only updates `binaryIndex`.
|
|
350
|
+
*/
|
|
351
|
+
async function onHumanSave(rel, content) {
|
|
352
|
+
pendingWrite.add(rel);
|
|
353
|
+
try {
|
|
354
|
+
if (content instanceof Uint8Array && looksBinary(content)) {
|
|
355
|
+
await enqueueLedgerTurn(async () => {
|
|
356
|
+
binaryIndex.set(rel, { size: content.length, sha256: await sha256hex(content) });
|
|
357
|
+
});
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
const text = typeof content === 'string' ? content : decoder.decode(content);
|
|
361
|
+
await enqueueLedgerTurn(async () => {
|
|
362
|
+
const unchanged = await client
|
|
363
|
+
.read(rel)
|
|
364
|
+
.then((r) => r.content === text)
|
|
365
|
+
.catch(() => false);
|
|
366
|
+
if (!unchanged)
|
|
367
|
+
await client.write(rel, text, { actor: 'human' });
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
finally {
|
|
371
|
+
pendingWrite.delete(rel);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
return {
|
|
375
|
+
populateLedger: seedFromDisk,
|
|
376
|
+
onHumanSave,
|
|
377
|
+
consumeInboundEcho,
|
|
378
|
+
start,
|
|
379
|
+
stop,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export const OP = { WRITE: 1, RM: 2, MV: 3, MKDIR: 4, CHECKPOINT: 5, RESTORE: 6 };
|
|
2
|
+
export const OP_NAME = { 1: 'write', 2: 'rm', 3: 'mv', 4: 'mkdir', 5: 'checkpoint', 6: 'restore' };
|
|
3
|
+
// §4.7 restore 冲突检查只关心"写类"操作(改变文件内容/存在性),checkpoint 本身不算
|
|
4
|
+
export const WRITE_OPCODES = new Set([OP.WRITE, OP.RM, OP.MV, OP.RESTORE]);
|
|
5
|
+
export const INLINE_MAX = 4096; // payload ≤4KB 内联进 WAL 记录
|
|
6
|
+
export const GROUP_WINDOW_MS = 50; // 人类写组提交窗口
|
|
7
|
+
export const SEGMENT_ROTATE_BYTES = 4 * 1024 * 1024;
|
|
8
|
+
export const OPID_WINDOW = 1024;
|
|
9
|
+
// P4 turn 能力:agent 写必须在有效 turn 内(fs-core 侧执法,不信任调用方透传)
|
|
10
|
+
export const TURN_DEFAULT_TTL_MS = 120000;
|
|
11
|
+
export const TURN_MAX_OPS = 1000; // per-turn 限额(跑飞的 agent 刹车)
|
|
12
|
+
export const AUDIT_CAP = 4096; // 内存审计环(fs_diff 的数据源;重启由 WAL 回放重建)
|
|
13
|
+
// P5 checkpoint LRU:保留最近 N 个;被淘汰者的 blob 在下次 compaction GC 回收
|
|
14
|
+
export const CHECKPOINT_KEEP = 20;
|
|
15
|
+
export function rpcErr(code, message, extra) {
|
|
16
|
+
const e = new Error(message);
|
|
17
|
+
e.code = code;
|
|
18
|
+
if (extra)
|
|
19
|
+
e.extra = extra;
|
|
20
|
+
return e;
|
|
21
|
+
}
|
|
22
|
+
/** 供 fs-core-recovery.ts 的回放循环使用;纯函数(无副作用)。 */
|
|
23
|
+
export function epochFloor(replayed) {
|
|
24
|
+
return replayed.length ? replayed[replayed.length - 1].epoch : 0; // epoch 单调不减
|
|
25
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** 路径监狱(纯函数,lib 中立)—— fs-core.worker.ts 的写路径统一走这里做归一化/校验。 */
|
|
2
|
+
export const DERIVED_PREFIXES = ['node_modules/', '.checkpoints/'];
|
|
3
|
+
export function normalizePath(p) {
|
|
4
|
+
if (typeof p !== 'string' || !p || p.includes('\0') || p.includes('\\'))
|
|
5
|
+
return null;
|
|
6
|
+
if (p.startsWith('/'))
|
|
7
|
+
return null;
|
|
8
|
+
const parts = [];
|
|
9
|
+
for (const seg of p.split('/')) {
|
|
10
|
+
if (seg === '' || seg === '.')
|
|
11
|
+
continue;
|
|
12
|
+
if (seg === '..')
|
|
13
|
+
return null;
|
|
14
|
+
parts.push(seg);
|
|
15
|
+
}
|
|
16
|
+
return parts.length ? parts.join('/') : null;
|
|
17
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WAL 编解码(纯函数,lib 中立)—— crc32、superblock 槽、WAL 记录成帧/解析。
|
|
3
|
+
* 从 fs-core.worker.ts 抽出:不含任何 OPFS/Worker 专属 API,可被主 tsconfig
|
|
4
|
+
* (DOM lib)与 tsconfig.worker.json(WebWorker lib)两个 program 同时编译,
|
|
5
|
+
* 因此 zip.ts(主 program 侧)与 fs-core.worker.ts(worker program 侧)
|
|
6
|
+
* 都能 import 同一份 crc32/CRC_TABLE 实现(消除重复代码)。
|
|
7
|
+
*
|
|
8
|
+
* WAL 记录成帧:
|
|
9
|
+
* [u32 len][u64 gen][u32 epoch][u8 opcode][u16 metaLen][meta JSON][u32 crc32][u8 0xC1]
|
|
10
|
+
* len = len 字段之后的字节数;crc 覆盖 gen..meta 末尾。
|
|
11
|
+
*/
|
|
12
|
+
export const enc = new TextEncoder();
|
|
13
|
+
export const dec = new TextDecoder();
|
|
14
|
+
// ───────────────────────── crc32(查表法) ─────────────────────────
|
|
15
|
+
export const CRC_TABLE = (() => {
|
|
16
|
+
const t = new Uint32Array(256);
|
|
17
|
+
for (let n = 0; n < 256; n++) {
|
|
18
|
+
let c = n;
|
|
19
|
+
for (let k = 0; k < 8; k++)
|
|
20
|
+
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
|
21
|
+
t[n] = c >>> 0;
|
|
22
|
+
}
|
|
23
|
+
return t;
|
|
24
|
+
})();
|
|
25
|
+
export function crc32(bytes, start = 0, end = bytes.length) {
|
|
26
|
+
let c = 0xffffffff;
|
|
27
|
+
for (let i = start; i < end; i++)
|
|
28
|
+
c = CRC_TABLE[(c ^ bytes[i]) & 0xff] ^ (c >>> 8);
|
|
29
|
+
return (c ^ 0xffffffff) >>> 0;
|
|
30
|
+
}
|
|
31
|
+
export async function sha256hex(bytes) {
|
|
32
|
+
const d = await crypto.subtle.digest('SHA-256', bytes);
|
|
33
|
+
return [...new Uint8Array(d)].map((b) => b.toString(16).padStart(2, '0')).join('');
|
|
34
|
+
}
|
|
35
|
+
const SB_MAGIC = 0x44574331; // 'DWC1'
|
|
36
|
+
export const SLOT_SIZE = 64;
|
|
37
|
+
// ───────────────────────── superblock 槽编解码 ─────────────────────────
|
|
38
|
+
// 布局:magic u32 | epoch u32 | compactGen f64 | walStartGen f64 | manifestCrc u32 | pad→60 | crc32 u32
|
|
39
|
+
export function encodeSlot(s) {
|
|
40
|
+
const buf = new ArrayBuffer(SLOT_SIZE);
|
|
41
|
+
const dv = new DataView(buf);
|
|
42
|
+
dv.setUint32(0, SB_MAGIC);
|
|
43
|
+
dv.setUint32(4, s.epoch);
|
|
44
|
+
dv.setFloat64(8, s.compactGen);
|
|
45
|
+
dv.setFloat64(16, s.walStartGen);
|
|
46
|
+
dv.setUint32(24, s.manifestCrc);
|
|
47
|
+
dv.setUint32(60, crc32(new Uint8Array(buf, 0, 60)));
|
|
48
|
+
return new Uint8Array(buf);
|
|
49
|
+
}
|
|
50
|
+
export function decodeSlot(bytes) {
|
|
51
|
+
if (bytes.length < SLOT_SIZE)
|
|
52
|
+
return null;
|
|
53
|
+
const dv = new DataView(bytes.buffer, bytes.byteOffset, SLOT_SIZE);
|
|
54
|
+
if (dv.getUint32(60) !== crc32(bytes, 0, 60))
|
|
55
|
+
return null;
|
|
56
|
+
if (dv.getUint32(0) !== SB_MAGIC)
|
|
57
|
+
return null;
|
|
58
|
+
return {
|
|
59
|
+
epoch: dv.getUint32(4),
|
|
60
|
+
compactGen: dv.getFloat64(8),
|
|
61
|
+
walStartGen: dv.getFloat64(16),
|
|
62
|
+
manifestCrc: dv.getUint32(24),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
// ───────────────────────── WAL 记录编解码 ─────────────────────────
|
|
66
|
+
export function frameRecord(gen, epoch, opcode, meta) {
|
|
67
|
+
const metaBytes = enc.encode(JSON.stringify(meta));
|
|
68
|
+
const len = 8 + 4 + 1 + 2 + metaBytes.length + 4 + 1;
|
|
69
|
+
const buf = new ArrayBuffer(4 + len);
|
|
70
|
+
const dv = new DataView(buf);
|
|
71
|
+
const u8 = new Uint8Array(buf);
|
|
72
|
+
dv.setUint32(0, len);
|
|
73
|
+
dv.setBigUint64(4, BigInt(gen));
|
|
74
|
+
dv.setUint32(12, epoch);
|
|
75
|
+
dv.setUint8(16, opcode);
|
|
76
|
+
dv.setUint16(17, metaBytes.length);
|
|
77
|
+
u8.set(metaBytes, 19);
|
|
78
|
+
const crcEnd = 19 + metaBytes.length;
|
|
79
|
+
dv.setUint32(crcEnd, crc32(u8, 4, crcEnd));
|
|
80
|
+
dv.setUint8(crcEnd + 4, 0xc1);
|
|
81
|
+
return u8;
|
|
82
|
+
}
|
|
83
|
+
/** 解析一条记录;返回 {rec, next} 或 null(framing/CRC/commit 任一失败)。 */
|
|
84
|
+
export function parseRecord(u8, off) {
|
|
85
|
+
if (off + 4 > u8.length)
|
|
86
|
+
return null;
|
|
87
|
+
const dv = new DataView(u8.buffer, u8.byteOffset);
|
|
88
|
+
const len = dv.getUint32(off);
|
|
89
|
+
if (len < 20 || off + 4 + len > u8.length)
|
|
90
|
+
return null;
|
|
91
|
+
const gen = Number(dv.getBigUint64(off + 4));
|
|
92
|
+
const epoch = dv.getUint32(off + 12);
|
|
93
|
+
const opcode = dv.getUint8(off + 16);
|
|
94
|
+
const metaLen = dv.getUint16(off + 17);
|
|
95
|
+
const metaStart = off + 19;
|
|
96
|
+
const crcAt = metaStart + metaLen;
|
|
97
|
+
if (crcAt + 5 !== off + 4 + len)
|
|
98
|
+
return null;
|
|
99
|
+
if (dv.getUint32(crcAt) !== crc32(u8, off + 4, crcAt))
|
|
100
|
+
return null;
|
|
101
|
+
if (dv.getUint8(crcAt + 4) !== 0xc1)
|
|
102
|
+
return null;
|
|
103
|
+
let meta;
|
|
104
|
+
try {
|
|
105
|
+
meta = JSON.parse(dec.decode(u8.subarray(metaStart, crcAt)));
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
return { rec: { gen, epoch, opcode, meta }, next: off + 4 + len };
|
|
111
|
+
}
|