@frockbot/plugin-fly-sprite 0.3.20 → 0.3.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/computer.ts +9 -1
- package/src/doctor.test.ts +14 -2
- package/src/host-double.ts +15 -6
- package/src/provider.ts +3 -1
- package/src/sync.test.ts +148 -8
- package/src/sync.ts +223 -14
- package/src/workspace.ts +5 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frockbot/plugin-fly-sprite",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.22",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -25,16 +25,16 @@
|
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@cordisjs/plugin-webui": "0.8.2",
|
|
28
|
-
"@frockbot/computer-core": "0.3.
|
|
29
|
-
"@frockbot/computer-host-protocol": "0.3.
|
|
30
|
-
"@frockbot/computer-host-runtime": "0.3.
|
|
31
|
-
"@frockbot/kernel-contracts": "0.3.
|
|
32
|
-
"@frockbot/plugin-computer": "0.3.
|
|
28
|
+
"@frockbot/computer-core": "0.3.22",
|
|
29
|
+
"@frockbot/computer-host-protocol": "0.3.22",
|
|
30
|
+
"@frockbot/computer-host-runtime": "0.3.22",
|
|
31
|
+
"@frockbot/kernel-contracts": "0.3.22",
|
|
32
|
+
"@frockbot/plugin-computer": "0.3.22",
|
|
33
33
|
"cordis": "4.0.0-rc.8"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
|
-
"@frockbot/plugin-testkit": "0.3.
|
|
37
|
-
"@frockbot/workspace-store": "0.3.
|
|
36
|
+
"@frockbot/plugin-testkit": "0.3.22",
|
|
37
|
+
"@frockbot/workspace-store": "0.3.22",
|
|
38
38
|
"@types/bun": "1.4.0",
|
|
39
39
|
"@types/node": "26.2.0",
|
|
40
40
|
"typescript": "^7.0.2"
|
package/src/computer.ts
CHANGED
|
@@ -185,8 +185,16 @@ export interface FlySpriteComputerOptions {
|
|
|
185
185
|
}
|
|
186
186
|
|
|
187
187
|
export interface BrowserAction {
|
|
188
|
-
action:
|
|
188
|
+
action:
|
|
189
|
+
| "snapshot"
|
|
190
|
+
| "navigate"
|
|
191
|
+
| "close-origins"
|
|
192
|
+
| "click"
|
|
193
|
+
| "fill"
|
|
194
|
+
| "press"
|
|
195
|
+
| "wait";
|
|
189
196
|
url?: string;
|
|
197
|
+
origins?: readonly string[];
|
|
190
198
|
role?: string;
|
|
191
199
|
name?: string;
|
|
192
200
|
label?: string;
|
package/src/doctor.test.ts
CHANGED
|
@@ -22,10 +22,20 @@ function report(generation: number): string {
|
|
|
22
22
|
generation,
|
|
23
23
|
capturedAt: "2026-09-01T00:00:00Z",
|
|
24
24
|
checks: [
|
|
25
|
+
{
|
|
26
|
+
name: "watchdog",
|
|
27
|
+
status: "pass",
|
|
28
|
+
detail: "recent actions: none",
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
name: "memory-top",
|
|
32
|
+
status: "pass",
|
|
33
|
+
detail: "123 2048 chromium",
|
|
34
|
+
},
|
|
25
35
|
{ name: "disk-root", status: "pass", detail: "12% full" },
|
|
26
36
|
{ name: "dns", status: "fail", detail: "no resolver" },
|
|
27
37
|
],
|
|
28
|
-
summary: "
|
|
38
|
+
summary: "4 checks, 3 passed, 1 failed",
|
|
29
39
|
})}\n`;
|
|
30
40
|
}
|
|
31
41
|
|
|
@@ -52,8 +62,10 @@ describe("doctorForAgent", () => {
|
|
|
52
62
|
|
|
53
63
|
const decoded = await bot.doctor(signal());
|
|
54
64
|
|
|
55
|
-
expect(decoded.summary).toBe("
|
|
65
|
+
expect(decoded.summary).toBe("4 checks, 3 passed, 1 failed");
|
|
56
66
|
expect(decoded.checks.map((check) => check.status)).toEqual([
|
|
67
|
+
"pass",
|
|
68
|
+
"pass",
|
|
57
69
|
"pass",
|
|
58
70
|
"fail",
|
|
59
71
|
]);
|
package/src/host-double.ts
CHANGED
|
@@ -16,12 +16,13 @@
|
|
|
16
16
|
* `host-client.test.ts`'s subject and the workerd suite's, and repeating it
|
|
17
17
|
* here would test the transport three more times and the provider none.
|
|
18
18
|
*/
|
|
19
|
-
import
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
import {
|
|
20
|
+
COMPUTER_HOST_LIMITS,
|
|
21
|
+
type ComputerHostControlResultV1,
|
|
22
|
+
type ComputerHostFileReadResultV1,
|
|
23
|
+
type ComputerHostOpenResultV1,
|
|
24
|
+
type ComputerHostProvisioningV1,
|
|
25
|
+
type ComputerHostViewerResultV1,
|
|
25
26
|
} from "@frockbot/computer-host-protocol";
|
|
26
27
|
import { DESKTOP_GUI_LEASE_KEY } from "@frockbot/computer-host-runtime";
|
|
27
28
|
import {
|
|
@@ -136,6 +137,14 @@ export class FakeComputerHost {
|
|
|
136
137
|
options?: ComputerHostCallOptions,
|
|
137
138
|
): Promise<ComputerHostExecOutcomeV1> {
|
|
138
139
|
options?.signal?.throwIfAborted();
|
|
140
|
+
// The real host's decoder refuses an oversized script, and a double
|
|
141
|
+
// that accepted one would let a suite prove a push works at a size the
|
|
142
|
+
// Computer would never have been handed.
|
|
143
|
+
if (command.script.length > COMPUTER_HOST_LIMITS.script) {
|
|
144
|
+
throw new Error(
|
|
145
|
+
`script exceeds ${COMPUTER_HOST_LIMITS.script} characters`,
|
|
146
|
+
);
|
|
147
|
+
}
|
|
139
148
|
host.commands.push({
|
|
140
149
|
botId,
|
|
141
150
|
script: command.script,
|
package/src/provider.ts
CHANGED
|
@@ -109,6 +109,8 @@ function browserAction(action: ComputerBrowserAction): BrowserAction {
|
|
|
109
109
|
return { action: "snapshot" };
|
|
110
110
|
case "navigate":
|
|
111
111
|
return { action: "navigate", url: action.url };
|
|
112
|
+
case "close-origins":
|
|
113
|
+
return { action: "close-origins", origins: action.origins };
|
|
112
114
|
case "click":
|
|
113
115
|
return {
|
|
114
116
|
action: "click",
|
|
@@ -315,7 +317,7 @@ function summarize(report: WorkspaceSyncReportV1): ComputerSyncSummaryV1 {
|
|
|
315
317
|
}
|
|
316
318
|
if (failed) {
|
|
317
319
|
detail.push(
|
|
318
|
-
`${report.failures.length} sync ${report.failures.length === 1 ? "operation" : "operations"} failed: ${failed.status}: ${failed.reason}.`,
|
|
320
|
+
`${report.failures.length} sync ${report.failures.length === 1 ? "operation" : "operations"} failed${failed.path ? ` at "${failed.path}"` : ""}: ${failed.status}: ${failed.reason}.`,
|
|
319
321
|
);
|
|
320
322
|
}
|
|
321
323
|
const summary: ComputerSyncSummaryV1 = {
|
package/src/sync.test.ts
CHANGED
|
@@ -25,8 +25,10 @@ import {
|
|
|
25
25
|
FlySpriteSyncSurface,
|
|
26
26
|
isWorkspaceSyncIgnoredPathV1,
|
|
27
27
|
WORKSPACE_SYNC_IGNORED_DIRECTORIES_V1,
|
|
28
|
+
WORKSPACE_SYNC_CHUNK_BYTES_V1,
|
|
28
29
|
WORKSPACE_SYNC_MANIFEST_MAX_BYTES_V1,
|
|
29
30
|
WORKSPACE_SYNC_MANIFEST_MAX_ENTRIES_V1,
|
|
31
|
+
WORKSPACE_SYNC_MAX_FILE_BYTES_V1,
|
|
30
32
|
type WorkspaceSyncReportV1,
|
|
31
33
|
} from "./sync.ts";
|
|
32
34
|
import { WORKSPACE_EMPTY_SHA256 } from "./workspace.ts";
|
|
@@ -65,6 +67,19 @@ function sha256(bytes: Uint8Array): string {
|
|
|
65
67
|
return createHash("sha256").update(bytes).digest("hex");
|
|
66
68
|
}
|
|
67
69
|
|
|
70
|
+
/**
|
|
71
|
+
* Text of an exact length whose every chunk differs from every other, so a
|
|
72
|
+
* chunk carried twice, dropped, or reordered changes the bytes rather than
|
|
73
|
+
* hiding inside a repeated pattern.
|
|
74
|
+
*/
|
|
75
|
+
function largeText(length: number): string {
|
|
76
|
+
let text = "";
|
|
77
|
+
for (let index = 0; text.length < length; index += 1) {
|
|
78
|
+
text += `${index}:${"abcdefghijklmnopqrstuvwxyz".repeat(3)}\n`;
|
|
79
|
+
}
|
|
80
|
+
return text.slice(0, length);
|
|
81
|
+
}
|
|
82
|
+
|
|
68
83
|
function quoted(shell: string, name: string): string | undefined {
|
|
69
84
|
return new RegExp(`${name}='([^']*)'`).exec(shell)?.[1];
|
|
70
85
|
}
|
|
@@ -95,9 +110,12 @@ class FakeSyncSprite {
|
|
|
95
110
|
lastScanScript?: string;
|
|
96
111
|
/** Drops the next sidecar write, as a pause between store and Computer does. */
|
|
97
112
|
dropNextMaterialize = false;
|
|
113
|
+
/** Every script this Computer was handed, in order. */
|
|
114
|
+
readonly scripts: string[] = [];
|
|
98
115
|
|
|
99
116
|
/** Runs one script for the host double. */
|
|
100
117
|
readonly run = (script: string): FakeComputerRunV1 => {
|
|
118
|
+
this.scripts.push(script);
|
|
101
119
|
// A paused Sprite answers nothing, and the host reports the failed exit;
|
|
102
120
|
// the provider turns that into `Sprite storage operation failed: …`.
|
|
103
121
|
if (this.paused) return { exitCode: 1, stderr: "Sprite is paused" };
|
|
@@ -149,6 +167,7 @@ class FakeSyncSprite {
|
|
|
149
167
|
);
|
|
150
168
|
return this.scan(root ?? "", required);
|
|
151
169
|
}
|
|
170
|
+
if (shell.includes("__STAGED__")) return this.stageChunk(shell);
|
|
152
171
|
if (root && relative) {
|
|
153
172
|
if (shell.includes("__SYNCED__")) {
|
|
154
173
|
if (this.dropNextMaterialize) {
|
|
@@ -162,7 +181,7 @@ class FakeSyncSprite {
|
|
|
162
181
|
if (shell.includes("__FORGOTTEN__")) return this.forget(root, relative);
|
|
163
182
|
if (shell.includes("__PRESERVED__"))
|
|
164
183
|
return this.preserve(root, relative, shell);
|
|
165
|
-
return this.
|
|
184
|
+
return this.readFile(`${root}/${relative}`, shell);
|
|
166
185
|
}
|
|
167
186
|
const note = quoted(shell, "NOTE");
|
|
168
187
|
if (note) {
|
|
@@ -182,6 +201,65 @@ class FakeSyncSprite {
|
|
|
182
201
|
return "";
|
|
183
202
|
}
|
|
184
203
|
|
|
204
|
+
/**
|
|
205
|
+
* The chunked file read: a header of size and digest with the first chunk,
|
|
206
|
+
* then one chunk per further command. `head -c` and `tail -c +N` are the
|
|
207
|
+
* coreutils the sync emits; the arithmetic is theirs, not an approximation.
|
|
208
|
+
*/
|
|
209
|
+
private readFile(path: string, shell: string): string {
|
|
210
|
+
const bytes = this.files.get(path);
|
|
211
|
+
const chunk = /tail -c \+(\d+) "\$FILE" \| head -c (\d+)/.exec(shell);
|
|
212
|
+
if (chunk) {
|
|
213
|
+
const offset = Number(chunk[1]) - 1;
|
|
214
|
+
const limit = Number(chunk[2]);
|
|
215
|
+
const slice = bytes?.subarray(offset, offset + limit) ?? new Uint8Array();
|
|
216
|
+
return `${Buffer.from(slice).toString("base64")}\n`;
|
|
217
|
+
}
|
|
218
|
+
if (!bytes) return "__MISSING__\n";
|
|
219
|
+
if (bytes.byteLength > WORKSPACE_SYNC_MAX_FILE_BYTES_V1) {
|
|
220
|
+
return "__TOO_LARGE__\n";
|
|
221
|
+
}
|
|
222
|
+
const head = bytes.subarray(0, WORKSPACE_SYNC_CHUNK_BYTES_V1);
|
|
223
|
+
return `${bytes.byteLength}\t${sha256(bytes)}\n${Buffer.from(head).toString("base64")}\n`;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** One appended chunk of a staged file, as the push writes it. */
|
|
227
|
+
private stageChunk(shell: string): string {
|
|
228
|
+
const path = quoted(shell, "STAGE") ?? "";
|
|
229
|
+
const encoded =
|
|
230
|
+
/printf %s '([^']*)' \| base64 -d >> "\$STAGE"/.exec(shell)?.[1] ?? "";
|
|
231
|
+
const chunk = Buffer.from(encoded, "base64");
|
|
232
|
+
const held = shell.includes('rm -f "$STAGE"')
|
|
233
|
+
? undefined
|
|
234
|
+
: this.files.get(path);
|
|
235
|
+
this.files.set(
|
|
236
|
+
path,
|
|
237
|
+
Uint8Array.from(held ? Buffer.concat([Buffer.from(held), chunk]) : chunk),
|
|
238
|
+
);
|
|
239
|
+
return "__STAGED__\n";
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* The bytes a commit command puts in place: written inline when they fitted
|
|
244
|
+
* in one command, otherwise the staged file it moves, digest-checked exactly
|
|
245
|
+
* as the emitted `sha256sum` line checks it.
|
|
246
|
+
*/
|
|
247
|
+
private committed(
|
|
248
|
+
shell: string,
|
|
249
|
+
target: string,
|
|
250
|
+
): Uint8Array | "corrupt" | undefined {
|
|
251
|
+
const stage = quoted(shell, "STAGE");
|
|
252
|
+
if (stage) {
|
|
253
|
+
const expected = /f1\)" != '([0-9a-f]{64})'/.exec(shell)?.[1];
|
|
254
|
+
const bytes = this.files.get(stage) ?? new Uint8Array();
|
|
255
|
+
this.files.delete(stage);
|
|
256
|
+
return expected && sha256(bytes) !== expected ? "corrupt" : bytes;
|
|
257
|
+
}
|
|
258
|
+
const encoded = payload(shell, target);
|
|
259
|
+
if (encoded === undefined) return undefined;
|
|
260
|
+
return Uint8Array.from(Buffer.from(encoded, "base64"));
|
|
261
|
+
}
|
|
262
|
+
|
|
185
263
|
private read(path: string, base64 = true): string {
|
|
186
264
|
const bytes = this.files.get(path);
|
|
187
265
|
if (!bytes) return "__MISSING__\n";
|
|
@@ -280,12 +358,10 @@ class FakeSyncSprite {
|
|
|
280
358
|
}
|
|
281
359
|
|
|
282
360
|
private materialize(root: string, relative: string, shell: string): string {
|
|
283
|
-
const bytes =
|
|
361
|
+
const bytes = this.committed(shell, "TMP");
|
|
362
|
+
if (bytes === "corrupt") return "__CORRUPT__\n";
|
|
284
363
|
const meta = payload(shell, "MTMP") ?? "";
|
|
285
|
-
this.files.set(
|
|
286
|
-
`${root}/${relative}`,
|
|
287
|
-
Uint8Array.from(Buffer.from(bytes, "base64")),
|
|
288
|
-
);
|
|
364
|
+
this.files.set(`${root}/${relative}`, bytes ?? new Uint8Array());
|
|
289
365
|
this.files.set(
|
|
290
366
|
`${root}/.frockbot-generations/${relative}`,
|
|
291
367
|
Uint8Array.from(Buffer.from(meta, "base64")),
|
|
@@ -316,10 +392,11 @@ class FakeSyncSprite {
|
|
|
316
392
|
private preserve(root: string, relative: string, shell: string): string {
|
|
317
393
|
const generationId =
|
|
318
394
|
/conflicts\/\$REL\/([^"]*)"/.exec(shell)?.[1] ?? "unknown";
|
|
319
|
-
const bytes =
|
|
395
|
+
const bytes = this.committed(shell, "TMP");
|
|
396
|
+
if (bytes === "corrupt") return "__CORRUPT__\n";
|
|
320
397
|
this.files.set(
|
|
321
398
|
`${root}/.frockbot-sync/conflicts/${relative}/${generationId}`,
|
|
322
|
-
Uint8Array
|
|
399
|
+
bytes ?? new Uint8Array(),
|
|
323
400
|
);
|
|
324
401
|
return "__PRESERVED__\n";
|
|
325
402
|
}
|
|
@@ -1409,6 +1486,69 @@ describe("the durable-root sync on the Computer handle", () => {
|
|
|
1409
1486
|
expect(refused.status).toBe("refused");
|
|
1410
1487
|
});
|
|
1411
1488
|
|
|
1489
|
+
// Production, 2026-09-04: `applet build` wrote a 470 KB `dist/ui.html`, and
|
|
1490
|
+
// every publish that followed failed, because one `base64` of the file was
|
|
1491
|
+
// 627 KB of answer and the storage command may answer 500 KB. Both directions
|
|
1492
|
+
// are checked at 700 KB, which is past that ceiling and past the point where
|
|
1493
|
+
// one script could carry the base64 either.
|
|
1494
|
+
test("pushes a file far larger than one storage command can answer, byte for byte", async () => {
|
|
1495
|
+
const { sprite, store, open } = providerHarness([
|
|
1496
|
+
APPLET_SOURCE_PACKAGE_ROOT,
|
|
1497
|
+
]);
|
|
1498
|
+
const handle = await open();
|
|
1499
|
+
const appletId = "pub-user-1.0123456789abcdef0123456789abcdef";
|
|
1500
|
+
const page = largeText(700_000);
|
|
1501
|
+
sprite.shellWrite(MOUNTS.appletSource, `${appletId}/dist/ui.html`, page);
|
|
1502
|
+
|
|
1503
|
+
const summary = await handle.sync!.reconcileRoot!(
|
|
1504
|
+
appletSourceRoot,
|
|
1505
|
+
"publish",
|
|
1506
|
+
{ requiredPaths: [`${appletId}/dist/ui.html`] },
|
|
1507
|
+
);
|
|
1508
|
+
|
|
1509
|
+
expect(summary).toMatchObject({ status: "ok", pushed: 1, failures: 0 });
|
|
1510
|
+
const stored = await store.read({
|
|
1511
|
+
root: appletSourceRoot,
|
|
1512
|
+
path: `${appletId}/dist/ui.html`,
|
|
1513
|
+
});
|
|
1514
|
+
expect(stored.status).toBe("ok");
|
|
1515
|
+
if (stored.status === "ok") {
|
|
1516
|
+
expect(stored.file.bytes.byteLength).toBe(page.length);
|
|
1517
|
+
expect(decoder.decode(stored.file.bytes)).toBe(page);
|
|
1518
|
+
}
|
|
1519
|
+
});
|
|
1520
|
+
|
|
1521
|
+
// The other direction has the other ceiling: the bytes travel out as base64
|
|
1522
|
+
// inside the script, and a script may be 1 MB, so 900 KB of file is 1.2 MB of
|
|
1523
|
+
// script and the whole command was refused before it ran.
|
|
1524
|
+
test("pulls a file far larger than one storage command can carry, byte for byte", async () => {
|
|
1525
|
+
const { sprite, store, open } = providerHarness();
|
|
1526
|
+
const page = largeText(900_000);
|
|
1527
|
+
await writeToStore(store, skillsRoot, "reference.md", page, BOT_WRITER);
|
|
1528
|
+
|
|
1529
|
+
const handle = await open();
|
|
1530
|
+
const summary = await handle.sync!.reconcile("open");
|
|
1531
|
+
|
|
1532
|
+
expect(summary).toMatchObject({ status: "ok", pulled: 1, failures: 0 });
|
|
1533
|
+
expect(sprite.text(`${MOUNTS.skills}/reference.md`)).toBe(page);
|
|
1534
|
+
// The staging file the chunks were assembled in is moved into place, never
|
|
1535
|
+
// left behind for the next scan to find.
|
|
1536
|
+
expect(sprite.keys(`${MOUNTS.skills}/.frockbot-sync/staging`)).toEqual([]);
|
|
1537
|
+
});
|
|
1538
|
+
|
|
1539
|
+
test("carries a file that fits in one command in one command", async () => {
|
|
1540
|
+
const { sprite, store, open } = providerHarness();
|
|
1541
|
+
await writeToStore(store, skillsRoot, "small.md", "# small", BOT_WRITER);
|
|
1542
|
+
const handle = await open();
|
|
1543
|
+
|
|
1544
|
+
await handle.sync!.reconcile("open");
|
|
1545
|
+
|
|
1546
|
+
expect(sprite.text(`${MOUNTS.skills}/small.md`)).toBe("# small");
|
|
1547
|
+
expect(
|
|
1548
|
+
sprite.scripts.filter((script) => script.includes("__STAGED__")),
|
|
1549
|
+
).toEqual([]);
|
|
1550
|
+
});
|
|
1551
|
+
|
|
1412
1552
|
test("answers unavailable rather than throwing when the Sprite is paused", async () => {
|
|
1413
1553
|
const { sprite, open } = providerHarness();
|
|
1414
1554
|
const handle = await open();
|
package/src/sync.ts
CHANGED
|
@@ -103,6 +103,7 @@ import {
|
|
|
103
103
|
import type { FlySpriteAgentComputer } from "./computer.js";
|
|
104
104
|
import {
|
|
105
105
|
SYNC_CONFLICTS_DIR,
|
|
106
|
+
SYNC_STAGING_DIR,
|
|
106
107
|
SYNC_TOMBSTONES_DIR,
|
|
107
108
|
WORKSPACE_EMPTY_SHA256,
|
|
108
109
|
WORKSPACE_GENERATIONS_DIR,
|
|
@@ -145,6 +146,49 @@ export const WORKSPACE_SYNC_MANIFEST_MAX_BYTES_V1 = 400_000;
|
|
|
145
146
|
/** Bounds work and rows even when unusually short paths fit under the bytes. */
|
|
146
147
|
export const WORKSPACE_SYNC_MANIFEST_MAX_ENTRIES_V1 = 2_000;
|
|
147
148
|
|
|
149
|
+
/**
|
|
150
|
+
* The most file bytes one storage command carries, in either direction.
|
|
151
|
+
*
|
|
152
|
+
* A command's answer travels as base64, so a chunk this size comes back as
|
|
153
|
+
* roughly 350 KB — inside the 500 KB an answer may be — and goes out inside a
|
|
154
|
+
* script well inside the 1 MB a script may be. A file larger than one chunk
|
|
155
|
+
* travels as several commands rather than as one command that would be refused
|
|
156
|
+
* for its size, which is the whole reason the sync reads and writes in chunks.
|
|
157
|
+
*/
|
|
158
|
+
export const WORKSPACE_SYNC_CHUNK_BYTES_V1 = 256 * 1024;
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The largest durable-root file the sync carries between the Computer and the
|
|
162
|
+
* store.
|
|
163
|
+
*
|
|
164
|
+
* It sits above `WORKSPACE_MAX_FILE_BYTES`, the bound the Workspace puts
|
|
165
|
+
* on one durable-root file, so the transport is never what decides: a file too
|
|
166
|
+
* large to keep is refused for its size by the Workspace, never for the shape
|
|
167
|
+
* of the commands that would have moved it.
|
|
168
|
+
*/
|
|
169
|
+
export const WORKSPACE_SYNC_MAX_FILE_BYTES_V1 = 4 * 1024 * 1024;
|
|
170
|
+
|
|
171
|
+
/** Every chunk offset after the first, in bytes, for a file of this size. */
|
|
172
|
+
function chunkOffsets(size: number): number[] {
|
|
173
|
+
const offsets: number[] = [];
|
|
174
|
+
for (
|
|
175
|
+
let offset = WORKSPACE_SYNC_CHUNK_BYTES_V1;
|
|
176
|
+
offset < size;
|
|
177
|
+
offset += WORKSPACE_SYNC_CHUNK_BYTES_V1
|
|
178
|
+
) {
|
|
179
|
+
offsets.push(offset);
|
|
180
|
+
}
|
|
181
|
+
return offsets;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** A staging file name that is one path segment whatever the id looks like. */
|
|
185
|
+
function stagingName(generationId: string, kind: string): string {
|
|
186
|
+
const safe = generationId.replaceAll(/[^A-Za-z0-9._-]/g, "-").slice(0, 128);
|
|
187
|
+
return `${safe}.${kind}`;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const SHA256_HEX = /^[0-9a-f]{64}$/;
|
|
191
|
+
|
|
148
192
|
const ignoredDirectoryNames = new Set<string>(
|
|
149
193
|
WORKSPACE_SYNC_IGNORED_DIRECTORIES_V1,
|
|
150
194
|
);
|
|
@@ -1132,6 +1176,18 @@ export class FlySpriteSyncSurface implements ComputerSyncSurfaceV1 {
|
|
|
1132
1176
|
};
|
|
1133
1177
|
}
|
|
1134
1178
|
|
|
1179
|
+
/**
|
|
1180
|
+
* Pulls one file off the Computer, in bounded chunks.
|
|
1181
|
+
*
|
|
1182
|
+
* A command's answer has a hard ceiling, and base64 is a third larger than
|
|
1183
|
+
* the bytes it carries, so one `base64` of a whole file stopped working long
|
|
1184
|
+
* before a file got large: half a megabyte of built Applet page is two thirds
|
|
1185
|
+
* of a megabyte of answer, and the command was refused rather than truncated.
|
|
1186
|
+
* So the first command answers the size and digest and the first chunk, and
|
|
1187
|
+
* one command per further chunk brings the rest. The digest is the proof the
|
|
1188
|
+
* pieces are one file: a file rewritten between two chunks fails the check
|
|
1189
|
+
* and is reported, never stitched together from two different versions.
|
|
1190
|
+
*/
|
|
1135
1191
|
async read(
|
|
1136
1192
|
root: WorkspaceRootV1,
|
|
1137
1193
|
path: string,
|
|
@@ -1140,21 +1196,109 @@ export class FlySpriteSyncSurface implements ComputerSyncSurfaceV1 {
|
|
|
1140
1196
|
if (typeof mount !== "string") return mount;
|
|
1141
1197
|
const relative = this.relative(path);
|
|
1142
1198
|
if (typeof relative !== "string") return relative;
|
|
1143
|
-
const
|
|
1199
|
+
const prelude = [
|
|
1200
|
+
"set -eu",
|
|
1144
1201
|
`ROOT=${shellQuote(mount)}`,
|
|
1145
1202
|
`REL=${shellQuote(relative)}`,
|
|
1146
|
-
'
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1203
|
+
'FILE="$ROOT/$REL"',
|
|
1204
|
+
];
|
|
1205
|
+
const head = await this.run(
|
|
1206
|
+
[
|
|
1207
|
+
...prelude,
|
|
1208
|
+
`CHUNK=${WORKSPACE_SYNC_CHUNK_BYTES_V1}`,
|
|
1209
|
+
'if [ ! -f "$FILE" ]; then echo __MISSING__; exit 0; fi',
|
|
1210
|
+
'SIZE=$(stat -c %s "$FILE")',
|
|
1211
|
+
`if [ "$SIZE" -gt ${WORKSPACE_SYNC_MAX_FILE_BYTES_V1} ]; then echo __TOO_LARGE__; exit 0; fi`,
|
|
1212
|
+
'printf "%s\\t%s\\n" "$SIZE" "$(sha256sum "$FILE" | cut -d" " -f1)"',
|
|
1213
|
+
'head -c "$CHUNK" "$FILE" | base64 -w0; echo',
|
|
1214
|
+
].join("\n"),
|
|
1215
|
+
);
|
|
1216
|
+
if (typeof head !== "string") return head;
|
|
1217
|
+
if (head.includes("__MISSING__")) {
|
|
1152
1218
|
return failure("not-found", `No such Workspace file: ${relative}`);
|
|
1153
1219
|
}
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1220
|
+
if (head.includes("__TOO_LARGE__")) {
|
|
1221
|
+
return failure(
|
|
1222
|
+
"refused",
|
|
1223
|
+
`"${relative}" is past the ${WORKSPACE_SYNC_MAX_FILE_BYTES_V1}-byte limit on a synced Workspace file`,
|
|
1224
|
+
);
|
|
1225
|
+
}
|
|
1226
|
+
const [header = "", first = ""] = head.split("\n");
|
|
1227
|
+
const [declared = "", digest = ""] = header.split("\t");
|
|
1228
|
+
const size = Number(declared.trim());
|
|
1229
|
+
if (!Number.isSafeInteger(size) || size < 0 || !SHA256_HEX.test(digest)) {
|
|
1230
|
+
return failure("unavailable", "Invalid Fly Workspace sync response");
|
|
1231
|
+
}
|
|
1232
|
+
const chunks = [Buffer.from(first.trim(), "base64")];
|
|
1233
|
+
for (const offset of chunkOffsets(size)) {
|
|
1234
|
+
const next = await this.run(
|
|
1235
|
+
[
|
|
1236
|
+
...prelude,
|
|
1237
|
+
// `tail -c +N` counts from one, so the offset is one past the bytes
|
|
1238
|
+
// already carried.
|
|
1239
|
+
`tail -c +${offset + 1} "$FILE" | head -c ${WORKSPACE_SYNC_CHUNK_BYTES_V1} | base64 -w0; echo`,
|
|
1240
|
+
].join("\n"),
|
|
1241
|
+
);
|
|
1242
|
+
if (typeof next !== "string") return next;
|
|
1243
|
+
chunks.push(Buffer.from(next.trim(), "base64"));
|
|
1244
|
+
}
|
|
1245
|
+
const bytes = Buffer.concat(chunks);
|
|
1246
|
+
if (
|
|
1247
|
+
bytes.byteLength !== size ||
|
|
1248
|
+
createHash("sha256").update(bytes).digest("hex") !== digest
|
|
1249
|
+
) {
|
|
1250
|
+
return failure(
|
|
1251
|
+
"unavailable",
|
|
1252
|
+
`"${relative}" changed on the Computer while it was being read`,
|
|
1253
|
+
);
|
|
1254
|
+
}
|
|
1255
|
+
return { status: "ok", bytes: Uint8Array.from(bytes) };
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
/**
|
|
1259
|
+
* Puts bytes on the Computer under the sync's own staging directory, in
|
|
1260
|
+
* bounded chunks, and answers where they landed.
|
|
1261
|
+
*
|
|
1262
|
+
* The push has the pull's problem in the other direction: a script carrying a
|
|
1263
|
+
* whole file as base64 is refused for its length. Bytes that fit in one chunk
|
|
1264
|
+
* take no staging file at all — the caller writes them inline, which is one
|
|
1265
|
+
* command and the overwhelmingly common case. Anything larger is appended
|
|
1266
|
+
* chunk by chunk and the caller's own command moves it into place, so a file
|
|
1267
|
+
* only ever appears at its real path complete.
|
|
1268
|
+
*/
|
|
1269
|
+
private async stage(
|
|
1270
|
+
mount: string,
|
|
1271
|
+
name: string,
|
|
1272
|
+
bytes: Uint8Array,
|
|
1273
|
+
): Promise<string | WorkspaceFailureV1 | undefined> {
|
|
1274
|
+
if (bytes.byteLength <= WORKSPACE_SYNC_CHUNK_BYTES_V1) return undefined;
|
|
1275
|
+
const staged = `${mount}/${WORKSPACE_SYNC_DIR}/${SYNC_STAGING_DIR}/${name}`;
|
|
1276
|
+
for (
|
|
1277
|
+
let offset = 0;
|
|
1278
|
+
offset < bytes.byteLength;
|
|
1279
|
+
offset += WORKSPACE_SYNC_CHUNK_BYTES_V1
|
|
1280
|
+
) {
|
|
1281
|
+
const chunk = bytes.subarray(
|
|
1282
|
+
offset,
|
|
1283
|
+
offset + WORKSPACE_SYNC_CHUNK_BYTES_V1,
|
|
1284
|
+
);
|
|
1285
|
+
const output = await this.run(
|
|
1286
|
+
[
|
|
1287
|
+
"set -eu",
|
|
1288
|
+
`STAGE=${shellQuote(staged)}`,
|
|
1289
|
+
'mkdir -p "$(dirname "$STAGE")"',
|
|
1290
|
+
...(offset === 0 ? ['rm -f "$STAGE"'] : []),
|
|
1291
|
+
`printf %s ${shellQuote(Buffer.from(chunk).toString("base64"))} | base64 -d >> "$STAGE"`,
|
|
1292
|
+
'chmod 600 "$STAGE"',
|
|
1293
|
+
"echo __STAGED__",
|
|
1294
|
+
].join("\n"),
|
|
1295
|
+
);
|
|
1296
|
+
if (typeof output !== "string") return output;
|
|
1297
|
+
if (!output.includes("__STAGED__")) {
|
|
1298
|
+
return failure("unavailable", "Invalid Fly Workspace sync response");
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
return staged;
|
|
1158
1302
|
}
|
|
1159
1303
|
|
|
1160
1304
|
async materialize(
|
|
@@ -1167,6 +1311,14 @@ export class FlySpriteSyncSurface implements ComputerSyncSurfaceV1 {
|
|
|
1167
1311
|
if (typeof mount !== "string") return mount;
|
|
1168
1312
|
const relative = this.relative(path);
|
|
1169
1313
|
if (typeof relative !== "string") return relative;
|
|
1314
|
+
const oversized = this.oversized(relative, bytes);
|
|
1315
|
+
if (oversized) return oversized;
|
|
1316
|
+
const staged = await this.stage(
|
|
1317
|
+
mount,
|
|
1318
|
+
stagingName(generation.generationId, "pull"),
|
|
1319
|
+
bytes,
|
|
1320
|
+
);
|
|
1321
|
+
if (staged && typeof staged !== "string") return staged;
|
|
1170
1322
|
const script = [
|
|
1171
1323
|
"set -eu",
|
|
1172
1324
|
`ROOT=${shellQuote(mount)}`,
|
|
@@ -1176,7 +1328,7 @@ export class FlySpriteSyncSurface implements ComputerSyncSurfaceV1 {
|
|
|
1176
1328
|
`GRAVE="$ROOT/${WORKSPACE_SYNC_DIR}/${SYNC_TOMBSTONES_DIR}/$REL"`,
|
|
1177
1329
|
'mkdir -p "$(dirname "$TARGET")" "$(dirname "$META")"',
|
|
1178
1330
|
'TMP=$(mktemp "${TARGET}.XXXXXX")',
|
|
1179
|
-
|
|
1331
|
+
...this.assemble(staged, bytes, generation.contentHash),
|
|
1180
1332
|
'chmod 600 "$TMP"',
|
|
1181
1333
|
'mv "$TMP" "$TARGET"',
|
|
1182
1334
|
'MTMP=$(mktemp "${META}.XXXXXX")',
|
|
@@ -1188,12 +1340,56 @@ export class FlySpriteSyncSurface implements ComputerSyncSurfaceV1 {
|
|
|
1188
1340
|
].join("\n");
|
|
1189
1341
|
const output = await this.run(script);
|
|
1190
1342
|
if (typeof output !== "string") return output;
|
|
1343
|
+
if (output.includes("__CORRUPT__")) {
|
|
1344
|
+
return this.corrupt(relative);
|
|
1345
|
+
}
|
|
1191
1346
|
if (!output.includes("__SYNCED__")) {
|
|
1192
1347
|
return failure("unavailable", "Invalid Fly Workspace sync response");
|
|
1193
1348
|
}
|
|
1194
1349
|
return { status: "ok" };
|
|
1195
1350
|
}
|
|
1196
1351
|
|
|
1352
|
+
/** Refuses bytes the sync will not carry, before a command is composed. */
|
|
1353
|
+
private oversized(
|
|
1354
|
+
relative: string,
|
|
1355
|
+
bytes: Uint8Array,
|
|
1356
|
+
): WorkspaceFailureV1 | undefined {
|
|
1357
|
+
if (bytes.byteLength <= WORKSPACE_SYNC_MAX_FILE_BYTES_V1) return undefined;
|
|
1358
|
+
return failure(
|
|
1359
|
+
"refused",
|
|
1360
|
+
`"${relative}" is past the ${WORKSPACE_SYNC_MAX_FILE_BYTES_V1}-byte limit on a synced Workspace file`,
|
|
1361
|
+
);
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
private corrupt(relative: string): WorkspaceFailureV1 {
|
|
1365
|
+
return failure(
|
|
1366
|
+
"unavailable",
|
|
1367
|
+
`"${relative}" did not reach the Computer intact`,
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
/**
|
|
1372
|
+
* The lines that fill `$TMP` with the bytes: inline when they fitted in one
|
|
1373
|
+
* command, otherwise the staged file, checked against its digest before it is
|
|
1374
|
+
* used so a half-written staging file never becomes a durable-root file.
|
|
1375
|
+
*/
|
|
1376
|
+
private assemble(
|
|
1377
|
+
staged: string | undefined,
|
|
1378
|
+
bytes: Uint8Array,
|
|
1379
|
+
contentHash: string,
|
|
1380
|
+
): string[] {
|
|
1381
|
+
if (!staged) {
|
|
1382
|
+
return [
|
|
1383
|
+
`printf %s ${shellQuote(Buffer.from(bytes).toString("base64"))} | base64 -d > "$TMP"`,
|
|
1384
|
+
];
|
|
1385
|
+
}
|
|
1386
|
+
return [
|
|
1387
|
+
`STAGE=${shellQuote(staged)}`,
|
|
1388
|
+
`if [ "$(sha256sum "$STAGE" | cut -d" " -f1)" != ${shellQuote(contentHash)} ]; then rm -f "$STAGE"; echo __CORRUPT__; exit 0; fi`,
|
|
1389
|
+
'mv "$STAGE" "$TMP"',
|
|
1390
|
+
];
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1197
1393
|
async remove(
|
|
1198
1394
|
root: WorkspaceRootV1,
|
|
1199
1395
|
path: string,
|
|
@@ -1262,18 +1458,31 @@ export class FlySpriteSyncSurface implements ComputerSyncSurfaceV1 {
|
|
|
1262
1458
|
if (typeof mount !== "string") return mount;
|
|
1263
1459
|
const relative = this.relative(path);
|
|
1264
1460
|
if (typeof relative !== "string") return relative;
|
|
1461
|
+
const oversized = this.oversized(relative, bytes);
|
|
1462
|
+
if (oversized) return oversized;
|
|
1463
|
+
const staged = await this.stage(
|
|
1464
|
+
mount,
|
|
1465
|
+
stagingName(generation.generationId, "conflict"),
|
|
1466
|
+
bytes,
|
|
1467
|
+
);
|
|
1468
|
+
if (staged && typeof staged !== "string") return staged;
|
|
1265
1469
|
const script = [
|
|
1266
1470
|
"set -eu",
|
|
1267
1471
|
`ROOT=${shellQuote(mount)}`,
|
|
1268
1472
|
`REL=${shellQuote(relative)}`,
|
|
1269
1473
|
`KEPT="$ROOT/${WORKSPACE_SYNC_DIR}/${SYNC_CONFLICTS_DIR}/$REL/${generation.generationId}"`,
|
|
1270
1474
|
'mkdir -p "$(dirname "$KEPT")"',
|
|
1271
|
-
|
|
1272
|
-
|
|
1475
|
+
'TMP=$(mktemp "${KEPT}.XXXXXX")',
|
|
1476
|
+
...this.assemble(staged, bytes, generation.contentHash),
|
|
1477
|
+
'chmod 600 "$TMP"',
|
|
1478
|
+
'mv "$TMP" "$KEPT"',
|
|
1273
1479
|
"echo __PRESERVED__",
|
|
1274
1480
|
].join("\n");
|
|
1275
1481
|
const output = await this.run(script);
|
|
1276
1482
|
if (typeof output !== "string") return output;
|
|
1483
|
+
if (output.includes("__CORRUPT__")) {
|
|
1484
|
+
return this.corrupt(relative);
|
|
1485
|
+
}
|
|
1277
1486
|
if (!output.includes("__PRESERVED__")) {
|
|
1278
1487
|
return failure("unavailable", "Invalid Fly Workspace sync response");
|
|
1279
1488
|
}
|
package/src/workspace.ts
CHANGED
|
@@ -71,6 +71,11 @@ export const WORKSPACE_SYNC_DIR = ".frockbot-sync";
|
|
|
71
71
|
export const SYNC_TOMBSTONES_DIR = "tombstones";
|
|
72
72
|
/** Where a losing write is preserved on the Computer, under `WORKSPACE_SYNC_DIR`. */
|
|
73
73
|
export const SYNC_CONFLICTS_DIR = "conflicts";
|
|
74
|
+
/**
|
|
75
|
+
* Where a file too large for one command is assembled chunk by chunk before it
|
|
76
|
+
* is moved into place, under `WORKSPACE_SYNC_DIR`.
|
|
77
|
+
*/
|
|
78
|
+
export const SYNC_STAGING_DIR = "staging";
|
|
74
79
|
const GENERATIONS_DIR = WORKSPACE_GENERATIONS_DIR;
|
|
75
80
|
const LOCKS_DIR = ".frockbot-locks";
|
|
76
81
|
/** The sha-256 of no bytes; a deletion tombstone's content address. */
|