@tickernelz/paperclip-pro-plugin-daytona 2026.925.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -0
- package/dist/duplex-command-stream.d.ts +97 -0
- package/dist/duplex-command-stream.d.ts.map +1 -0
- package/dist/duplex-command-stream.js +205 -0
- package/dist/duplex-command-stream.js.map +1 -0
- package/dist/duplex-command-stream.live.test.d.ts +2 -0
- package/dist/duplex-command-stream.live.test.d.ts.map +1 -0
- package/dist/duplex-command-stream.live.test.js +324 -0
- package/dist/duplex-command-stream.live.test.js.map +1 -0
- package/dist/duplex-command-stream.test.d.ts +2 -0
- package/dist/duplex-command-stream.test.d.ts.map +1 -0
- package/dist/duplex-command-stream.test.js +519 -0
- package/dist/duplex-command-stream.test.js.map +1 -0
- package/dist/file-sync.d.ts +77 -0
- package/dist/file-sync.d.ts.map +1 -0
- package/dist/file-sync.js +1055 -0
- package/dist/file-sync.js.map +1 -0
- package/dist/file-sync.test.d.ts +2 -0
- package/dist/file-sync.test.d.ts.map +1 -0
- package/dist/file-sync.test.js +974 -0
- package/dist/file-sync.test.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/login-pty.d.ts +162 -0
- package/dist/login-pty.d.ts.map +1 -0
- package/dist/login-pty.js +258 -0
- package/dist/login-pty.js.map +1 -0
- package/dist/login-pty.test.d.ts +2 -0
- package/dist/login-pty.test.d.ts.map +1 -0
- package/dist/login-pty.test.js +319 -0
- package/dist/login-pty.test.js.map +1 -0
- package/dist/manifest.d.ts +4 -0
- package/dist/manifest.d.ts.map +1 -0
- package/dist/manifest.js +179 -0
- package/dist/manifest.js.map +1 -0
- package/dist/plugin.d.ts +49 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/plugin.js +2563 -0
- package/dist/plugin.js.map +1 -0
- package/dist/plugin.test.d.ts +2 -0
- package/dist/plugin.test.d.ts.map +1 -0
- package/dist/plugin.test.js +4701 -0
- package/dist/plugin.test.js.map +1 -0
- package/dist/pty-chunked-input.d.ts +48 -0
- package/dist/pty-chunked-input.d.ts.map +1 -0
- package/dist/pty-chunked-input.js +74 -0
- package/dist/pty-chunked-input.js.map +1 -0
- package/dist/pty-chunked-input.test.d.ts +2 -0
- package/dist/pty-chunked-input.test.d.ts.map +1 -0
- package/dist/pty-chunked-input.test.js +115 -0
- package/dist/pty-chunked-input.test.js.map +1 -0
- package/dist/worker.d.ts +3 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +5 -0
- package/dist/worker.js.map +1 -0
- package/package.json +44 -0
|
@@ -0,0 +1,974 @@
|
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import crypto from "node:crypto";
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
import zlib from "node:zlib";
|
|
7
|
+
import { Transform } from "node:stream";
|
|
8
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
9
|
+
// The plugin module imports `@daytonaio/sdk` as a value, but the sync tests never
|
|
10
|
+
// touch a real Daytona client — every sandbox call goes through a local mock. Stub
|
|
11
|
+
// the SDK so the import resolves without the excluded provider package.
|
|
12
|
+
import { vi } from "vitest";
|
|
13
|
+
vi.mock("@daytonaio/sdk", () => ({
|
|
14
|
+
Daytona: class MockDaytona {
|
|
15
|
+
},
|
|
16
|
+
DaytonaNotFoundError: class MockDaytonaNotFoundError extends Error {
|
|
17
|
+
},
|
|
18
|
+
DaytonaTimeoutError: class MockDaytonaTimeoutError extends Error {
|
|
19
|
+
},
|
|
20
|
+
}));
|
|
21
|
+
import { performSyncIn } from "./file-sync.js";
|
|
22
|
+
import { __setDaytonaPluginContextForTest } from "./plugin.js";
|
|
23
|
+
// Build a mock Daytona sandbox for the inbound directory path. `executeCommand`
|
|
24
|
+
// records every command and returns exit 0, except that any command whose text
|
|
25
|
+
// matches `failCommandMatch` returns exit 1 (to simulate an extract failure).
|
|
26
|
+
// `uploadFiles` records each upload destination so a test can read the reserved
|
|
27
|
+
// scratch tar path the runtime chose.
|
|
28
|
+
function createMockSandbox(input) {
|
|
29
|
+
return {
|
|
30
|
+
process: {
|
|
31
|
+
executeCommand: async (command) => {
|
|
32
|
+
input.commands.push({ command });
|
|
33
|
+
if (input.failCommandMatch && input.failCommandMatch.test(command)) {
|
|
34
|
+
return { exitCode: 1, result: "simulated extract failure" };
|
|
35
|
+
}
|
|
36
|
+
return { exitCode: 0, result: "" };
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
fs: {
|
|
40
|
+
uploadFiles: async (uploads) => {
|
|
41
|
+
for (const upload of uploads)
|
|
42
|
+
input.uploadedDestinations.push(upload.destination);
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
describe("daytona file-sync inbound scratch cleanup", () => {
|
|
48
|
+
const cleanupDirs = [];
|
|
49
|
+
afterEach(async () => {
|
|
50
|
+
while (cleanupDirs.length > 0) {
|
|
51
|
+
const dir = cleanupDirs.pop();
|
|
52
|
+
if (!dir)
|
|
53
|
+
continue;
|
|
54
|
+
await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
it("removes the reserved scratch tar when a directory extraction fails", async () => {
|
|
58
|
+
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-daytona-scratch-"));
|
|
59
|
+
cleanupDirs.push(rootDir);
|
|
60
|
+
const sourceDir = path.join(rootDir, "referenced-project");
|
|
61
|
+
await fs.mkdir(sourceDir, { recursive: true });
|
|
62
|
+
await fs.writeFile(path.join(sourceDir, "README.md"), "referenced project\n", "utf8");
|
|
63
|
+
const remoteDir = "/workspace";
|
|
64
|
+
const targetPath = "/workspace/.paperclip-runtime/test-adapter/project-abc";
|
|
65
|
+
const uploadedDestinations = [];
|
|
66
|
+
const commands = [];
|
|
67
|
+
// Fail the extract round trip (the only command that runs `tar -xf`).
|
|
68
|
+
const sandbox = createMockSandbox({
|
|
69
|
+
failCommandMatch: /tar -xf/,
|
|
70
|
+
uploadedDestinations,
|
|
71
|
+
commands,
|
|
72
|
+
});
|
|
73
|
+
const operations = [{
|
|
74
|
+
operationId: "sync-op-1",
|
|
75
|
+
files: [{ sourcePath: sourceDir, targetPath, kind: "directory" }],
|
|
76
|
+
}];
|
|
77
|
+
await expect(performSyncIn({
|
|
78
|
+
// The mock stands in for the Daytona SDK Sandbox; only the two methods the
|
|
79
|
+
// inbound directory path calls are needed.
|
|
80
|
+
sandbox: sandbox,
|
|
81
|
+
operations,
|
|
82
|
+
remoteDir,
|
|
83
|
+
timeoutSeconds: 30,
|
|
84
|
+
})).rejects.toThrow(/syncIn extract/);
|
|
85
|
+
// The runtime uploaded exactly one reserved scratch tar under the workspace
|
|
86
|
+
// root. Its name carries the reserved `.paperclip-upload-` prefix.
|
|
87
|
+
expect(uploadedDestinations).toHaveLength(1);
|
|
88
|
+
const scratchTar = uploadedDestinations[0];
|
|
89
|
+
expect(scratchTar).toContain(".paperclip-upload-");
|
|
90
|
+
expect(scratchTar.startsWith(`${remoteDir}/`)).toBe(true);
|
|
91
|
+
// The failure path swept the scratch tar: a standalone `rm -f` of the exact
|
|
92
|
+
// scratch path ran after the failed extract. The extract command itself also
|
|
93
|
+
// contains an `rm -f`, so the cleanup is the `rm -f` command that does NOT run
|
|
94
|
+
// `tar -xf`.
|
|
95
|
+
const cleanupCommands = commands.filter((entry) => entry.command.includes(scratchTar) &&
|
|
96
|
+
entry.command.includes("rm -f") &&
|
|
97
|
+
!entry.command.includes("tar -xf"));
|
|
98
|
+
expect(cleanupCommands.length).toBeGreaterThan(0);
|
|
99
|
+
});
|
|
100
|
+
it("does not sweep scratch on the happy path (extract removes it)", async () => {
|
|
101
|
+
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-daytona-scratch-"));
|
|
102
|
+
cleanupDirs.push(rootDir);
|
|
103
|
+
const sourceDir = path.join(rootDir, "referenced-project");
|
|
104
|
+
await fs.mkdir(sourceDir, { recursive: true });
|
|
105
|
+
await fs.writeFile(path.join(sourceDir, "README.md"), "referenced project\n", "utf8");
|
|
106
|
+
const remoteDir = "/workspace";
|
|
107
|
+
const targetPath = "/workspace/.paperclip-runtime/test-adapter/project-abc";
|
|
108
|
+
const uploadedDestinations = [];
|
|
109
|
+
const commands = [];
|
|
110
|
+
// No failure: every command succeeds, so the extract's own `rm -f` clears the
|
|
111
|
+
// scratch and no extra cleanup round trip runs.
|
|
112
|
+
const sandbox = createMockSandbox({ uploadedDestinations, commands });
|
|
113
|
+
const operations = [{
|
|
114
|
+
operationId: "sync-op-1",
|
|
115
|
+
files: [{ sourcePath: sourceDir, targetPath, kind: "directory" }],
|
|
116
|
+
}];
|
|
117
|
+
await performSyncIn({
|
|
118
|
+
sandbox: sandbox,
|
|
119
|
+
operations,
|
|
120
|
+
remoteDir,
|
|
121
|
+
timeoutSeconds: 30,
|
|
122
|
+
});
|
|
123
|
+
const scratchTar = uploadedDestinations[0];
|
|
124
|
+
// The standalone cleanup command (a `rm -f` without `tar -xf`) never runs on the
|
|
125
|
+
// happy path — only the extract command, which ends with its own `rm -f`.
|
|
126
|
+
const standaloneRemoves = commands.filter((entry) => entry.command.includes(scratchTar) &&
|
|
127
|
+
entry.command.includes("rm -f") &&
|
|
128
|
+
!entry.command.includes("tar -xf"));
|
|
129
|
+
expect(standaloneRemoves).toHaveLength(0);
|
|
130
|
+
});
|
|
131
|
+
it("writes an inbound file mapping to a sandbox path outside the workspace root", async () => {
|
|
132
|
+
const hostDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-daytona-outside-root-"));
|
|
133
|
+
cleanupDirs.push(hostDir);
|
|
134
|
+
const sourcePath = path.join(hostDir, "source.txt");
|
|
135
|
+
await fs.writeFile(sourcePath, "payload");
|
|
136
|
+
// A target outside the workspace remote dir. The inbound direction writes
|
|
137
|
+
// host data into the sandbox, and the sandbox already has read/write
|
|
138
|
+
// authority over its own filesystem, so the provider no longer confines
|
|
139
|
+
// the target to the remote dir.
|
|
140
|
+
const remoteDir = "/workspace";
|
|
141
|
+
const targetPath = "/etc/paperclip-outside-root.txt";
|
|
142
|
+
const uploadedDestinations = [];
|
|
143
|
+
const commands = [];
|
|
144
|
+
const sandbox = createMockSandbox({ uploadedDestinations, commands });
|
|
145
|
+
const operations = [{
|
|
146
|
+
operationId: "sync-op-outside-root",
|
|
147
|
+
files: [{ sourcePath, targetPath, kind: "file" }],
|
|
148
|
+
}];
|
|
149
|
+
const result = await performSyncIn({ sandbox: sandbox, operations, remoteDir, timeoutSeconds: 30 });
|
|
150
|
+
expect(result.operations[0].filesTransferred).toBe(1);
|
|
151
|
+
const promoteCommand = commands.map((entry) => entry.command).find((command) => command.includes("mv -f"));
|
|
152
|
+
expect(promoteCommand).toBeDefined();
|
|
153
|
+
expect(promoteCommand).toContain(targetPath);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
// ---------------------------------------------------------------
|
|
157
|
+
// zstd-3 transport compression on the inbound file-mapping path
|
|
158
|
+
// ---------------------------------------------------------------
|
|
159
|
+
const ZSTD_MIN_SOURCE_BYTES_FOR_TEST = 8 * 1024 * 1024;
|
|
160
|
+
// A real `zstd` binary is required to run the tests that execute a real
|
|
161
|
+
// promotion script (the sandbox stand-in below shells out to THIS host). The
|
|
162
|
+
// package never depends on a `zstd` binary on the production host — only the
|
|
163
|
+
// sandbox side does, per the design — but the test double needs one to prove
|
|
164
|
+
// the decompression contract for real rather than only recording commands.
|
|
165
|
+
// Skip cleanly, like the existing `describeLinux`/`describeLive` gates in this
|
|
166
|
+
// package, when the test host has none.
|
|
167
|
+
const hasZstdBinary = spawnSync("zstd", ["--version"]).status === 0;
|
|
168
|
+
const describeWithZstd = hasZstdBinary ? describe : describe.skip;
|
|
169
|
+
function sha256OfFile(filePath) {
|
|
170
|
+
return fs.readFile(filePath).then((buf) => crypto.createHash("sha256").update(buf).digest("hex"));
|
|
171
|
+
}
|
|
172
|
+
async function writeCompressibleFile(filePath, sizeBytes) {
|
|
173
|
+
// Low-entropy repeated content compresses well past the 10% saving bar.
|
|
174
|
+
const chunk = Buffer.from("paperclip-zstd-transport-compression-fixture-".repeat(64));
|
|
175
|
+
const parts = [];
|
|
176
|
+
for (let written = 0; written < sizeBytes; written += chunk.length)
|
|
177
|
+
parts.push(chunk);
|
|
178
|
+
await fs.writeFile(filePath, Buffer.concat(parts).subarray(0, sizeBytes));
|
|
179
|
+
}
|
|
180
|
+
async function writeIncompressibleFile(filePath, sizeBytes) {
|
|
181
|
+
await fs.writeFile(filePath, crypto.randomBytes(sizeBytes));
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* A lightweight recording sandbox double for the fallback-condition tests: it
|
|
185
|
+
* never runs a real shell, only records commands and reports a canned exit
|
|
186
|
+
* code, and simulates the zstd availability probe via `probeReportsZstd`.
|
|
187
|
+
* Host-side compression (`node:zlib`) still runs for real in the code under
|
|
188
|
+
* test — only the SANDBOX side is faked here — so these tests genuinely
|
|
189
|
+
* exercise the host compression/ratio decision.
|
|
190
|
+
*/
|
|
191
|
+
function createRecordingSandbox(input) {
|
|
192
|
+
return {
|
|
193
|
+
process: {
|
|
194
|
+
executeCommand: async (command) => {
|
|
195
|
+
input.commands.push({ command });
|
|
196
|
+
if (command.includes("mkdir -p")) {
|
|
197
|
+
return { exitCode: 0, result: input.probeReportsZstd ? "PAPERCLIP_ZSTD_AVAILABLE\n" : "" };
|
|
198
|
+
}
|
|
199
|
+
return { exitCode: 0, result: "" };
|
|
200
|
+
},
|
|
201
|
+
},
|
|
202
|
+
fs: {
|
|
203
|
+
uploadFiles: async (uploads) => {
|
|
204
|
+
for (const upload of uploads) {
|
|
205
|
+
input.uploadedSources.push(upload.source);
|
|
206
|
+
input.uploadedDestinations.push(upload.destination);
|
|
207
|
+
}
|
|
208
|
+
},
|
|
209
|
+
setFilePermissions: async () => undefined,
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* A real POSIX-shell-backed sandbox double: `executeCommand` runs the exact
|
|
215
|
+
* command string on THIS host via `/bin/sh -c`, and `uploadFiles`/
|
|
216
|
+
* `setFilePermissions` apply real bytes/modes onto a real directory standing
|
|
217
|
+
* in for the sandbox root. This proves the decompression/promotion script
|
|
218
|
+
* for real, instead of only recording which commands the code would send.
|
|
219
|
+
*/
|
|
220
|
+
function createRealExecSandbox(input) {
|
|
221
|
+
const commands = [];
|
|
222
|
+
return {
|
|
223
|
+
commands,
|
|
224
|
+
sandbox: {
|
|
225
|
+
process: {
|
|
226
|
+
executeCommand: async (command) => {
|
|
227
|
+
commands.push({ command });
|
|
228
|
+
const result = spawnSync("/bin/sh", ["-c", command], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, env: input?.commandEnv });
|
|
229
|
+
return { exitCode: result.status ?? 1, result: (result.stdout ?? "") + (result.stderr ?? "") };
|
|
230
|
+
},
|
|
231
|
+
},
|
|
232
|
+
fs: {
|
|
233
|
+
uploadFiles: async (uploads) => {
|
|
234
|
+
if (input?.uploadOverride && (await input.uploadOverride(uploads)))
|
|
235
|
+
return;
|
|
236
|
+
for (const upload of uploads)
|
|
237
|
+
await fs.copyFile(upload.source, upload.destination);
|
|
238
|
+
},
|
|
239
|
+
setFilePermissions: async (target, options) => {
|
|
240
|
+
await fs.chmod(target, parseInt(options.mode, 8));
|
|
241
|
+
},
|
|
242
|
+
},
|
|
243
|
+
},
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
// Daytona uses GNU tar. macOS contributors can install gnu-tar; the usual
|
|
247
|
+
// Linux CI path runs this directly without extra dependencies.
|
|
248
|
+
const gnuTar = ["gtar", "tar"].map((candidate) => {
|
|
249
|
+
const resolved = spawnSync("/bin/sh", ["-c", 'command -v "$1"', "sh", candidate], { encoding: "utf8" }).stdout.trim();
|
|
250
|
+
return resolved && spawnSync(resolved, ["--version"], { encoding: "utf8" }).stdout?.includes("GNU tar") ? resolved : null;
|
|
251
|
+
}).find(Boolean);
|
|
252
|
+
it.skipIf(!gnuTar)("extracts interleaved read-only skill directories with GNU tar and preserves their modes", async () => {
|
|
253
|
+
const root = await fs.mkdtemp("/tmp/paperclip-daytona-readonly-");
|
|
254
|
+
const source = path.join(root, "source");
|
|
255
|
+
const remoteDir = path.join(root, "remote");
|
|
256
|
+
const target = path.join(remoteDir, "skill");
|
|
257
|
+
const bin = path.join(root, "bin");
|
|
258
|
+
await fs.mkdir(path.join(source, "references", "agents"), { recursive: true });
|
|
259
|
+
await fs.mkdir(remoteDir);
|
|
260
|
+
await fs.mkdir(bin);
|
|
261
|
+
await fs.symlink(gnuTar, path.join(bin, "tar"));
|
|
262
|
+
await fs.writeFile(path.join(source, "references", "overview.md"), "overview", { mode: 0o444 });
|
|
263
|
+
await fs.writeFile(path.join(source, "references", "agents", "qa.md"), "QA instructions", { mode: 0o444 });
|
|
264
|
+
for (const dir of ["references/agents", "references"])
|
|
265
|
+
await fs.chmod(path.join(source, dir), 0o555);
|
|
266
|
+
try {
|
|
267
|
+
const { sandbox } = createRealExecSandbox({
|
|
268
|
+
commandEnv: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
|
|
269
|
+
uploadOverride: async (uploads) => {
|
|
270
|
+
for (const upload of uploads) {
|
|
271
|
+
// BSD tar can list a child directory before its parent's files, then
|
|
272
|
+
// visit that child later. GNU tar must not finalize its 0555 mode early.
|
|
273
|
+
const archive = spawnSync(gnuTar, ["-c", "--owner=12345", "--group=12345", "--no-xattrs", "--no-recursion", "-f", upload.destination, "-C", source,
|
|
274
|
+
"references", "references/agents", "references/overview.md", "references/agents/qa.md"], { encoding: "utf8", env: { ...process.env, COPYFILE_DISABLE: "1" } });
|
|
275
|
+
expect(archive.status, archive.stderr).toBe(0);
|
|
276
|
+
}
|
|
277
|
+
return true;
|
|
278
|
+
},
|
|
279
|
+
});
|
|
280
|
+
await performSyncIn({ sandbox: sandbox, remoteDir, timeoutSeconds: 30,
|
|
281
|
+
operations: [{ operationId: "readonly-skill", files: [{ sourcePath: source, targetPath: target, kind: "directory", mode: 0o555 }] }] });
|
|
282
|
+
// A resumed sandbox already contains the previous read-only skill bundle.
|
|
283
|
+
await fs.writeFile(path.join(target, "unrelated.txt"), "keep me");
|
|
284
|
+
await performSyncIn({ sandbox: sandbox, remoteDir, timeoutSeconds: 30,
|
|
285
|
+
operations: [{ operationId: "readonly-skill-resume", files: [{ sourcePath: source, targetPath: target, kind: "directory", mode: 0o555 }] }] });
|
|
286
|
+
expect(await fs.readFile(path.join(target, "unrelated.txt"), "utf8")).toBe("keep me");
|
|
287
|
+
expect(await fs.readFile(path.join(target, "references", "agents", "qa.md"), "utf8")).toBe("QA instructions");
|
|
288
|
+
expect((await fs.stat(path.join(target, "references", "agents"))).mode & 0o777).toBe(0o555);
|
|
289
|
+
expect((await fs.stat(path.join(target, "references", "agents", "qa.md"))).mode & 0o777).toBe(0o444);
|
|
290
|
+
// Never treat a corrupted read-only bundle as a cache hit, even if its
|
|
291
|
+
// file size, permissions and timestamp still match the source archive.
|
|
292
|
+
const qa = path.join(target, "references", "agents", "qa.md");
|
|
293
|
+
const before = await fs.stat(qa);
|
|
294
|
+
await fs.chmod(qa, 0o644);
|
|
295
|
+
await fs.writeFile(qa, "XX instructions");
|
|
296
|
+
await fs.chmod(qa, 0o444);
|
|
297
|
+
await fs.utimes(qa, before.atime, before.mtime);
|
|
298
|
+
await expect(performSyncIn({ sandbox: sandbox, remoteDir, timeoutSeconds: 30,
|
|
299
|
+
operations: [{ operationId: "corrupt-skill-resume", files: [{ sourcePath: source, targetPath: target, kind: "directory", mode: 0o555 }] }] })).rejects.toThrow("syncIn extract");
|
|
300
|
+
expect(await fs.readFile(qa, "utf8")).toBe("XX instructions");
|
|
301
|
+
expect((await fs.readdir(remoteDir)).filter((name) => name.startsWith(".paperclip-upload"))).toEqual([]);
|
|
302
|
+
}
|
|
303
|
+
finally {
|
|
304
|
+
for (const base of [source, target]) {
|
|
305
|
+
for (const dir of ["references/agents", "references"])
|
|
306
|
+
await fs.chmod(path.join(base, dir), 0o700).catch(() => undefined);
|
|
307
|
+
}
|
|
308
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
309
|
+
}
|
|
310
|
+
}, 30_000);
|
|
311
|
+
it.skipIf(!gnuTar)("uploads gzip directory archives and preserves content, executable modes, and symlinks", async () => {
|
|
312
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-daytona-gzip-dir-"));
|
|
313
|
+
const source = path.join(root, "source");
|
|
314
|
+
const remoteDir = path.join(root, "remote");
|
|
315
|
+
const target = path.join(remoteDir, "target");
|
|
316
|
+
const bin = path.join(root, "bin");
|
|
317
|
+
await fs.mkdir(path.join(source, "bin"), { recursive: true });
|
|
318
|
+
await fs.mkdir(remoteDir);
|
|
319
|
+
await fs.mkdir(bin);
|
|
320
|
+
await fs.symlink(gnuTar, path.join(bin, "tar"));
|
|
321
|
+
await fs.writeFile(path.join(source, "bin", "tool.sh"), "#!/bin/sh\necho ok\n", { mode: 0o755 });
|
|
322
|
+
await fs.symlink("bin/tool.sh", path.join(source, "tool-link"));
|
|
323
|
+
try {
|
|
324
|
+
let uploadedArchiveBytes;
|
|
325
|
+
const { sandbox } = createRealExecSandbox({
|
|
326
|
+
commandEnv: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
|
|
327
|
+
uploadOverride: async (uploads) => {
|
|
328
|
+
uploadedArchiveBytes = await fs.readFile(uploads[0].source);
|
|
329
|
+
for (const upload of uploads)
|
|
330
|
+
await fs.copyFile(upload.source, upload.destination);
|
|
331
|
+
return true;
|
|
332
|
+
},
|
|
333
|
+
});
|
|
334
|
+
await performSyncIn({
|
|
335
|
+
sandbox: sandbox,
|
|
336
|
+
remoteDir,
|
|
337
|
+
timeoutSeconds: 30,
|
|
338
|
+
operations: [{
|
|
339
|
+
operationId: "gzip-directory",
|
|
340
|
+
files: [{ sourcePath: source, targetPath: target, kind: "directory" }],
|
|
341
|
+
}],
|
|
342
|
+
});
|
|
343
|
+
expect(uploadedArchiveBytes).toBeDefined();
|
|
344
|
+
expect(uploadedArchiveBytes.subarray(0, 2)).toEqual(Buffer.from([0x1f, 0x8b]));
|
|
345
|
+
expect(await fs.readFile(path.join(target, "bin", "tool.sh"), "utf8")).toBe("#!/bin/sh\necho ok\n");
|
|
346
|
+
expect((await fs.stat(path.join(target, "bin", "tool.sh"))).mode & 0o777).toBe(0o755);
|
|
347
|
+
expect(await fs.readlink(path.join(target, "tool-link"))).toBe("bin/tool.sh");
|
|
348
|
+
}
|
|
349
|
+
finally {
|
|
350
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
351
|
+
}
|
|
352
|
+
}, 30_000);
|
|
353
|
+
it.skipIf(!gnuTar)("uploads and extracts a gzip empty directory archive", async () => {
|
|
354
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-daytona-gzip-empty-"));
|
|
355
|
+
const source = path.join(root, "source");
|
|
356
|
+
const remoteDir = path.join(root, "remote");
|
|
357
|
+
const target = path.join(remoteDir, "target");
|
|
358
|
+
const bin = path.join(root, "bin");
|
|
359
|
+
await fs.mkdir(source);
|
|
360
|
+
await fs.mkdir(remoteDir);
|
|
361
|
+
await fs.mkdir(bin);
|
|
362
|
+
await fs.symlink(gnuTar, path.join(bin, "tar"));
|
|
363
|
+
try {
|
|
364
|
+
let uploadedArchiveBytes;
|
|
365
|
+
const { sandbox } = createRealExecSandbox({
|
|
366
|
+
commandEnv: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
|
|
367
|
+
uploadOverride: async (uploads) => {
|
|
368
|
+
uploadedArchiveBytes = await fs.readFile(uploads[0].source);
|
|
369
|
+
for (const upload of uploads)
|
|
370
|
+
await fs.copyFile(upload.source, upload.destination);
|
|
371
|
+
return true;
|
|
372
|
+
},
|
|
373
|
+
});
|
|
374
|
+
await performSyncIn({
|
|
375
|
+
sandbox: sandbox,
|
|
376
|
+
remoteDir,
|
|
377
|
+
timeoutSeconds: 30,
|
|
378
|
+
operations: [{
|
|
379
|
+
operationId: "gzip-empty-directory",
|
|
380
|
+
files: [{ sourcePath: source, targetPath: target, kind: "directory" }],
|
|
381
|
+
}],
|
|
382
|
+
});
|
|
383
|
+
expect(uploadedArchiveBytes).toBeDefined();
|
|
384
|
+
expect(uploadedArchiveBytes.subarray(0, 2)).toEqual(Buffer.from([0x1f, 0x8b]));
|
|
385
|
+
expect(await fs.readdir(target)).toEqual([]);
|
|
386
|
+
}
|
|
387
|
+
finally {
|
|
388
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
389
|
+
}
|
|
390
|
+
}, 30_000);
|
|
391
|
+
describe("daytona file-sync inbound zstd transport compression", () => {
|
|
392
|
+
const cleanupDirs = [];
|
|
393
|
+
afterEach(async () => {
|
|
394
|
+
while (cleanupDirs.length > 0) {
|
|
395
|
+
const dir = cleanupDirs.pop();
|
|
396
|
+
if (!dir)
|
|
397
|
+
continue;
|
|
398
|
+
await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined);
|
|
399
|
+
}
|
|
400
|
+
});
|
|
401
|
+
const mkTempDir = async (prefix) => {
|
|
402
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
|
403
|
+
cleanupDirs.push(dir);
|
|
404
|
+
return dir;
|
|
405
|
+
};
|
|
406
|
+
describeWithZstd("compressed path (real promotion script, real zstd)", () => {
|
|
407
|
+
it("compresses on the host and decompresses in-sandbox to a byte-identical file", async () => {
|
|
408
|
+
const remoteDir = await mkTempDir("paperclip-daytona-zstd-remote-");
|
|
409
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
410
|
+
const sourcePath = path.join(hostDir, "workspace-upload.tar");
|
|
411
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
412
|
+
const targetPath = path.posix.join(remoteDir, "workspace-upload.tar");
|
|
413
|
+
const { sandbox, commands } = createRealExecSandbox();
|
|
414
|
+
const operations = [{
|
|
415
|
+
operationId: "sync-op-1",
|
|
416
|
+
files: [{ sourcePath, targetPath, kind: "file" }],
|
|
417
|
+
}];
|
|
418
|
+
const result = await performSyncIn({ sandbox: sandbox, operations, remoteDir, timeoutSeconds: 30 });
|
|
419
|
+
expect(result.operations[0].filesTransferred).toBe(1);
|
|
420
|
+
expect(result.operations[0].bytesTransferred).toBe(ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
421
|
+
// The promote script actually ran a real `zstd -d -o` — proves decompression
|
|
422
|
+
// happened, not just that the code called `uploadFiles`.
|
|
423
|
+
expect(commands.some((entry) => entry.command.includes("zstd -d -o"))).toBe(true);
|
|
424
|
+
expect(await sha256OfFile(targetPath)).toBe(await sha256OfFile(sourcePath));
|
|
425
|
+
// Cleanup on success: no reserved scratch (raw or `.zst`) remains.
|
|
426
|
+
const remaining = await fs.readdir(remoteDir);
|
|
427
|
+
expect(remaining.filter((name) => name.includes(".paperclip-upload"))).toHaveLength(0);
|
|
428
|
+
});
|
|
429
|
+
it("removes the private compressed host temp directory after a successful sync", async () => {
|
|
430
|
+
const remoteDir = await mkTempDir("paperclip-daytona-zstd-remote-");
|
|
431
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
432
|
+
const sourcePath = path.join(hostDir, "workspace-upload.tar");
|
|
433
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
434
|
+
const targetPath = path.posix.join(remoteDir, "target.bin");
|
|
435
|
+
let capturedHostTempDir = "";
|
|
436
|
+
const { sandbox } = createRealExecSandbox({
|
|
437
|
+
uploadOverride: async (uploads) => {
|
|
438
|
+
const zstdUpload = uploads.find((upload) => upload.destination.endsWith(".zst"));
|
|
439
|
+
if (zstdUpload)
|
|
440
|
+
capturedHostTempDir = path.dirname(zstdUpload.source);
|
|
441
|
+
for (const upload of uploads)
|
|
442
|
+
await fs.copyFile(upload.source, upload.destination);
|
|
443
|
+
return true;
|
|
444
|
+
},
|
|
445
|
+
});
|
|
446
|
+
const operations = [{
|
|
447
|
+
operationId: "sync-op-1",
|
|
448
|
+
files: [{ sourcePath, targetPath, kind: "file" }],
|
|
449
|
+
}];
|
|
450
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir, timeoutSeconds: 30 });
|
|
451
|
+
expect(capturedHostTempDir).toContain("paperclip-daytona-zstd-");
|
|
452
|
+
// The private host temp directory (not just the file inside it) is gone
|
|
453
|
+
// after a successful sync.
|
|
454
|
+
await expect(fs.stat(capturedHostTempDir)).rejects.toThrow();
|
|
455
|
+
});
|
|
456
|
+
it("reports the sync as successful when the post-promotion `.zst` cleanup fails, and warns with a leftover count but no path", async () => {
|
|
457
|
+
const remoteDir = await mkTempDir("paperclip-daytona-zstd-remote-");
|
|
458
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
459
|
+
const sourcePath = path.join(hostDir, "workspace-upload.tar");
|
|
460
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
461
|
+
const targetPath = path.posix.join(remoteDir, "target.bin");
|
|
462
|
+
// Put a stand-in `rm` first on PATH. It always exits 1. So BOTH the
|
|
463
|
+
// promote script's OWN `.zst` cleanup and the later bounded sweep's
|
|
464
|
+
// separate `rm -f` fail, the same way a persistent cleanup error would.
|
|
465
|
+
// The other commands (`mv`, `zstd`, `chmod`) still resolve to the real
|
|
466
|
+
// binaries later on PATH.
|
|
467
|
+
const fakeBinDir = await mkTempDir("paperclip-daytona-zstd-fakebin-");
|
|
468
|
+
const fakeRmPath = path.join(fakeBinDir, "rm");
|
|
469
|
+
await fs.writeFile(fakeRmPath, "#!/bin/sh\nexit 1\n");
|
|
470
|
+
await fs.chmod(fakeRmPath, 0o755);
|
|
471
|
+
const originalPath = process.env.PATH;
|
|
472
|
+
process.env.PATH = `${fakeBinDir}${path.delimiter}${originalPath}`;
|
|
473
|
+
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
|
474
|
+
try {
|
|
475
|
+
const { sandbox } = createRealExecSandbox();
|
|
476
|
+
const operations = [{
|
|
477
|
+
operationId: "sync-op-1",
|
|
478
|
+
files: [{ sourcePath, targetPath, kind: "file" }],
|
|
479
|
+
}];
|
|
480
|
+
// Every target file is already installed by the time the `.zst`
|
|
481
|
+
// cleanup runs, so a failing cleanup must never turn into a sync
|
|
482
|
+
// failure.
|
|
483
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir, timeoutSeconds: 30 });
|
|
484
|
+
expect(await sha256OfFile(targetPath)).toBe(await sha256OfFile(sourcePath));
|
|
485
|
+
// The forced `rm` failure left the `.zst` scratch behind — proof the
|
|
486
|
+
// fake `rm` actually ran and failed, not that cleanup was skipped.
|
|
487
|
+
const remaining = await fs.readdir(remoteDir);
|
|
488
|
+
const zstdName = remaining.find((name) => name.endsWith(".zst"));
|
|
489
|
+
expect(zstdName).toBeDefined();
|
|
490
|
+
// A leftover that survives both the inline cleanup and the bounded
|
|
491
|
+
// sweep is observable: exactly one warning, carrying a count, never
|
|
492
|
+
// the scratch pathname itself.
|
|
493
|
+
expect(warnSpy).toHaveBeenCalledTimes(1);
|
|
494
|
+
const warning = warnSpy.mock.calls[0]?.[0];
|
|
495
|
+
expect(warning).toContain("1 post-promotion scratch file");
|
|
496
|
+
expect(warning).not.toContain(zstdName);
|
|
497
|
+
}
|
|
498
|
+
finally {
|
|
499
|
+
process.env.PATH = originalPath;
|
|
500
|
+
warnSpy.mockRestore();
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
it("recovers a transient post-promotion `.zst` cleanup failure with the bounded sweep, without warning", async () => {
|
|
504
|
+
const remoteDir = await mkTempDir("paperclip-daytona-zstd-remote-");
|
|
505
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
506
|
+
const sourcePath = path.join(hostDir, "workspace-upload.tar");
|
|
507
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
508
|
+
const targetPath = path.posix.join(remoteDir, "target.bin");
|
|
509
|
+
// A stand-in `rm` that fails only its first call — the promote script's
|
|
510
|
+
// own inline `.zst` cleanup. It defers to the real `rm` for every later
|
|
511
|
+
// call. This simulates a transient cleanup failure: the bounded sweep's
|
|
512
|
+
// own, separate `rm -f` is the second call, and it succeeds.
|
|
513
|
+
const fakeBinDir = await mkTempDir("paperclip-daytona-zstd-fakebin-");
|
|
514
|
+
const counterFile = path.join(fakeBinDir, "rm-call-count");
|
|
515
|
+
const fakeRmPath = path.join(fakeBinDir, "rm");
|
|
516
|
+
await fs.writeFile(fakeRmPath, [
|
|
517
|
+
"#!/bin/sh",
|
|
518
|
+
"n=0",
|
|
519
|
+
`[ -f "${counterFile}" ] && n=$(cat "${counterFile}")`,
|
|
520
|
+
"n=$((n + 1))",
|
|
521
|
+
`printf '%s' "$n" > "${counterFile}"`,
|
|
522
|
+
'[ "$n" -eq 1 ] && exit 1',
|
|
523
|
+
'exec /bin/rm "$@"',
|
|
524
|
+
"",
|
|
525
|
+
].join("\n"));
|
|
526
|
+
await fs.chmod(fakeRmPath, 0o755);
|
|
527
|
+
const originalPath = process.env.PATH;
|
|
528
|
+
process.env.PATH = `${fakeBinDir}${path.delimiter}${originalPath}`;
|
|
529
|
+
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
|
530
|
+
try {
|
|
531
|
+
const { sandbox } = createRealExecSandbox();
|
|
532
|
+
const operations = [{
|
|
533
|
+
operationId: "sync-op-1",
|
|
534
|
+
files: [{ sourcePath, targetPath, kind: "file" }],
|
|
535
|
+
}];
|
|
536
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir, timeoutSeconds: 30 });
|
|
537
|
+
expect(await sha256OfFile(targetPath)).toBe(await sha256OfFile(sourcePath));
|
|
538
|
+
expect(await fs.readFile(counterFile, "utf8")).toBe("2"); // proves the sweep actually ran a second `rm`
|
|
539
|
+
// The sweep's separate, later `rm -f` succeeded where the inline
|
|
540
|
+
// cleanup failed — no `.zst` scratch remains, so there is nothing to
|
|
541
|
+
// warn about.
|
|
542
|
+
const remaining = await fs.readdir(remoteDir);
|
|
543
|
+
expect(remaining.some((name) => name.endsWith(".zst"))).toBe(false);
|
|
544
|
+
expect(warnSpy).not.toHaveBeenCalled();
|
|
545
|
+
}
|
|
546
|
+
finally {
|
|
547
|
+
process.env.PATH = originalPath;
|
|
548
|
+
warnSpy.mockRestore();
|
|
549
|
+
}
|
|
550
|
+
});
|
|
551
|
+
it("never promotes a partial file when decompression fails, and sweeps all reserved scratch", async () => {
|
|
552
|
+
const remoteDir = await mkTempDir("paperclip-daytona-zstd-remote-");
|
|
553
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
554
|
+
const sourcePath = path.join(hostDir, "workspace-upload.tar");
|
|
555
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
556
|
+
const targetPath = path.posix.join(remoteDir, "target.bin");
|
|
557
|
+
const { sandbox } = createRealExecSandbox({
|
|
558
|
+
// Simulate the `.zst` upload landing corrupted: the in-sandbox
|
|
559
|
+
// `zstd -d -c` step will fail for real on this invalid input.
|
|
560
|
+
uploadOverride: async (uploads) => {
|
|
561
|
+
for (const upload of uploads) {
|
|
562
|
+
if (upload.destination.endsWith(".zst")) {
|
|
563
|
+
await fs.writeFile(upload.destination, Buffer.from("not a valid zstd frame at all"));
|
|
564
|
+
}
|
|
565
|
+
else {
|
|
566
|
+
await fs.copyFile(upload.source, upload.destination);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
return true;
|
|
570
|
+
},
|
|
571
|
+
});
|
|
572
|
+
const operations = [{
|
|
573
|
+
operationId: "sync-op-1",
|
|
574
|
+
files: [{ sourcePath, targetPath, kind: "file" }],
|
|
575
|
+
}];
|
|
576
|
+
await expect(performSyncIn({ sandbox: sandbox, operations, remoteDir, timeoutSeconds: 30 })).rejects.toThrow(/syncIn rename/);
|
|
577
|
+
await expect(fs.stat(targetPath)).rejects.toThrow(); // never promoted
|
|
578
|
+
const remaining = await fs.readdir(remoteDir);
|
|
579
|
+
expect(remaining.filter((name) => name.includes(".paperclip-upload"))).toHaveLength(0); // scratch swept
|
|
580
|
+
});
|
|
581
|
+
it("applies mapping.mode via chmod before promotion, when set", async () => {
|
|
582
|
+
const remoteDir = await mkTempDir("paperclip-daytona-zstd-remote-");
|
|
583
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
584
|
+
const sourceNoMode = path.join(hostDir, "no-mode.tar");
|
|
585
|
+
const sourceWithMode = path.join(hostDir, "with-mode.tar");
|
|
586
|
+
await writeCompressibleFile(sourceNoMode, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
587
|
+
await writeCompressibleFile(sourceWithMode, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 2048);
|
|
588
|
+
const targetNoMode = path.posix.join(remoteDir, "no-mode.bin");
|
|
589
|
+
const targetWithMode = path.posix.join(remoteDir, "with-mode.bin");
|
|
590
|
+
// `zstd -d -o` copies the mode of its INPUT (the uploaded `.zst`
|
|
591
|
+
// scratch) onto its output. Widen every `.zst` scratch to 0644 here,
|
|
592
|
+
// standing in for a Daytona upload that does not preserve a host-side
|
|
593
|
+
// 0600 origin. A pass on the no-mode assertion below then proves the
|
|
594
|
+
// promote script's OWN `chmod` forces 0600 — not that the scratch
|
|
595
|
+
// happened to arrive owner-only already.
|
|
596
|
+
const { sandbox } = createRealExecSandbox({
|
|
597
|
+
uploadOverride: async (uploads) => {
|
|
598
|
+
for (const upload of uploads) {
|
|
599
|
+
await fs.copyFile(upload.source, upload.destination);
|
|
600
|
+
if (upload.destination.endsWith(".zst"))
|
|
601
|
+
await fs.chmod(upload.destination, 0o644);
|
|
602
|
+
}
|
|
603
|
+
return true;
|
|
604
|
+
},
|
|
605
|
+
});
|
|
606
|
+
const operations = [{
|
|
607
|
+
operationId: "sync-op-1",
|
|
608
|
+
files: [
|
|
609
|
+
{ sourcePath: sourceNoMode, targetPath: targetNoMode, kind: "file" },
|
|
610
|
+
{ sourcePath: sourceWithMode, targetPath: targetWithMode, kind: "file", mode: 0o640 },
|
|
611
|
+
],
|
|
612
|
+
}];
|
|
613
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir, timeoutSeconds: 30 });
|
|
614
|
+
// A mapping with no `mode` lands owner-only (0600): the promote script
|
|
615
|
+
// always runs `chmod` on the decompressed file right after
|
|
616
|
+
// `zstd -d -o`, and uses 0600 when the mapping sets no `mode` — even
|
|
617
|
+
// though its `.zst` scratch arrived at 0644 above. A mapping with an
|
|
618
|
+
// explicit `mode` gets that mode instead.
|
|
619
|
+
expect((await fs.stat(targetNoMode)).mode & 0o777).toBe(0o600);
|
|
620
|
+
expect((await fs.stat(targetWithMode)).mode & 0o777).toBe(0o640);
|
|
621
|
+
});
|
|
622
|
+
it("runs two concurrent compressed sync operations without cross-talk", async () => {
|
|
623
|
+
const remoteDirA = await mkTempDir("paperclip-daytona-zstd-remote-a-");
|
|
624
|
+
const remoteDirB = await mkTempDir("paperclip-daytona-zstd-remote-b-");
|
|
625
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
626
|
+
const sourceA = path.join(hostDir, "a.tar");
|
|
627
|
+
const sourceB = path.join(hostDir, "b.tar");
|
|
628
|
+
await writeCompressibleFile(sourceA, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 2048);
|
|
629
|
+
await writeCompressibleFile(sourceB, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 4096);
|
|
630
|
+
await fs.appendFile(sourceA, "AAAA-marker-a");
|
|
631
|
+
await fs.appendFile(sourceB, "BBBB-marker-b");
|
|
632
|
+
const targetA = path.posix.join(remoteDirA, "target.bin");
|
|
633
|
+
const targetB = path.posix.join(remoteDirB, "target.bin");
|
|
634
|
+
const { sandbox: sandboxA } = createRealExecSandbox();
|
|
635
|
+
const { sandbox: sandboxB } = createRealExecSandbox();
|
|
636
|
+
await Promise.all([
|
|
637
|
+
performSyncIn({
|
|
638
|
+
sandbox: sandboxA,
|
|
639
|
+
operations: [{ operationId: "a", files: [{ sourcePath: sourceA, targetPath: targetA, kind: "file" }] }],
|
|
640
|
+
remoteDir: remoteDirA,
|
|
641
|
+
timeoutSeconds: 30,
|
|
642
|
+
}),
|
|
643
|
+
performSyncIn({
|
|
644
|
+
sandbox: sandboxB,
|
|
645
|
+
operations: [{ operationId: "b", files: [{ sourcePath: sourceB, targetPath: targetB, kind: "file" }] }],
|
|
646
|
+
remoteDir: remoteDirB,
|
|
647
|
+
timeoutSeconds: 30,
|
|
648
|
+
}),
|
|
649
|
+
]);
|
|
650
|
+
expect(await sha256OfFile(targetA)).toBe(await sha256OfFile(sourceA));
|
|
651
|
+
expect(await sha256OfFile(targetB)).toBe(await sha256OfFile(sourceB));
|
|
652
|
+
});
|
|
653
|
+
it("emits exactly the five compression/decompress span attributes, with closed-set/numeric values", async () => {
|
|
654
|
+
const spans = [];
|
|
655
|
+
const tracer = {
|
|
656
|
+
startSpan(_name, options) {
|
|
657
|
+
const span = { attributes: { ...(options?.attributes ?? {}) } };
|
|
658
|
+
spans.push(span);
|
|
659
|
+
return {
|
|
660
|
+
setAttribute(key, value) {
|
|
661
|
+
span.attributes[key] = value;
|
|
662
|
+
},
|
|
663
|
+
setStatus() { },
|
|
664
|
+
end() { },
|
|
665
|
+
};
|
|
666
|
+
},
|
|
667
|
+
};
|
|
668
|
+
const restore = __setDaytonaPluginContextForTest({ tracer });
|
|
669
|
+
let targetPath = "";
|
|
670
|
+
let remoteDir = "";
|
|
671
|
+
try {
|
|
672
|
+
remoteDir = await mkTempDir("paperclip-daytona-zstd-remote-");
|
|
673
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
674
|
+
const sourcePath = path.join(hostDir, "workspace-upload.tar");
|
|
675
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
676
|
+
targetPath = path.posix.join(remoteDir, "target.bin");
|
|
677
|
+
const { sandbox } = createRealExecSandbox();
|
|
678
|
+
await performSyncIn({
|
|
679
|
+
sandbox: sandbox,
|
|
680
|
+
operations: [{ operationId: "op-1", files: [{ sourcePath, targetPath, kind: "file" }] }],
|
|
681
|
+
remoteDir,
|
|
682
|
+
timeoutSeconds: 30,
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
finally {
|
|
686
|
+
restore();
|
|
687
|
+
}
|
|
688
|
+
const allAttrs = {};
|
|
689
|
+
for (const span of spans)
|
|
690
|
+
Object.assign(allAttrs, span.attributes);
|
|
691
|
+
const compressionKeys = Object.keys(allAttrs).filter((key) => key.includes(".transfer.compression.") || key.includes(".transfer.decompress."));
|
|
692
|
+
expect(new Set(compressionKeys)).toEqual(new Set([
|
|
693
|
+
"paperclip.sandbox.startup.transfer.compression.codec",
|
|
694
|
+
"paperclip.sandbox.startup.transfer.compression.wall_ms",
|
|
695
|
+
"paperclip.sandbox.startup.transfer.compression.bytes_in",
|
|
696
|
+
"paperclip.sandbox.startup.transfer.compression.bytes_out",
|
|
697
|
+
"paperclip.sandbox.startup.transfer.decompress.wall_ms",
|
|
698
|
+
]));
|
|
699
|
+
expect(allAttrs["paperclip.sandbox.startup.transfer.compression.codec"]).toBe("zstd");
|
|
700
|
+
for (const key of [
|
|
701
|
+
"paperclip.sandbox.startup.transfer.compression.wall_ms",
|
|
702
|
+
"paperclip.sandbox.startup.transfer.compression.bytes_in",
|
|
703
|
+
"paperclip.sandbox.startup.transfer.compression.bytes_out",
|
|
704
|
+
"paperclip.sandbox.startup.transfer.decompress.wall_ms",
|
|
705
|
+
]) {
|
|
706
|
+
expect(Number.isFinite(allAttrs[key])).toBe(true);
|
|
707
|
+
}
|
|
708
|
+
});
|
|
709
|
+
});
|
|
710
|
+
describe("raw-path fallback conditions (no real sandbox exec needed)", () => {
|
|
711
|
+
it("falls back to the raw path when the sandbox reports no zstd binary", async () => {
|
|
712
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
713
|
+
const sourcePath = path.join(hostDir, "big.tar");
|
|
714
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
715
|
+
const uploadedSources = [];
|
|
716
|
+
const uploadedDestinations = [];
|
|
717
|
+
const sandbox = createRecordingSandbox({
|
|
718
|
+
probeReportsZstd: false,
|
|
719
|
+
uploadedSources,
|
|
720
|
+
uploadedDestinations,
|
|
721
|
+
commands: [],
|
|
722
|
+
});
|
|
723
|
+
const operations = [{
|
|
724
|
+
operationId: "op-1",
|
|
725
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
726
|
+
}];
|
|
727
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 });
|
|
728
|
+
expect(uploadedSources).toEqual([sourcePath]); // raw source uploaded directly
|
|
729
|
+
expect(uploadedDestinations.some((dest) => dest.endsWith(".zst"))).toBe(false);
|
|
730
|
+
});
|
|
731
|
+
it("falls back to the raw path when the source is below ZSTD_MIN_SOURCE_BYTES", async () => {
|
|
732
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
733
|
+
const sourcePath = path.join(hostDir, "small.tar");
|
|
734
|
+
await fs.writeFile(sourcePath, "well below the 8 MiB compression floor\n");
|
|
735
|
+
const uploadedSources = [];
|
|
736
|
+
const uploadedDestinations = [];
|
|
737
|
+
const sandbox = createRecordingSandbox({
|
|
738
|
+
probeReportsZstd: true,
|
|
739
|
+
uploadedSources,
|
|
740
|
+
uploadedDestinations,
|
|
741
|
+
commands: [],
|
|
742
|
+
});
|
|
743
|
+
const operations = [{
|
|
744
|
+
operationId: "op-1",
|
|
745
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
746
|
+
}];
|
|
747
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 });
|
|
748
|
+
expect(uploadedSources).toEqual([sourcePath]);
|
|
749
|
+
expect(uploadedDestinations.some((dest) => dest.endsWith(".zst"))).toBe(false);
|
|
750
|
+
});
|
|
751
|
+
it("falls back to the raw path when the saving ratio is below ZSTD_MIN_SAVING_RATIO", async () => {
|
|
752
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
753
|
+
const sourcePath = path.join(hostDir, "incompressible.tar");
|
|
754
|
+
await writeIncompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
755
|
+
const uploadedSources = [];
|
|
756
|
+
const uploadedDestinations = [];
|
|
757
|
+
const sandbox = createRecordingSandbox({
|
|
758
|
+
probeReportsZstd: true,
|
|
759
|
+
uploadedSources,
|
|
760
|
+
uploadedDestinations,
|
|
761
|
+
commands: [],
|
|
762
|
+
});
|
|
763
|
+
const operations = [{
|
|
764
|
+
operationId: "op-1",
|
|
765
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
766
|
+
}];
|
|
767
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 });
|
|
768
|
+
expect(uploadedSources).toEqual([sourcePath]); // host discarded the compressed candidate
|
|
769
|
+
expect(uploadedDestinations.some((dest) => dest.endsWith(".zst"))).toBe(false);
|
|
770
|
+
});
|
|
771
|
+
it("falls back to the raw path when zlib.createZstdCompress is not a function (feature-detect)", async () => {
|
|
772
|
+
// `createZstdCompress` is a non-writable (but configurable) property on
|
|
773
|
+
// `node:zlib` — simulate an older Node runtime without zstd support by
|
|
774
|
+
// redefining it, not assigning it.
|
|
775
|
+
const original = zlib.createZstdCompress;
|
|
776
|
+
Object.defineProperty(zlib, "createZstdCompress", { value: undefined, configurable: true, writable: true });
|
|
777
|
+
try {
|
|
778
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
779
|
+
const sourcePath = path.join(hostDir, "big.tar");
|
|
780
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
781
|
+
const uploadedSources = [];
|
|
782
|
+
const uploadedDestinations = [];
|
|
783
|
+
const sandbox = createRecordingSandbox({
|
|
784
|
+
probeReportsZstd: true,
|
|
785
|
+
uploadedSources,
|
|
786
|
+
uploadedDestinations,
|
|
787
|
+
commands: [],
|
|
788
|
+
});
|
|
789
|
+
const operations = [{
|
|
790
|
+
operationId: "op-1",
|
|
791
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
792
|
+
}];
|
|
793
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 });
|
|
794
|
+
expect(uploadedSources).toEqual([sourcePath]);
|
|
795
|
+
expect(uploadedDestinations.some((dest) => dest.endsWith(".zst"))).toBe(false);
|
|
796
|
+
}
|
|
797
|
+
finally {
|
|
798
|
+
Object.defineProperty(zlib, "createZstdCompress", { value: original, configurable: true, writable: true });
|
|
799
|
+
}
|
|
800
|
+
});
|
|
801
|
+
it("falls back to the raw path when host compression throws, and leaves no host temp file", async () => {
|
|
802
|
+
const original = zlib.createZstdCompress;
|
|
803
|
+
const throwingCompressor = () => new Transform({
|
|
804
|
+
transform(_chunk, _encoding, callback) {
|
|
805
|
+
callback(new Error("simulated host compression failure"));
|
|
806
|
+
},
|
|
807
|
+
});
|
|
808
|
+
Object.defineProperty(zlib, "createZstdCompress", {
|
|
809
|
+
value: throwingCompressor,
|
|
810
|
+
configurable: true,
|
|
811
|
+
writable: true,
|
|
812
|
+
});
|
|
813
|
+
try {
|
|
814
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
815
|
+
const sourcePath = path.join(hostDir, "big.tar");
|
|
816
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
817
|
+
const uploadedSources = [];
|
|
818
|
+
const uploadedDestinations = [];
|
|
819
|
+
const sandbox = createRecordingSandbox({
|
|
820
|
+
probeReportsZstd: true,
|
|
821
|
+
uploadedSources,
|
|
822
|
+
uploadedDestinations,
|
|
823
|
+
commands: [],
|
|
824
|
+
});
|
|
825
|
+
const operations = [{
|
|
826
|
+
operationId: "op-1",
|
|
827
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
828
|
+
}];
|
|
829
|
+
const before = (await fs.readdir(os.tmpdir())).filter((name) => name.startsWith("paperclip-daytona-zstd-"));
|
|
830
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 });
|
|
831
|
+
expect(uploadedSources).toEqual([sourcePath]);
|
|
832
|
+
expect(uploadedDestinations.some((dest) => dest.endsWith(".zst"))).toBe(false);
|
|
833
|
+
const after = (await fs.readdir(os.tmpdir())).filter((name) => name.startsWith("paperclip-daytona-zstd-"));
|
|
834
|
+
expect(after).toEqual(before); // no leftover host temp file
|
|
835
|
+
}
|
|
836
|
+
finally {
|
|
837
|
+
Object.defineProperty(zlib, "createZstdCompress", { value: original, configurable: true, writable: true });
|
|
838
|
+
}
|
|
839
|
+
});
|
|
840
|
+
it("removes the private host temp directory when the post-compression size stat throws", async () => {
|
|
841
|
+
const realStat = fs.stat.bind(fs);
|
|
842
|
+
const statSpy = vi.spyOn(fs, "stat").mockImplementation(async (targetPath, ...rest) => {
|
|
843
|
+
if (typeof targetPath === "string" && path.basename(targetPath) === "artifact.zst") {
|
|
844
|
+
throw new Error("simulated post-compression stat failure");
|
|
845
|
+
}
|
|
846
|
+
return realStat(targetPath, ...rest);
|
|
847
|
+
});
|
|
848
|
+
try {
|
|
849
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
850
|
+
const sourcePath = path.join(hostDir, "big.tar");
|
|
851
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
852
|
+
const uploadedSources = [];
|
|
853
|
+
const uploadedDestinations = [];
|
|
854
|
+
const sandbox = createRecordingSandbox({
|
|
855
|
+
probeReportsZstd: true,
|
|
856
|
+
uploadedSources,
|
|
857
|
+
uploadedDestinations,
|
|
858
|
+
commands: [],
|
|
859
|
+
});
|
|
860
|
+
const operations = [{
|
|
861
|
+
operationId: "op-1",
|
|
862
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
863
|
+
}];
|
|
864
|
+
const before = (await fs.readdir(os.tmpdir())).filter((name) => name.startsWith("paperclip-daytona-zstd-"));
|
|
865
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 });
|
|
866
|
+
// The post-compression stat failed, so the candidate falls back to the raw path.
|
|
867
|
+
expect(uploadedSources).toEqual([sourcePath]);
|
|
868
|
+
expect(uploadedDestinations.some((dest) => dest.endsWith(".zst"))).toBe(false);
|
|
869
|
+
const after = (await fs.readdir(os.tmpdir())).filter((name) => name.startsWith("paperclip-daytona-zstd-"));
|
|
870
|
+
expect(after).toEqual(before); // the stat failure did not leak the private host temp directory
|
|
871
|
+
}
|
|
872
|
+
finally {
|
|
873
|
+
statSpy.mockRestore();
|
|
874
|
+
}
|
|
875
|
+
});
|
|
876
|
+
it("removes the host compressed temp file and sweeps sandbox scratch when the upload itself is rejected (cancellation)", async () => {
|
|
877
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
878
|
+
const sourcePath = path.join(hostDir, "big.tar");
|
|
879
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
880
|
+
const commands = [];
|
|
881
|
+
const sandbox = {
|
|
882
|
+
process: {
|
|
883
|
+
executeCommand: async (command) => {
|
|
884
|
+
commands.push({ command });
|
|
885
|
+
return { exitCode: 0, result: command.includes("mkdir -p") ? "PAPERCLIP_ZSTD_AVAILABLE\n" : "" };
|
|
886
|
+
},
|
|
887
|
+
},
|
|
888
|
+
fs: {
|
|
889
|
+
uploadFiles: async () => {
|
|
890
|
+
throw new Error("simulated cancellation");
|
|
891
|
+
},
|
|
892
|
+
setFilePermissions: async () => undefined,
|
|
893
|
+
},
|
|
894
|
+
};
|
|
895
|
+
const before = (await fs.readdir(os.tmpdir())).filter((name) => name.startsWith("paperclip-daytona-zstd-"));
|
|
896
|
+
const operations = [{
|
|
897
|
+
operationId: "op-1",
|
|
898
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
899
|
+
}];
|
|
900
|
+
await expect(performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 })).rejects.toThrow(/simulated cancellation/);
|
|
901
|
+
const after = (await fs.readdir(os.tmpdir())).filter((name) => name.startsWith("paperclip-daytona-zstd-"));
|
|
902
|
+
expect(after).toEqual(before); // no leftover host temp file
|
|
903
|
+
const rmCommands = commands.filter((entry) => entry.command.includes("rm -f"));
|
|
904
|
+
expect(rmCommands.length).toBeGreaterThan(0); // both reserved scratch names swept
|
|
905
|
+
});
|
|
906
|
+
it("stages the compressed artifact in a private 0700 directory with a 0600 file, and removes the directory when the upload fails", async () => {
|
|
907
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
908
|
+
const sourcePath = path.join(hostDir, "big.tar");
|
|
909
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
910
|
+
let capturedDir = "";
|
|
911
|
+
let capturedDirMode = -1;
|
|
912
|
+
let capturedFileMode = -1;
|
|
913
|
+
const sandbox = {
|
|
914
|
+
process: {
|
|
915
|
+
executeCommand: async (command) => ({
|
|
916
|
+
exitCode: 0,
|
|
917
|
+
result: command.includes("mkdir -p") ? "PAPERCLIP_ZSTD_AVAILABLE\n" : "",
|
|
918
|
+
}),
|
|
919
|
+
},
|
|
920
|
+
fs: {
|
|
921
|
+
uploadFiles: async (uploads) => {
|
|
922
|
+
const zstdUpload = uploads.find((upload) => upload.destination.endsWith(".zst"));
|
|
923
|
+
if (!zstdUpload)
|
|
924
|
+
throw new Error("test setup: expected a compressed upload");
|
|
925
|
+
// Read the modes BEFORE throwing: the directory and its file still
|
|
926
|
+
// exist at this point, on the way to the (simulated) failed upload.
|
|
927
|
+
capturedDir = path.dirname(zstdUpload.source);
|
|
928
|
+
capturedDirMode = (await fs.stat(capturedDir)).mode & 0o777;
|
|
929
|
+
capturedFileMode = (await fs.stat(zstdUpload.source)).mode & 0o777;
|
|
930
|
+
throw new Error("simulated upload failure");
|
|
931
|
+
},
|
|
932
|
+
setFilePermissions: async () => undefined,
|
|
933
|
+
},
|
|
934
|
+
};
|
|
935
|
+
const operations = [{
|
|
936
|
+
operationId: "op-1",
|
|
937
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
938
|
+
}];
|
|
939
|
+
await expect(performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 30 })).rejects.toThrow(/simulated upload failure/);
|
|
940
|
+
expect(capturedDirMode).toBe(0o700);
|
|
941
|
+
expect(capturedFileMode).toBe(0o600);
|
|
942
|
+
// The private directory (not only the file) is removed after the upload fails.
|
|
943
|
+
await expect(fs.stat(capturedDir)).rejects.toThrow();
|
|
944
|
+
});
|
|
945
|
+
it("passes the caller's timeoutSeconds unchanged through every round trip on the compressed path", async () => {
|
|
946
|
+
const hostDir = await mkTempDir("paperclip-daytona-zstd-host-");
|
|
947
|
+
const sourcePath = path.join(hostDir, "big.tar");
|
|
948
|
+
await writeCompressibleFile(sourcePath, ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024);
|
|
949
|
+
const seenTimeouts = [];
|
|
950
|
+
const sandbox = {
|
|
951
|
+
process: {
|
|
952
|
+
executeCommand: async (command, _cwd, _env, timeoutSeconds) => {
|
|
953
|
+
seenTimeouts.push(timeoutSeconds);
|
|
954
|
+
return { exitCode: 0, result: command.includes("mkdir -p") ? "PAPERCLIP_ZSTD_AVAILABLE\n" : "" };
|
|
955
|
+
},
|
|
956
|
+
},
|
|
957
|
+
fs: {
|
|
958
|
+
uploadFiles: async (_uploads, timeoutSeconds) => {
|
|
959
|
+
seenTimeouts.push(timeoutSeconds);
|
|
960
|
+
},
|
|
961
|
+
setFilePermissions: async () => undefined,
|
|
962
|
+
},
|
|
963
|
+
};
|
|
964
|
+
const operations = [{
|
|
965
|
+
operationId: "op-1",
|
|
966
|
+
files: [{ sourcePath, targetPath: "/workspace/target.bin", kind: "file" }],
|
|
967
|
+
}];
|
|
968
|
+
await performSyncIn({ sandbox: sandbox, operations, remoteDir: "/workspace", timeoutSeconds: 123 });
|
|
969
|
+
expect(seenTimeouts.length).toBeGreaterThanOrEqual(3); // mkdir and probe, transfer, promote
|
|
970
|
+
expect(seenTimeouts.every((timeout) => timeout === 123)).toBe(true);
|
|
971
|
+
});
|
|
972
|
+
});
|
|
973
|
+
});
|
|
974
|
+
//# sourceMappingURL=file-sync.test.js.map
|