@fml-inc/fml 0.7.0 → 0.8.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +2 -4
- package/README.md +323 -79
- package/agent-assets/fml/SKILL.md +180 -0
- package/bin/fml +4 -18
- package/bin/hook-handler +7 -0
- package/bin/mcp-server +4 -18
- package/bin/observer +6 -0
- package/dist/chunk-3KISRT57.js +140 -0
- package/dist/chunk-56TPYMNK.js +4657 -0
- package/dist/chunk-7V7G7NVF.js +142 -0
- package/dist/chunk-A6HAEVJX.js +1606 -0
- package/dist/chunk-DZKYW327.js +1037 -0
- package/dist/chunk-ENW2GJNO.js +109 -0
- package/dist/{chunk-PZ5AY32C.js → chunk-MLKGABMK.js} +0 -1
- package/dist/{chunk-PTUZMKSN.js → chunk-R45CWWZO.js} +34 -23
- package/dist/chunk-WZLLO563.js +953 -0
- package/dist/chunk-YSU5CS2V.js +3331 -0
- package/dist/cli.js +7742 -1072
- package/dist/{esm-2EUDFZWP.js → esm-4YKLKFRX.js} +546 -741
- package/dist/hook-handler.js +440 -0
- package/dist/mcp/server.js +112 -98
- package/dist/observer-server.js +1566 -0
- package/dist/sync-cycle.js +31 -0
- package/dist/{token-store-3IQSLOTQ.js → token-store-BT5THIAJ.js} +10 -3
- package/hooks/hooks.json +290 -0
- package/package.json +24 -27
- package/dist/chunk-PTUZMKSN.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-S4TJNLEN.js +0 -61
- package/dist/chunk-S4TJNLEN.js.map +0 -1
- package/dist/chunk-VV4JHH4H.js +0 -437
- package/dist/chunk-VV4JHH4H.js.map +0 -1
- package/dist/chunk-YH5ROR7G.js +0 -69
- package/dist/chunk-YH5ROR7G.js.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/daemon-MSLDHLMH.js +0 -21
- package/dist/daemon-MSLDHLMH.js.map +0 -1
- package/dist/daemon-utils-OODBIYB7.js +0 -10
- package/dist/daemon-utils-OODBIYB7.js.map +0 -1
- package/dist/esm-2EUDFZWP.js.map +0 -1
- package/dist/mcp/server.js.map +0 -1
- package/dist/postinstall.js +0 -16
- package/dist/postinstall.js.map +0 -1
- package/dist/token-store-3IQSLOTQ.js.map +0 -1
- package/scripts/postinstall.cjs +0 -25
|
@@ -0,0 +1,3331 @@
|
|
|
1
|
+
import {
|
|
2
|
+
FML_INSTALLATION_ID_FILE,
|
|
3
|
+
FML_OBSERVER_DIR,
|
|
4
|
+
getActiveEnv,
|
|
5
|
+
getSelectedOrg,
|
|
6
|
+
getSiteUrlForConvexUrl,
|
|
7
|
+
getSyncDestinationIdentity,
|
|
8
|
+
getValidToken
|
|
9
|
+
} from "./chunk-A6HAEVJX.js";
|
|
10
|
+
|
|
11
|
+
// ../../packages/panopticon/src/artifact.ts
|
|
12
|
+
import {
|
|
13
|
+
createHash,
|
|
14
|
+
verify as verifySignature
|
|
15
|
+
} from "crypto";
|
|
16
|
+
function sha256Hex(content) {
|
|
17
|
+
return createHash("sha256").update(content).digest("hex");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// ../../packages/panopticon/src/download.ts
|
|
21
|
+
import path from "path";
|
|
22
|
+
import { fileURLToPath } from "url";
|
|
23
|
+
|
|
24
|
+
// ../../packages/panopticon/src/install.ts
|
|
25
|
+
import { spawn } from "child_process";
|
|
26
|
+
import path2 from "path";
|
|
27
|
+
|
|
28
|
+
// ../../packages/panopticon/src/install-id.ts
|
|
29
|
+
import { randomUUID } from "crypto";
|
|
30
|
+
import fs from "fs";
|
|
31
|
+
import os from "os";
|
|
32
|
+
import path3 from "path";
|
|
33
|
+
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
34
|
+
function readInstallationId(filePath) {
|
|
35
|
+
try {
|
|
36
|
+
const value = fs.readFileSync(filePath, "utf8").trim();
|
|
37
|
+
return UUID_RE.test(value) ? value.toLowerCase() : null;
|
|
38
|
+
} catch {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function loadOrCreateInstallationId(filePath) {
|
|
43
|
+
const existing = readInstallationId(filePath);
|
|
44
|
+
if (existing) return existing;
|
|
45
|
+
const minted = randomUUID();
|
|
46
|
+
fs.mkdirSync(path3.dirname(filePath), { recursive: true, mode: 448 });
|
|
47
|
+
try {
|
|
48
|
+
fs.writeFileSync(filePath, `${minted}
|
|
49
|
+
`, { flag: "wx", mode: 384 });
|
|
50
|
+
return minted;
|
|
51
|
+
} catch (error) {
|
|
52
|
+
if (error.code !== "EEXIST") throw error;
|
|
53
|
+
}
|
|
54
|
+
const winner = readInstallationId(filePath);
|
|
55
|
+
if (winner) return winner;
|
|
56
|
+
const temporaryFile = `${filePath}.tmp-${process.pid}-${randomUUID()}`;
|
|
57
|
+
try {
|
|
58
|
+
fs.writeFileSync(temporaryFile, `${minted}
|
|
59
|
+
`, { mode: 384 });
|
|
60
|
+
fs.renameSync(temporaryFile, filePath);
|
|
61
|
+
} finally {
|
|
62
|
+
fs.rmSync(temporaryFile, { force: true });
|
|
63
|
+
}
|
|
64
|
+
return minted;
|
|
65
|
+
}
|
|
66
|
+
function collectObserverIdentity(installIdFile) {
|
|
67
|
+
let fmlInstallId;
|
|
68
|
+
try {
|
|
69
|
+
fmlInstallId = loadOrCreateInstallationId(installIdFile);
|
|
70
|
+
} catch {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
let hostname;
|
|
74
|
+
try {
|
|
75
|
+
const value = os.hostname().trim();
|
|
76
|
+
if (value) hostname = value;
|
|
77
|
+
} catch {
|
|
78
|
+
hostname = void 0;
|
|
79
|
+
}
|
|
80
|
+
return { fmlInstallId, hostname, platform: process.platform };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ../../packages/panopticon/src/paths.ts
|
|
84
|
+
import os2 from "os";
|
|
85
|
+
import path4 from "path";
|
|
86
|
+
function getManagedClientPaths(env = process.env, homeDir = os2.homedir()) {
|
|
87
|
+
const fmlHome = env.FML_HOME ?? path4.join(homeDir, ".fml");
|
|
88
|
+
const root = env.PANOPTICON_HOME ?? path4.join(fmlHome, "panopticon");
|
|
89
|
+
return {
|
|
90
|
+
root,
|
|
91
|
+
current: path4.join(root, "current"),
|
|
92
|
+
previous: path4.join(root, "previous"),
|
|
93
|
+
versions: path4.join(root, "versions"),
|
|
94
|
+
downloads: path4.join(root, "downloads"),
|
|
95
|
+
staging: path4.join(root, "staging"),
|
|
96
|
+
stateFile: path4.join(root, "state.json"),
|
|
97
|
+
syncSettingsFile: path4.join(root, "sync.json"),
|
|
98
|
+
releaseManifest: path4.join(root, "release.json"),
|
|
99
|
+
artifactQueue: path4.join(root, "artifacts.queue.json"),
|
|
100
|
+
artifactCache: path4.join(root, "artifact-cache")
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ../../packages/panopticon/src/sync-settings.ts
|
|
105
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
106
|
+
import fs2 from "fs";
|
|
107
|
+
import path5 from "path";
|
|
108
|
+
var SYNC_SETTINGS_SCHEMA_VERSION = 1;
|
|
109
|
+
var DEFAULT_SYNC_ENABLED = true;
|
|
110
|
+
function defaultSyncSettings() {
|
|
111
|
+
return {
|
|
112
|
+
schemaVersion: SYNC_SETTINGS_SCHEMA_VERSION,
|
|
113
|
+
enabled: DEFAULT_SYNC_ENABLED
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
function disabledSyncSettings() {
|
|
117
|
+
return {
|
|
118
|
+
schemaVersion: SYNC_SETTINGS_SCHEMA_VERSION,
|
|
119
|
+
enabled: false
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function serializeSyncSettings(settings) {
|
|
123
|
+
return `${JSON.stringify(settings, null, 2)}
|
|
124
|
+
`;
|
|
125
|
+
}
|
|
126
|
+
function readSyncSettings(settingsFile) {
|
|
127
|
+
try {
|
|
128
|
+
const parsed = JSON.parse(fs2.readFileSync(settingsFile, "utf8"));
|
|
129
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
130
|
+
return disabledSyncSettings();
|
|
131
|
+
}
|
|
132
|
+
const settings = parsed;
|
|
133
|
+
if (settings.schemaVersion !== SYNC_SETTINGS_SCHEMA_VERSION || typeof settings.enabled !== "boolean") {
|
|
134
|
+
return disabledSyncSettings();
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
schemaVersion: SYNC_SETTINGS_SCHEMA_VERSION,
|
|
138
|
+
enabled: settings.enabled
|
|
139
|
+
};
|
|
140
|
+
} catch {
|
|
141
|
+
return disabledSyncSettings();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function initializeSyncSettings(settingsFile) {
|
|
145
|
+
const settings = defaultSyncSettings();
|
|
146
|
+
let descriptor;
|
|
147
|
+
fs2.mkdirSync(path5.dirname(settingsFile), { recursive: true });
|
|
148
|
+
try {
|
|
149
|
+
descriptor = fs2.openSync(settingsFile, "wx", 384);
|
|
150
|
+
fs2.writeFileSync(descriptor, serializeSyncSettings(settings));
|
|
151
|
+
fs2.fsyncSync(descriptor);
|
|
152
|
+
return settings;
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if (error.code === "EEXIST") {
|
|
155
|
+
return readSyncSettings(settingsFile);
|
|
156
|
+
}
|
|
157
|
+
throw error;
|
|
158
|
+
} finally {
|
|
159
|
+
if (descriptor !== void 0) fs2.closeSync(descriptor);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
function setSyncEnabled(settingsFile, enabled) {
|
|
163
|
+
const settings = {
|
|
164
|
+
schemaVersion: SYNC_SETTINGS_SCHEMA_VERSION,
|
|
165
|
+
enabled
|
|
166
|
+
};
|
|
167
|
+
const temporaryFile = `${settingsFile}.tmp-${process.pid}-${randomUUID2()}`;
|
|
168
|
+
fs2.mkdirSync(path5.dirname(settingsFile), { recursive: true });
|
|
169
|
+
try {
|
|
170
|
+
fs2.writeFileSync(temporaryFile, serializeSyncSettings(settings), {
|
|
171
|
+
mode: 384
|
|
172
|
+
});
|
|
173
|
+
try {
|
|
174
|
+
fs2.renameSync(temporaryFile, settingsFile);
|
|
175
|
+
} catch (error) {
|
|
176
|
+
if (process.platform !== "win32") throw error;
|
|
177
|
+
fs2.copyFileSync(temporaryFile, settingsFile);
|
|
178
|
+
}
|
|
179
|
+
} finally {
|
|
180
|
+
fs2.rmSync(temporaryFile, { force: true });
|
|
181
|
+
}
|
|
182
|
+
return settings;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// ../../packages/panopticon/src/artifacts.ts
|
|
186
|
+
import { createHash as createHash4, randomUUID as randomUUID3 } from "crypto";
|
|
187
|
+
import fs7 from "fs";
|
|
188
|
+
import os3 from "os";
|
|
189
|
+
import path17 from "path";
|
|
190
|
+
|
|
191
|
+
// ../../packages/agent-targets/src/artifacts/targets/claude.ts
|
|
192
|
+
import path8 from "path";
|
|
193
|
+
|
|
194
|
+
// ../../packages/agent-targets/src/config-paths.ts
|
|
195
|
+
import path6 from "path";
|
|
196
|
+
function claudeConfigDirectory(context) {
|
|
197
|
+
const configuredDir = context.claudeConfigDir ?? context.env?.CLAUDE_CONFIG_DIR;
|
|
198
|
+
return typeof configuredDir === "string" && configuredDir.trim().length > 0 ? path6.resolve(configuredDir) : path6.join(context.homeDir, ".claude");
|
|
199
|
+
}
|
|
200
|
+
function geminiConfigDirectory(context) {
|
|
201
|
+
const configuredHome = context.env.GEMINI_CLI_HOME;
|
|
202
|
+
const homeRoot = typeof configuredHome === "string" && configuredHome.trim().length > 0 ? path6.resolve(configuredHome) : context.homeDir;
|
|
203
|
+
return path6.join(homeRoot, ".gemini");
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// ../../packages/agent-targets/src/artifacts/filesystem.ts
|
|
207
|
+
import fs3 from "fs";
|
|
208
|
+
import path7 from "path";
|
|
209
|
+
var DEFAULT_SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
210
|
+
"node_modules",
|
|
211
|
+
"dist",
|
|
212
|
+
"build",
|
|
213
|
+
"coverage",
|
|
214
|
+
".cache",
|
|
215
|
+
".next",
|
|
216
|
+
".nuxt",
|
|
217
|
+
".turbo",
|
|
218
|
+
".vite",
|
|
219
|
+
"tmp",
|
|
220
|
+
"temp",
|
|
221
|
+
"backups",
|
|
222
|
+
"designqc-captures"
|
|
223
|
+
]);
|
|
224
|
+
function normalizeFilePath(filePath) {
|
|
225
|
+
return path7.resolve(filePath);
|
|
226
|
+
}
|
|
227
|
+
function safeReaddir(dir) {
|
|
228
|
+
try {
|
|
229
|
+
return fs3.readdirSync(dir, { withFileTypes: true });
|
|
230
|
+
} catch {
|
|
231
|
+
return [];
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
function walkFiles(input) {
|
|
235
|
+
const root = normalizeFilePath(input.root);
|
|
236
|
+
const files = [];
|
|
237
|
+
const seen = /* @__PURE__ */ new Set();
|
|
238
|
+
const skipDirs = input.skipDirs ?? DEFAULT_SKIP_DIRS;
|
|
239
|
+
function walk(dir, depth) {
|
|
240
|
+
if (depth > input.maxDepth) return;
|
|
241
|
+
for (const entry of safeReaddir(dir)) {
|
|
242
|
+
const entryPath = path7.join(dir, entry.name);
|
|
243
|
+
if (entry.isDirectory()) {
|
|
244
|
+
if (!skipDirs.has(entry.name)) walk(entryPath, depth + 1);
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
if (!entry.isFile() || !input.includeFile(entryPath)) continue;
|
|
248
|
+
const normalized = normalizeFilePath(entryPath);
|
|
249
|
+
if (seen.has(normalized)) continue;
|
|
250
|
+
seen.add(normalized);
|
|
251
|
+
files.push(normalized);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
walk(root, 0);
|
|
255
|
+
return files;
|
|
256
|
+
}
|
|
257
|
+
function transcriptCandidate(input) {
|
|
258
|
+
const sourcePath = normalizeFilePath(input.sourcePath);
|
|
259
|
+
const readPath = input.readPath ? normalizeFilePath(input.readPath) : void 0;
|
|
260
|
+
return {
|
|
261
|
+
artifactType: "agent_transcript",
|
|
262
|
+
source: input.targetId,
|
|
263
|
+
sourcePath,
|
|
264
|
+
...input.identityKey ? { identityKey: input.identityKey } : {},
|
|
265
|
+
...readPath && readPath !== sourcePath ? { readPath } : {},
|
|
266
|
+
sessionId: input.sessionId,
|
|
267
|
+
metadata: {
|
|
268
|
+
originalPath: sourcePath,
|
|
269
|
+
scope: "global",
|
|
270
|
+
target: input.targetId,
|
|
271
|
+
...input.metadata ?? {}
|
|
272
|
+
},
|
|
273
|
+
repositoryObservations: input.repositoryObservations
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// ../../packages/agent-targets/src/artifacts/harness-install-id.ts
|
|
278
|
+
import fs4 from "fs";
|
|
279
|
+
var MAX_HARNESS_INSTALL_ID_LENGTH = 128;
|
|
280
|
+
function normalizeHarnessInstallId(value) {
|
|
281
|
+
if (typeof value !== "string") return null;
|
|
282
|
+
const trimmed = value.trim();
|
|
283
|
+
if (!trimmed || trimmed.length > MAX_HARNESS_INSTALL_ID_LENGTH) return null;
|
|
284
|
+
if (!/^[\x21-\x7e]+$/.test(trimmed)) return null;
|
|
285
|
+
return trimmed;
|
|
286
|
+
}
|
|
287
|
+
function readHarnessInstallIdFile(filePath) {
|
|
288
|
+
try {
|
|
289
|
+
return normalizeHarnessInstallId(fs4.readFileSync(filePath, "utf8"));
|
|
290
|
+
} catch {
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
var claudeMachineIdCache = /* @__PURE__ */ new Map();
|
|
295
|
+
function readClaudeMachineId(configFile) {
|
|
296
|
+
let stat;
|
|
297
|
+
try {
|
|
298
|
+
stat = fs4.statSync(configFile);
|
|
299
|
+
} catch {
|
|
300
|
+
claudeMachineIdCache.delete(configFile);
|
|
301
|
+
return null;
|
|
302
|
+
}
|
|
303
|
+
const cached = claudeMachineIdCache.get(configFile);
|
|
304
|
+
if (cached && cached.mtimeMs === stat.mtimeMs && cached.sizeBytes === stat.size) {
|
|
305
|
+
return cached.machineId;
|
|
306
|
+
}
|
|
307
|
+
let machineId = null;
|
|
308
|
+
try {
|
|
309
|
+
const parsed = JSON.parse(fs4.readFileSync(configFile, "utf8"));
|
|
310
|
+
if (parsed && typeof parsed === "object") {
|
|
311
|
+
machineId = normalizeHarnessInstallId(
|
|
312
|
+
parsed.machineID
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
} catch {
|
|
316
|
+
machineId = null;
|
|
317
|
+
}
|
|
318
|
+
claudeMachineIdCache.set(configFile, {
|
|
319
|
+
mtimeMs: stat.mtimeMs,
|
|
320
|
+
sizeBytes: stat.size,
|
|
321
|
+
machineId
|
|
322
|
+
});
|
|
323
|
+
return machineId;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// ../../packages/agent-targets/src/artifacts/repository-context.ts
|
|
327
|
+
import { createHash as createHash2 } from "crypto";
|
|
328
|
+
import fs5 from "fs";
|
|
329
|
+
|
|
330
|
+
// ../../packages/shared/src/github-repository.ts
|
|
331
|
+
var MAX_REPOSITORY_BYTES = 201;
|
|
332
|
+
var UTF8_ENCODER = new TextEncoder();
|
|
333
|
+
function utf8Bytes(value) {
|
|
334
|
+
return UTF8_ENCODER.encode(value).byteLength;
|
|
335
|
+
}
|
|
336
|
+
function githubRepositoryPath(pathname) {
|
|
337
|
+
const parts = pathname.replace(/^\/+|\/+$/g, "").split("/");
|
|
338
|
+
if (parts.length !== 2) return null;
|
|
339
|
+
const owner = parts[0];
|
|
340
|
+
const repository = parts[1]?.replace(/\.git$/i, "");
|
|
341
|
+
if (!owner || !repository) return null;
|
|
342
|
+
const segmentPattern = /^[A-Za-z0-9_.-]+$/;
|
|
343
|
+
if (!segmentPattern.test(owner) || !segmentPattern.test(repository)) {
|
|
344
|
+
return null;
|
|
345
|
+
}
|
|
346
|
+
const canonical = `${owner.toLowerCase()}/${repository.toLowerCase()}`;
|
|
347
|
+
if (canonical.startsWith("./") || canonical.startsWith("../") || repository === "." || repository === ".." || repository.toLowerCase().endsWith(".git")) {
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
return utf8Bytes(canonical) <= MAX_REPOSITORY_BYTES ? canonical : null;
|
|
351
|
+
}
|
|
352
|
+
function canonicalGitHubRepository(remote) {
|
|
353
|
+
const trimmed = remote.trim();
|
|
354
|
+
if (!trimmed || trimmed.includes("\0")) return null;
|
|
355
|
+
const scp = trimmed.match(/^git@github\.com:([^?#]+)$/i);
|
|
356
|
+
if (scp?.[1]) return githubRepositoryPath(scp[1]);
|
|
357
|
+
let url;
|
|
358
|
+
try {
|
|
359
|
+
url = new URL(trimmed);
|
|
360
|
+
} catch {
|
|
361
|
+
return null;
|
|
362
|
+
}
|
|
363
|
+
if (url.hostname.toLowerCase() !== "github.com") return null;
|
|
364
|
+
if (url.search || url.hash || url.password || url.port) return null;
|
|
365
|
+
if (url.protocol === "https:" || url.protocol === "http:") {
|
|
366
|
+
if (url.username) return null;
|
|
367
|
+
} else if (url.protocol === "ssh:") {
|
|
368
|
+
if (url.username && url.username !== "git") return null;
|
|
369
|
+
} else {
|
|
370
|
+
return null;
|
|
371
|
+
}
|
|
372
|
+
return githubRepositoryPath(url.pathname);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// ../../packages/agent-targets/src/artifacts/repository-context.ts
|
|
376
|
+
var MAX_ARTIFACT_CONTEXT_PREFIX_BYTES = 256 * 1024;
|
|
377
|
+
var MAX_ARTIFACT_CONTEXT_RECORDS = 64;
|
|
378
|
+
var MAX_CWD_BYTES = 4096;
|
|
379
|
+
var MAX_BRANCH_BYTES = 1024;
|
|
380
|
+
function utf8Bytes2(value) {
|
|
381
|
+
return Buffer.byteLength(value, "utf8");
|
|
382
|
+
}
|
|
383
|
+
function boundedOptionalString(value, maxBytes) {
|
|
384
|
+
if (typeof value !== "string") return void 0;
|
|
385
|
+
const trimmed = value.trim();
|
|
386
|
+
if (!trimmed || new RegExp("\\p{Cc}", "u").test(trimmed) || utf8Bytes2(trimmed) > maxBytes) {
|
|
387
|
+
return void 0;
|
|
388
|
+
}
|
|
389
|
+
return trimmed;
|
|
390
|
+
}
|
|
391
|
+
function recordValue(value) {
|
|
392
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
393
|
+
}
|
|
394
|
+
function readJsonlContextRecords(sourcePath) {
|
|
395
|
+
let file;
|
|
396
|
+
try {
|
|
397
|
+
file = fs5.openSync(sourcePath, "r");
|
|
398
|
+
} catch {
|
|
399
|
+
return [];
|
|
400
|
+
}
|
|
401
|
+
try {
|
|
402
|
+
const stat = fs5.fstatSync(file);
|
|
403
|
+
const byteLimit = Math.min(stat.size, MAX_ARTIFACT_CONTEXT_PREFIX_BYTES);
|
|
404
|
+
if (byteLimit <= 0) return [];
|
|
405
|
+
const buffer = Buffer.allocUnsafe(byteLimit);
|
|
406
|
+
const bytesRead = fs5.readSync(file, buffer, 0, byteLimit, 0);
|
|
407
|
+
let content = buffer.subarray(0, bytesRead).toString("utf8");
|
|
408
|
+
if (bytesRead < stat.size) {
|
|
409
|
+
const lastNewline = content.lastIndexOf("\n");
|
|
410
|
+
if (lastNewline < 0) return [];
|
|
411
|
+
content = content.slice(0, lastNewline + 1);
|
|
412
|
+
}
|
|
413
|
+
const records = [];
|
|
414
|
+
for (const line of content.split(/\r?\n/)) {
|
|
415
|
+
if (records.length >= MAX_ARTIFACT_CONTEXT_RECORDS) break;
|
|
416
|
+
const jsonLine = line.startsWith("\uFEFF") ? line.slice(1) : line;
|
|
417
|
+
if (!jsonLine.trim()) continue;
|
|
418
|
+
try {
|
|
419
|
+
const record = recordValue(JSON.parse(jsonLine));
|
|
420
|
+
if (record) records.push(record);
|
|
421
|
+
} catch {
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
return records;
|
|
425
|
+
} catch {
|
|
426
|
+
return [];
|
|
427
|
+
} finally {
|
|
428
|
+
fs5.closeSync(file);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
function contextTimestampMs(value, fallback) {
|
|
432
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
|
|
433
|
+
const timestamp = value < 1e10 ? Math.round(value * 1e3) : Math.round(value);
|
|
434
|
+
return Number.isSafeInteger(timestamp) ? timestamp : fallback;
|
|
435
|
+
}
|
|
436
|
+
if (typeof value === "string") {
|
|
437
|
+
const numeric = Number(value);
|
|
438
|
+
if (Number.isFinite(numeric)) return contextTimestampMs(numeric, fallback);
|
|
439
|
+
const parsed = Date.parse(value);
|
|
440
|
+
if (Number.isSafeInteger(parsed) && parsed >= 0) return parsed;
|
|
441
|
+
}
|
|
442
|
+
return fallback;
|
|
443
|
+
}
|
|
444
|
+
function repositoryObservation(input) {
|
|
445
|
+
const cwd = boundedOptionalString(input.cwd, MAX_CWD_BYTES);
|
|
446
|
+
const branch = boundedOptionalString(input.branch, MAX_BRANCH_BYTES);
|
|
447
|
+
const rawCommitHash = boundedOptionalString(input.commitHash, 64);
|
|
448
|
+
const commitHash = rawCommitHash && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i.test(rawCommitHash) ? rawCommitHash.toLowerCase() : void 0;
|
|
449
|
+
const identity = JSON.stringify([
|
|
450
|
+
input.source,
|
|
451
|
+
input.repository,
|
|
452
|
+
cwd ?? null,
|
|
453
|
+
branch ?? null,
|
|
454
|
+
commitHash ?? null
|
|
455
|
+
]);
|
|
456
|
+
const id = `repo_observation:${createHash2("sha256").update(identity).digest("hex")}`;
|
|
457
|
+
return {
|
|
458
|
+
id,
|
|
459
|
+
repository: input.repository,
|
|
460
|
+
source: input.source,
|
|
461
|
+
observedAtMs: input.observedAtMs,
|
|
462
|
+
...cwd ? { cwd } : {},
|
|
463
|
+
...branch ? { branch } : {},
|
|
464
|
+
...commitHash ? { commitHash } : {}
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// ../../packages/agent-targets/src/artifacts/targets/claude.ts
|
|
469
|
+
function claudeTranscriptCandidate(sourcePath, context, harnessInstallId) {
|
|
470
|
+
const fallbackObservedAtMs = (context.nowMs ?? Date.now)();
|
|
471
|
+
const observations = /* @__PURE__ */ new Map();
|
|
472
|
+
for (const record of readJsonlContextRecords(sourcePath)) {
|
|
473
|
+
if (typeof record.cwd !== "string" || !record.cwd.trim()) continue;
|
|
474
|
+
const origin = context.resolveGitHubOrigin?.(record.cwd);
|
|
475
|
+
if (!origin) continue;
|
|
476
|
+
const observation = repositoryObservation({
|
|
477
|
+
repository: origin.repository,
|
|
478
|
+
source: "git_origin",
|
|
479
|
+
observedAtMs: fallbackObservedAtMs,
|
|
480
|
+
cwd: record.cwd,
|
|
481
|
+
branch: origin.branch,
|
|
482
|
+
commitHash: origin.commitHash
|
|
483
|
+
});
|
|
484
|
+
observations.set(observation.id, observation);
|
|
485
|
+
}
|
|
486
|
+
return transcriptCandidate({
|
|
487
|
+
targetId: "claude",
|
|
488
|
+
sourcePath,
|
|
489
|
+
metadata: harnessInstallId ? { harnessInstallId } : void 0,
|
|
490
|
+
repositoryObservations: [...observations.values()]
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
var claudeArtifactDiscovery = Object.freeze({
|
|
494
|
+
targetId: "claude",
|
|
495
|
+
discover: (context) => {
|
|
496
|
+
const { homeDir, env } = context;
|
|
497
|
+
const configDir = claudeConfigDirectory({ homeDir, env });
|
|
498
|
+
const harnessInstallId = readClaudeMachineId(path8.join(configDir, ".claude.json")) ?? readClaudeMachineId(path8.join(homeDir, ".claude.json"));
|
|
499
|
+
return walkFiles({
|
|
500
|
+
root: path8.join(configDir, "projects"),
|
|
501
|
+
maxDepth: 6,
|
|
502
|
+
includeFile: (filePath) => filePath.endsWith(".jsonl")
|
|
503
|
+
}).map(
|
|
504
|
+
(sourcePath) => claudeTranscriptCandidate(sourcePath, context, harnessInstallId)
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
});
|
|
508
|
+
|
|
509
|
+
// ../../packages/agent-targets/src/artifacts/targets/codex.ts
|
|
510
|
+
import path9 from "path";
|
|
511
|
+
var CODEX_ROLLOUT_ID = /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl(?:\.zst)?$/i;
|
|
512
|
+
function codexRolloutId(sourcePath) {
|
|
513
|
+
return CODEX_ROLLOUT_ID.exec(path9.basename(sourcePath))?.[1]?.toLowerCase() ?? null;
|
|
514
|
+
}
|
|
515
|
+
function codexTranscriptCandidate(sourcePath, readPath, store, context, harnessInstallId) {
|
|
516
|
+
const rolloutId = codexRolloutId(sourcePath);
|
|
517
|
+
const identityKey = rolloutId ? `codex_rollout:${rolloutId}` : null;
|
|
518
|
+
const identityMetadata = harnessInstallId ? { harnessInstallId } : {};
|
|
519
|
+
const fallbackObservedAtMs = (context.nowMs ?? Date.now)();
|
|
520
|
+
for (const record of readJsonlContextRecords(readPath)) {
|
|
521
|
+
if (record.type !== "session_meta") continue;
|
|
522
|
+
const payload = recordValue(record.payload);
|
|
523
|
+
if (!payload) continue;
|
|
524
|
+
const sessionId = typeof payload.id === "string" && payload.id.trim() ? payload.id.trim() : null;
|
|
525
|
+
const cwd = typeof payload.cwd === "string" ? payload.cwd : void 0;
|
|
526
|
+
const git = recordValue(payload.git);
|
|
527
|
+
const nativeRepository = typeof git?.repository_url === "string" ? canonicalGitHubRepository(git.repository_url) : null;
|
|
528
|
+
const origin = cwd ? context.resolveGitHubOrigin?.(cwd) : null;
|
|
529
|
+
const observations = [];
|
|
530
|
+
if (nativeRepository) {
|
|
531
|
+
observations.push(
|
|
532
|
+
repositoryObservation({
|
|
533
|
+
repository: nativeRepository,
|
|
534
|
+
source: "codex_session_meta",
|
|
535
|
+
observedAtMs: contextTimestampMs(
|
|
536
|
+
payload.timestamp ?? record.timestamp,
|
|
537
|
+
fallbackObservedAtMs
|
|
538
|
+
),
|
|
539
|
+
cwd,
|
|
540
|
+
branch: git?.branch,
|
|
541
|
+
commitHash: git?.commit_hash
|
|
542
|
+
})
|
|
543
|
+
);
|
|
544
|
+
}
|
|
545
|
+
if (origin) {
|
|
546
|
+
observations.push(
|
|
547
|
+
repositoryObservation({
|
|
548
|
+
repository: origin.repository,
|
|
549
|
+
source: "git_origin",
|
|
550
|
+
observedAtMs: fallbackObservedAtMs,
|
|
551
|
+
cwd,
|
|
552
|
+
branch: origin.branch,
|
|
553
|
+
commitHash: origin.commitHash
|
|
554
|
+
})
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
return transcriptCandidate({
|
|
558
|
+
targetId: "codex",
|
|
559
|
+
sourcePath,
|
|
560
|
+
identityKey,
|
|
561
|
+
readPath,
|
|
562
|
+
sessionId,
|
|
563
|
+
metadata: { store, ...identityMetadata },
|
|
564
|
+
repositoryObservations: observations
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
return transcriptCandidate({
|
|
568
|
+
targetId: "codex",
|
|
569
|
+
sourcePath,
|
|
570
|
+
identityKey,
|
|
571
|
+
readPath,
|
|
572
|
+
// A rollout whose session_meta is absent from the bounded prefix still
|
|
573
|
+
// reads its bytes from the decoded copy; only the evidence is missing.
|
|
574
|
+
...rolloutId ? { sessionId: rolloutId } : {},
|
|
575
|
+
metadata: { store, ...identityMetadata }
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
var codexArtifactDiscovery = Object.freeze({
|
|
579
|
+
targetId: "codex",
|
|
580
|
+
discover: (context) => {
|
|
581
|
+
const { homeDir, env } = context;
|
|
582
|
+
const configuredHome = env.CODEX_HOME?.trim();
|
|
583
|
+
const codexHome = configuredHome ? path9.resolve(configuredHome) : path9.join(homeDir, ".codex");
|
|
584
|
+
const harnessInstallId = readHarnessInstallIdFile(
|
|
585
|
+
path9.join(codexHome, "installation_id")
|
|
586
|
+
);
|
|
587
|
+
const roots = [
|
|
588
|
+
{ store: "sessions", root: path9.join(codexHome, "sessions") },
|
|
589
|
+
{
|
|
590
|
+
store: "archived_sessions",
|
|
591
|
+
root: path9.join(codexHome, "archived_sessions")
|
|
592
|
+
}
|
|
593
|
+
];
|
|
594
|
+
const files = roots.flatMap(
|
|
595
|
+
({ root, store }) => walkFiles({
|
|
596
|
+
root,
|
|
597
|
+
maxDepth: 6,
|
|
598
|
+
includeFile: (filePath) => filePath.endsWith(".jsonl") || filePath.endsWith(".jsonl.zst")
|
|
599
|
+
}).sort((left, right) => {
|
|
600
|
+
const leftCompressed = left.endsWith(".zst") ? 1 : 0;
|
|
601
|
+
const rightCompressed = right.endsWith(".zst") ? 1 : 0;
|
|
602
|
+
return leftCompressed - rightCompressed || left.localeCompare(right);
|
|
603
|
+
}).map((sourcePath) => ({ sourcePath, store }))
|
|
604
|
+
);
|
|
605
|
+
const seenRollouts = /* @__PURE__ */ new Set();
|
|
606
|
+
return files.filter(({ sourcePath }) => {
|
|
607
|
+
const rolloutId = codexRolloutId(sourcePath);
|
|
608
|
+
if (!rolloutId) return true;
|
|
609
|
+
if (seenRollouts.has(rolloutId)) return false;
|
|
610
|
+
seenRollouts.add(rolloutId);
|
|
611
|
+
return true;
|
|
612
|
+
}).flatMap(({ sourcePath, store }) => {
|
|
613
|
+
const readPath = sourcePath.endsWith(".zst") ? context.materializeArtifact?.(sourcePath) ?? null : sourcePath;
|
|
614
|
+
if (!readPath) return [];
|
|
615
|
+
return [
|
|
616
|
+
codexTranscriptCandidate(
|
|
617
|
+
sourcePath,
|
|
618
|
+
readPath,
|
|
619
|
+
store,
|
|
620
|
+
context,
|
|
621
|
+
harnessInstallId
|
|
622
|
+
)
|
|
623
|
+
];
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
});
|
|
627
|
+
|
|
628
|
+
// ../../packages/agent-targets/src/artifacts/targets/copilot-cli.ts
|
|
629
|
+
import path10 from "path";
|
|
630
|
+
var copilotCliArtifactDiscovery = Object.freeze({
|
|
631
|
+
targetId: "copilot-cli",
|
|
632
|
+
discover: ({ homeDir }) => walkFiles({
|
|
633
|
+
root: path10.join(homeDir, ".copilot", "session-state"),
|
|
634
|
+
maxDepth: 3,
|
|
635
|
+
includeFile: (filePath) => path10.basename(filePath) === "events.jsonl"
|
|
636
|
+
}).map(
|
|
637
|
+
(sourcePath) => transcriptCandidate({ targetId: "copilot-cli", sourcePath })
|
|
638
|
+
)
|
|
639
|
+
});
|
|
640
|
+
|
|
641
|
+
// ../../packages/agent-targets/src/artifacts/targets/cursor.ts
|
|
642
|
+
import path11 from "path";
|
|
643
|
+
function hasPathSegment(filePath, segment) {
|
|
644
|
+
return normalizeFilePath(filePath).split(path11.sep).includes(segment);
|
|
645
|
+
}
|
|
646
|
+
var cursorArtifactDiscovery = Object.freeze({
|
|
647
|
+
targetId: "cursor",
|
|
648
|
+
discover: ({ homeDir }) => walkFiles({
|
|
649
|
+
root: path11.join(homeDir, ".cursor", "projects"),
|
|
650
|
+
maxDepth: 5,
|
|
651
|
+
includeFile: (filePath) => filePath.endsWith(".jsonl") && hasPathSegment(filePath, "agent-transcripts")
|
|
652
|
+
}).map(
|
|
653
|
+
(sourcePath) => transcriptCandidate({ targetId: "cursor", sourcePath })
|
|
654
|
+
)
|
|
655
|
+
});
|
|
656
|
+
|
|
657
|
+
// ../../packages/agent-targets/src/artifacts/targets/factory-droid.ts
|
|
658
|
+
import path12 from "path";
|
|
659
|
+
var factoryDroidArtifactDiscovery = Object.freeze({
|
|
660
|
+
targetId: "factory-droid",
|
|
661
|
+
discover: ({ homeDir }) => walkFiles({
|
|
662
|
+
root: path12.join(homeDir, ".factory", "sessions"),
|
|
663
|
+
maxDepth: 4,
|
|
664
|
+
includeFile: (filePath) => filePath.endsWith(".jsonl")
|
|
665
|
+
}).map(
|
|
666
|
+
(sourcePath) => transcriptCandidate({ targetId: "factory-droid", sourcePath })
|
|
667
|
+
)
|
|
668
|
+
});
|
|
669
|
+
|
|
670
|
+
// ../../packages/agent-targets/src/artifacts/targets/gemini.ts
|
|
671
|
+
import path13 from "path";
|
|
672
|
+
var GEMINI_DISCOVER_SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
673
|
+
"bin",
|
|
674
|
+
"checkpoints",
|
|
675
|
+
"logs",
|
|
676
|
+
"plans",
|
|
677
|
+
"tasks",
|
|
678
|
+
"tool-outputs",
|
|
679
|
+
"tracker"
|
|
680
|
+
]);
|
|
681
|
+
var geminiArtifactDiscovery = Object.freeze({
|
|
682
|
+
targetId: "gemini",
|
|
683
|
+
discover: ({ homeDir, env }) => {
|
|
684
|
+
const geminiDir = geminiConfigDirectory({ homeDir, env });
|
|
685
|
+
const harnessInstallId = readHarnessInstallIdFile(
|
|
686
|
+
path13.join(geminiDir, "installation_id")
|
|
687
|
+
);
|
|
688
|
+
const root = path13.join(geminiDir, "tmp");
|
|
689
|
+
const files = [];
|
|
690
|
+
const seen = /* @__PURE__ */ new Set();
|
|
691
|
+
function walk(dir, depth) {
|
|
692
|
+
if (depth > 4) return;
|
|
693
|
+
for (const entry of safeReaddir(dir)) {
|
|
694
|
+
const entryPath = path13.join(dir, entry.name);
|
|
695
|
+
if (!entry.isDirectory()) continue;
|
|
696
|
+
if (GEMINI_DISCOVER_SKIP_DIRS.has(entry.name)) continue;
|
|
697
|
+
if (entry.name === "chats") {
|
|
698
|
+
for (const chatEntry of safeReaddir(entryPath)) {
|
|
699
|
+
if (!chatEntry.isFile() || !chatEntry.name.endsWith(".json")) {
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
702
|
+
const sourcePath = normalizeFilePath(
|
|
703
|
+
path13.join(entryPath, chatEntry.name)
|
|
704
|
+
);
|
|
705
|
+
if (seen.has(sourcePath)) continue;
|
|
706
|
+
seen.add(sourcePath);
|
|
707
|
+
files.push(sourcePath);
|
|
708
|
+
}
|
|
709
|
+
continue;
|
|
710
|
+
}
|
|
711
|
+
walk(entryPath, depth + 1);
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
walk(root, 0);
|
|
715
|
+
return files.map(
|
|
716
|
+
(sourcePath) => transcriptCandidate({
|
|
717
|
+
targetId: "gemini",
|
|
718
|
+
sourcePath,
|
|
719
|
+
metadata: harnessInstallId ? { harnessInstallId } : void 0
|
|
720
|
+
})
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
});
|
|
724
|
+
|
|
725
|
+
// ../../packages/agent-targets/src/artifacts/targets/grok.ts
|
|
726
|
+
import path14 from "path";
|
|
727
|
+
function grokHome(homeDir, env) {
|
|
728
|
+
const configuredHome = env.GROK_HOME?.trim();
|
|
729
|
+
return configuredHome ? path14.resolve(configuredHome) : path14.join(homeDir, ".grok");
|
|
730
|
+
}
|
|
731
|
+
var grokArtifactDiscovery = Object.freeze({
|
|
732
|
+
targetId: "grok",
|
|
733
|
+
discover: ({ homeDir, env }) => walkFiles({
|
|
734
|
+
root: path14.join(grokHome(homeDir, env), "sessions"),
|
|
735
|
+
maxDepth: 5,
|
|
736
|
+
includeFile: (filePath) => path14.basename(filePath) === "updates.jsonl"
|
|
737
|
+
}).map(
|
|
738
|
+
(sourcePath) => transcriptCandidate({ targetId: "grok", sourcePath })
|
|
739
|
+
)
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
// ../../packages/agent-targets/src/artifacts/targets/pi.ts
|
|
743
|
+
import path15 from "path";
|
|
744
|
+
function piTranscriptCandidate(sourcePath, context) {
|
|
745
|
+
const fallbackObservedAtMs = (context.nowMs ?? Date.now)();
|
|
746
|
+
for (const record of readJsonlContextRecords(sourcePath)) {
|
|
747
|
+
if (record.type !== "session") continue;
|
|
748
|
+
const sessionId = typeof record.id === "string" && record.id.trim() ? record.id.trim() : null;
|
|
749
|
+
if (typeof record.cwd !== "string" || !record.cwd.trim()) {
|
|
750
|
+
return transcriptCandidate({
|
|
751
|
+
targetId: "pi",
|
|
752
|
+
sourcePath,
|
|
753
|
+
sessionId
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
const origin = context.resolveGitHubOrigin?.(record.cwd);
|
|
757
|
+
return transcriptCandidate({
|
|
758
|
+
targetId: "pi",
|
|
759
|
+
sourcePath,
|
|
760
|
+
sessionId,
|
|
761
|
+
repositoryObservations: origin ? [
|
|
762
|
+
repositoryObservation({
|
|
763
|
+
repository: origin.repository,
|
|
764
|
+
source: "git_origin",
|
|
765
|
+
observedAtMs: fallbackObservedAtMs,
|
|
766
|
+
cwd: record.cwd,
|
|
767
|
+
branch: origin.branch,
|
|
768
|
+
commitHash: origin.commitHash
|
|
769
|
+
})
|
|
770
|
+
] : []
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
return transcriptCandidate({ targetId: "pi", sourcePath });
|
|
774
|
+
}
|
|
775
|
+
var piArtifactDiscovery = Object.freeze({
|
|
776
|
+
targetId: "pi",
|
|
777
|
+
discover: (context) => {
|
|
778
|
+
const { homeDir } = context;
|
|
779
|
+
return walkFiles({
|
|
780
|
+
root: path15.join(homeDir, ".pi", "agent", "sessions"),
|
|
781
|
+
maxDepth: 8,
|
|
782
|
+
includeFile: (filePath) => filePath.endsWith(".jsonl")
|
|
783
|
+
}).map((sourcePath) => piTranscriptCandidate(sourcePath, context));
|
|
784
|
+
}
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
// ../../packages/agent-targets/src/artifacts/registry.ts
|
|
788
|
+
var ARTIFACT_DISCOVERY_CAPABILITIES = Object.freeze([
|
|
789
|
+
claudeArtifactDiscovery,
|
|
790
|
+
codexArtifactDiscovery,
|
|
791
|
+
geminiArtifactDiscovery,
|
|
792
|
+
grokArtifactDiscovery,
|
|
793
|
+
piArtifactDiscovery,
|
|
794
|
+
cursorArtifactDiscovery,
|
|
795
|
+
copilotCliArtifactDiscovery,
|
|
796
|
+
factoryDroidArtifactDiscovery
|
|
797
|
+
]);
|
|
798
|
+
function discoverTargetArtifactCandidates(context) {
|
|
799
|
+
const observedAtMs = (context.nowMs ?? Date.now)();
|
|
800
|
+
const capabilityContext = {
|
|
801
|
+
...context,
|
|
802
|
+
nowMs: () => observedAtMs
|
|
803
|
+
};
|
|
804
|
+
return ARTIFACT_DISCOVERY_CAPABILITIES.flatMap(
|
|
805
|
+
(capability) => capability.discover(capabilityContext)
|
|
806
|
+
);
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
// ../../packages/agent-targets/src/artifacts/materialize.ts
|
|
810
|
+
import { createHash as createHash3 } from "crypto";
|
|
811
|
+
import fs6 from "fs";
|
|
812
|
+
import path16 from "path";
|
|
813
|
+
import zlib from "zlib";
|
|
814
|
+
var MAX_MATERIALIZED_ARTIFACT_BYTES = 100 * 1024 * 1024;
|
|
815
|
+
var DEFAULT_MATERIALIZED_CACHE_BYTES = 2 * 1024 * 1024 * 1024;
|
|
816
|
+
function cacheEntryName(sourcePath, stat) {
|
|
817
|
+
const identity = `${path16.resolve(sourcePath)}\0${stat.size}\0${Math.floor(
|
|
818
|
+
stat.mtimeMs
|
|
819
|
+
)}`;
|
|
820
|
+
const digest = createHash3("sha256").update(identity).digest("hex");
|
|
821
|
+
return `${digest}.jsonl`;
|
|
822
|
+
}
|
|
823
|
+
function supportsCompressedArtifacts() {
|
|
824
|
+
return typeof zlib.zstdDecompressSync === "function";
|
|
825
|
+
}
|
|
826
|
+
function decodeZstd(sourcePath, maxOutputBytes) {
|
|
827
|
+
const decompress = zlib.zstdDecompressSync;
|
|
828
|
+
if (typeof decompress !== "function") return null;
|
|
829
|
+
try {
|
|
830
|
+
return decompress(fs6.readFileSync(sourcePath), {
|
|
831
|
+
maxOutputLength: maxOutputBytes
|
|
832
|
+
});
|
|
833
|
+
} catch {
|
|
834
|
+
return null;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
function materializedArtifactPath(input) {
|
|
838
|
+
try {
|
|
839
|
+
const stat = fs6.statSync(input.sourcePath);
|
|
840
|
+
if (!stat.isFile() || stat.size === 0) return null;
|
|
841
|
+
return path16.join(input.cacheDir, cacheEntryName(input.sourcePath, stat));
|
|
842
|
+
} catch {
|
|
843
|
+
return null;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
function materializeCompressedArtifact(input) {
|
|
847
|
+
let stat;
|
|
848
|
+
try {
|
|
849
|
+
stat = fs6.statSync(input.sourcePath);
|
|
850
|
+
} catch {
|
|
851
|
+
return null;
|
|
852
|
+
}
|
|
853
|
+
if (!stat.isFile() || stat.size === 0) return null;
|
|
854
|
+
const cachePath = path16.join(
|
|
855
|
+
input.cacheDir,
|
|
856
|
+
cacheEntryName(input.sourcePath, stat)
|
|
857
|
+
);
|
|
858
|
+
try {
|
|
859
|
+
if (fs6.statSync(cachePath).isFile()) {
|
|
860
|
+
const now = /* @__PURE__ */ new Date();
|
|
861
|
+
try {
|
|
862
|
+
fs6.utimesSync(cachePath, now, now);
|
|
863
|
+
} catch {
|
|
864
|
+
}
|
|
865
|
+
return cachePath;
|
|
866
|
+
}
|
|
867
|
+
} catch {
|
|
868
|
+
}
|
|
869
|
+
const decoded = decodeZstd(
|
|
870
|
+
input.sourcePath,
|
|
871
|
+
input.maxOutputBytes ?? MAX_MATERIALIZED_ARTIFACT_BYTES
|
|
872
|
+
);
|
|
873
|
+
if (!decoded) return null;
|
|
874
|
+
try {
|
|
875
|
+
fs6.mkdirSync(input.cacheDir, { recursive: true, mode: 448 });
|
|
876
|
+
const tempPath = `${cachePath}.${process.pid}.tmp`;
|
|
877
|
+
fs6.writeFileSync(tempPath, decoded, { mode: 384 });
|
|
878
|
+
fs6.renameSync(tempPath, cachePath);
|
|
879
|
+
return cachePath;
|
|
880
|
+
} catch {
|
|
881
|
+
return null;
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
function pruneMaterializedCache(input) {
|
|
885
|
+
const maxTotalBytes = input.maxTotalBytes ?? DEFAULT_MATERIALIZED_CACHE_BYTES;
|
|
886
|
+
let entries;
|
|
887
|
+
try {
|
|
888
|
+
entries = fs6.readdirSync(input.cacheDir, { withFileTypes: true });
|
|
889
|
+
} catch {
|
|
890
|
+
return { removed: 0, retainedBytes: 0 };
|
|
891
|
+
}
|
|
892
|
+
const files = [];
|
|
893
|
+
for (const entry of entries) {
|
|
894
|
+
if (!entry.isFile()) continue;
|
|
895
|
+
const filePath = path16.join(input.cacheDir, entry.name);
|
|
896
|
+
try {
|
|
897
|
+
const stat = fs6.statSync(filePath);
|
|
898
|
+
files.push({ path: filePath, size: stat.size, atimeMs: stat.atimeMs });
|
|
899
|
+
} catch {
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
files.sort((left, right) => right.atimeMs - left.atimeMs);
|
|
903
|
+
let retainedBytes = 0;
|
|
904
|
+
let removed = 0;
|
|
905
|
+
for (const file of files) {
|
|
906
|
+
const pinned = input.keepPaths?.has(file.path) ?? false;
|
|
907
|
+
if (pinned || retainedBytes + file.size <= maxTotalBytes) {
|
|
908
|
+
retainedBytes += file.size;
|
|
909
|
+
continue;
|
|
910
|
+
}
|
|
911
|
+
try {
|
|
912
|
+
fs6.rmSync(file.path);
|
|
913
|
+
removed += 1;
|
|
914
|
+
} catch {
|
|
915
|
+
retainedBytes += file.size;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
return { removed, retainedBytes };
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// ../../packages/panopticon/src/repository-context.ts
|
|
922
|
+
import { execFileSync } from "child_process";
|
|
923
|
+
var MAX_CWD_BYTES2 = 4096;
|
|
924
|
+
var GIT_OUTPUT_BYTES = 16 * 1024;
|
|
925
|
+
var GIT_TIMEOUT_MS = 1e3;
|
|
926
|
+
var DEFAULT_GITHUB_ORIGIN_CACHE_MAX_ENTRIES = 512;
|
|
927
|
+
function validCwd(cwd) {
|
|
928
|
+
const trimmed = cwd.trim();
|
|
929
|
+
return trimmed === cwd && trimmed.length > 0 && !new RegExp("\\p{Cc}", "u").test(trimmed) && Buffer.byteLength(trimmed, "utf8") <= MAX_CWD_BYTES2;
|
|
930
|
+
}
|
|
931
|
+
var runReadOnlyGit = (cwd, args) => {
|
|
932
|
+
if (!validCwd(cwd)) return null;
|
|
933
|
+
try {
|
|
934
|
+
return execFileSync("git", ["-C", cwd, ...args], {
|
|
935
|
+
encoding: "utf8",
|
|
936
|
+
maxBuffer: GIT_OUTPUT_BYTES,
|
|
937
|
+
timeout: GIT_TIMEOUT_MS,
|
|
938
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
939
|
+
windowsHide: true
|
|
940
|
+
}).trim() || null;
|
|
941
|
+
} catch {
|
|
942
|
+
return null;
|
|
943
|
+
}
|
|
944
|
+
};
|
|
945
|
+
function resolveGitHubOrigin(cwd, runGit = runReadOnlyGit) {
|
|
946
|
+
const remote = runGit(cwd, ["remote", "get-url", "origin"]);
|
|
947
|
+
const repository = remote ? canonicalGitHubRepository(remote) : null;
|
|
948
|
+
if (!repository) return null;
|
|
949
|
+
const branchResult = runGit(cwd, ["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
950
|
+
const branch = branchResult === "HEAD" ? null : branchResult;
|
|
951
|
+
const commitHash = runGit(cwd, ["rev-parse", "HEAD"]);
|
|
952
|
+
return {
|
|
953
|
+
repository,
|
|
954
|
+
...branch ? { branch } : {},
|
|
955
|
+
...commitHash ? { commitHash } : {}
|
|
956
|
+
};
|
|
957
|
+
}
|
|
958
|
+
function createMeasuredGitHubOriginResolver(resolve = (cwd) => resolveGitHubOrigin(cwd), nowMs = () => performance.now(), options = {}) {
|
|
959
|
+
const cache = /* @__PURE__ */ new Map();
|
|
960
|
+
const maxCacheEntries = options.maxCacheEntries ?? DEFAULT_GITHUB_ORIGIN_CACHE_MAX_ENTRIES;
|
|
961
|
+
const maxLookups = options.maxLookups ?? Number.POSITIVE_INFINITY;
|
|
962
|
+
const maxElapsedMs = options.maxElapsedMs ?? Number.POSITIVE_INFINITY;
|
|
963
|
+
let attempted = 0;
|
|
964
|
+
let resolved = 0;
|
|
965
|
+
let unresolved = 0;
|
|
966
|
+
let cacheHits = 0;
|
|
967
|
+
let budgetSkipped = 0;
|
|
968
|
+
let elapsedMs = 0;
|
|
969
|
+
let slowestMs = 0;
|
|
970
|
+
const writeLocal = (cwd, origin) => {
|
|
971
|
+
if (maxCacheEntries <= 0) return;
|
|
972
|
+
cache.delete(cwd);
|
|
973
|
+
cache.set(cwd, origin);
|
|
974
|
+
while (cache.size > maxCacheEntries) {
|
|
975
|
+
const oldest = cache.keys().next().value;
|
|
976
|
+
if (oldest === void 0) break;
|
|
977
|
+
cache.delete(oldest);
|
|
978
|
+
}
|
|
979
|
+
};
|
|
980
|
+
const measuredResolve = (cwd) => {
|
|
981
|
+
if (cache.has(cwd)) {
|
|
982
|
+
cacheHits += 1;
|
|
983
|
+
const origin = cache.get(cwd) ?? null;
|
|
984
|
+
writeLocal(cwd, origin);
|
|
985
|
+
return origin;
|
|
986
|
+
}
|
|
987
|
+
const lookupStartedAtMs = nowMs();
|
|
988
|
+
if (attempted >= maxLookups || elapsedMs >= maxElapsedMs) {
|
|
989
|
+
budgetSkipped += 1;
|
|
990
|
+
writeLocal(cwd, null);
|
|
991
|
+
return null;
|
|
992
|
+
}
|
|
993
|
+
attempted += 1;
|
|
994
|
+
try {
|
|
995
|
+
const origin = resolve(cwd);
|
|
996
|
+
writeLocal(cwd, origin);
|
|
997
|
+
if (origin) resolved += 1;
|
|
998
|
+
else unresolved += 1;
|
|
999
|
+
return origin;
|
|
1000
|
+
} catch (error) {
|
|
1001
|
+
unresolved += 1;
|
|
1002
|
+
throw error;
|
|
1003
|
+
} finally {
|
|
1004
|
+
const durationMs = nowMs() - lookupStartedAtMs;
|
|
1005
|
+
if (Number.isFinite(durationMs) && durationMs > 0) {
|
|
1006
|
+
elapsedMs += durationMs;
|
|
1007
|
+
slowestMs = Math.max(slowestMs, durationMs);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
};
|
|
1011
|
+
return {
|
|
1012
|
+
resolve: measuredResolve,
|
|
1013
|
+
snapshot: () => ({
|
|
1014
|
+
attempted,
|
|
1015
|
+
resolved,
|
|
1016
|
+
unresolved,
|
|
1017
|
+
cacheHits,
|
|
1018
|
+
budgetSkipped,
|
|
1019
|
+
elapsedMs,
|
|
1020
|
+
slowestMs
|
|
1021
|
+
})
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
function createArtifactDiscoveryGitHubOriginResolver() {
|
|
1025
|
+
return createMeasuredGitHubOriginResolver(
|
|
1026
|
+
(cwd) => resolveGitHubOrigin(cwd),
|
|
1027
|
+
() => performance.now()
|
|
1028
|
+
);
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
// ../../packages/panopticon/src/artifacts.ts
|
|
1032
|
+
var RAW_ARTIFACT_ENDPOINT = "/v1/ingest/raw-artifact";
|
|
1033
|
+
var RAW_ARTIFACT_SCHEMA_VERSION = 4;
|
|
1034
|
+
var RAW_ARTIFACT_IDENTITY_SCHEMA_VERSION = 1;
|
|
1035
|
+
var DEFAULT_MAX_ARTIFACT_BYTES = 50 * 1024 * 1024;
|
|
1036
|
+
var DEFAULT_UPLOAD_CHUNK_BYTES = 1024 * 1024;
|
|
1037
|
+
var MAX_UPLOAD_CHUNK_BYTES = 5 * 1024 * 1024;
|
|
1038
|
+
var DEFAULT_UPLOAD_LIMIT = 10;
|
|
1039
|
+
var DEFAULT_UPLOAD_BYTE_LIMIT = DEFAULT_UPLOAD_CHUNK_BYTES * DEFAULT_UPLOAD_LIMIT;
|
|
1040
|
+
var MAX_UPLOAD_ATTEMPTS = 8;
|
|
1041
|
+
var RETRY_BACKOFF_BASE_MS = 3e4;
|
|
1042
|
+
var RETRY_BACKOFF_MAX_MS = 30 * 60 * 1e3;
|
|
1043
|
+
var RATE_LIMIT_RETRY_FALLBACK_MS = 6e4;
|
|
1044
|
+
var MAX_RATE_LIMIT_RETRY_MS = 24 * 60 * 60 * 1e3;
|
|
1045
|
+
var MIN_RATE_LIMIT_RETRY_MS = 1e3;
|
|
1046
|
+
var RAW_ARTIFACT_POST_TIMEOUT_MS = 6e4;
|
|
1047
|
+
var QUEUE_LOCK_STALE_MS = 15 * 60 * 1e3;
|
|
1048
|
+
var RAW_ARTIFACT_REPOSITORY_CONTEXT_SCHEMA_VERSION = 1;
|
|
1049
|
+
var MAX_RAW_ARTIFACT_REPOSITORY_OBSERVATIONS = 32;
|
|
1050
|
+
var MAX_RAW_ARTIFACT_REPOSITORY_CONTEXT_BYTES = 8 * 1024;
|
|
1051
|
+
var MAX_RAW_ARTIFACT_NESTED_METADATA_BYTES = 16 * 1024;
|
|
1052
|
+
function artifactSourceEnabled(source) {
|
|
1053
|
+
return source !== "hermes";
|
|
1054
|
+
}
|
|
1055
|
+
var RawArtifactPostError = class extends Error {
|
|
1056
|
+
status;
|
|
1057
|
+
code;
|
|
1058
|
+
retryAfter;
|
|
1059
|
+
constructor(message, status, code, retryAfter) {
|
|
1060
|
+
super(message);
|
|
1061
|
+
this.name = "RawArtifactPostError";
|
|
1062
|
+
this.status = status;
|
|
1063
|
+
this.code = code;
|
|
1064
|
+
this.retryAfter = retryAfter;
|
|
1065
|
+
}
|
|
1066
|
+
};
|
|
1067
|
+
function isAbortError(error) {
|
|
1068
|
+
return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
|
|
1069
|
+
}
|
|
1070
|
+
function safeStat(filePath) {
|
|
1071
|
+
try {
|
|
1072
|
+
return fs7.statSync(filePath);
|
|
1073
|
+
} catch {
|
|
1074
|
+
return null;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
function normalizeFilePath2(filePath) {
|
|
1078
|
+
return path17.resolve(filePath);
|
|
1079
|
+
}
|
|
1080
|
+
function artifactReadPath(artifact) {
|
|
1081
|
+
return normalizeFilePath2(artifact.readPath ?? artifact.sourcePath);
|
|
1082
|
+
}
|
|
1083
|
+
function statCandidateSource(sourcePath, transientStatErrors) {
|
|
1084
|
+
try {
|
|
1085
|
+
return fs7.statSync(sourcePath);
|
|
1086
|
+
} catch (error) {
|
|
1087
|
+
if (error.code === "ENOENT") return null;
|
|
1088
|
+
if (transientStatErrors === "throw") throw error;
|
|
1089
|
+
return null;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
function contentTypeForPath(filePath) {
|
|
1093
|
+
const ext = path17.extname(filePath).toLowerCase();
|
|
1094
|
+
if (ext === ".json" || ext === ".jsonl") return "application/json";
|
|
1095
|
+
if (ext === ".md") return "text/markdown";
|
|
1096
|
+
if (ext === ".toml") return "application/toml";
|
|
1097
|
+
if (ext === ".yaml" || ext === ".yml") return "application/yaml";
|
|
1098
|
+
if (ext === ".js" || ext === ".mjs" || ext === ".cjs") {
|
|
1099
|
+
return "text/javascript";
|
|
1100
|
+
}
|
|
1101
|
+
if (ext === ".ts" || ext === ".tsx") return "text/typescript";
|
|
1102
|
+
return "text/plain";
|
|
1103
|
+
}
|
|
1104
|
+
function buildArtifactKey(candidate) {
|
|
1105
|
+
return `${candidate.artifactType}:${candidate.source}:${normalizeFilePath2(candidate.sourcePath)}`;
|
|
1106
|
+
}
|
|
1107
|
+
function buildArtifactId(args) {
|
|
1108
|
+
return `raw_artifact:${sha256Hex(
|
|
1109
|
+
`${args.artifactType}\0${args.source}\0${normalizeFilePath2(args.sourcePath)}`
|
|
1110
|
+
)}`;
|
|
1111
|
+
}
|
|
1112
|
+
function buildArtifactGenerationId(args) {
|
|
1113
|
+
return `raw_artifact_generation:${sha256Hex(
|
|
1114
|
+
`${args.artifactId}\0${args.generationStartedAtMs}`
|
|
1115
|
+
)}`;
|
|
1116
|
+
}
|
|
1117
|
+
function buildArtifactChunkId(args) {
|
|
1118
|
+
return `raw_artifact_chunk:${sha256Hex(
|
|
1119
|
+
`${args.artifactId}\0${args.artifactGenerationId}\0${args.chunkStartByte}\0${args.chunkEndByte}\0${args.chunkSha256}`
|
|
1120
|
+
)}`;
|
|
1121
|
+
}
|
|
1122
|
+
function discoverArtifacts(options = {}) {
|
|
1123
|
+
const repositoryResolution = createArtifactDiscoveryGitHubOriginResolver();
|
|
1124
|
+
const env = options.env ?? process.env;
|
|
1125
|
+
const homeDirInput = options.homeDir ?? env.HOME ?? os3.homedir();
|
|
1126
|
+
if (!homeDirInput) {
|
|
1127
|
+
return {
|
|
1128
|
+
candidates: [],
|
|
1129
|
+
oversized: [],
|
|
1130
|
+
repositoryResolution: repositoryResolution.snapshot()
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
const homeDir = normalizeFilePath2(homeDirInput);
|
|
1134
|
+
const candidates = discoverTargetArtifactCandidates({
|
|
1135
|
+
homeDir,
|
|
1136
|
+
env,
|
|
1137
|
+
resolveGitHubOrigin: repositoryResolution.resolve,
|
|
1138
|
+
...options.materializeArtifact ? { materializeArtifact: options.materializeArtifact } : {}
|
|
1139
|
+
});
|
|
1140
|
+
const maxBytes = options.maxFileBytes ?? DEFAULT_MAX_ARTIFACT_BYTES;
|
|
1141
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1142
|
+
const oversizedByKey = /* @__PURE__ */ new Map();
|
|
1143
|
+
for (const candidate of candidates) {
|
|
1144
|
+
const stat = safeStat(artifactReadPath(candidate));
|
|
1145
|
+
if (!stat?.isFile()) continue;
|
|
1146
|
+
if (stat.size > maxBytes) {
|
|
1147
|
+
oversizedByKey.set(buildArtifactKey(candidate), {
|
|
1148
|
+
artifactType: candidate.artifactType,
|
|
1149
|
+
source: candidate.source,
|
|
1150
|
+
sourcePath: candidate.sourcePath,
|
|
1151
|
+
sizeBytes: stat.size,
|
|
1152
|
+
maxFileBytes: maxBytes
|
|
1153
|
+
});
|
|
1154
|
+
continue;
|
|
1155
|
+
}
|
|
1156
|
+
byKey.set(buildArtifactKey(candidate), candidate);
|
|
1157
|
+
}
|
|
1158
|
+
return {
|
|
1159
|
+
candidates: [...byKey.values()],
|
|
1160
|
+
oversized: [...oversizedByKey.values()],
|
|
1161
|
+
repositoryResolution: repositoryResolution.snapshot()
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
function emptyQueue() {
|
|
1165
|
+
return { schemaVersion: RAW_ARTIFACT_SCHEMA_VERSION, artifacts: [] };
|
|
1166
|
+
}
|
|
1167
|
+
function isRawArtifactUploadMetadata(value) {
|
|
1168
|
+
if (!value || typeof value !== "object") return false;
|
|
1169
|
+
const metadata = value;
|
|
1170
|
+
return typeof metadata.artifactId === "string" && typeof metadata.artifactGenerationId === "string" && typeof metadata.generationStartedAtMs === "number" && typeof metadata.artifactChunkId === "string" && typeof metadata.artifactKey === "string" && typeof metadata.artifactType === "string" && typeof metadata.source === "string" && typeof metadata.sourcePath === "string" && typeof metadata.contentType === "string" && metadata.contentEncoding === "identity" && typeof metadata.sizeBytes === "number" && typeof metadata.chunkStartByte === "number" && typeof metadata.chunkEndByte === "number" && typeof metadata.chunkSizeBytes === "number" && typeof metadata.chunkSha256 === "string" && typeof metadata.firstSeenAtMs === "number" && typeof metadata.lastSeenAtMs === "number" && !!metadata.metadata && typeof metadata.metadata === "object";
|
|
1171
|
+
}
|
|
1172
|
+
function isRawArtifactInFlightChunk(value) {
|
|
1173
|
+
if (!value || typeof value !== "object") return false;
|
|
1174
|
+
const chunk = value;
|
|
1175
|
+
return typeof chunk.uploadDestinationId === "string" && isRawArtifactUploadMetadata(chunk.metadata);
|
|
1176
|
+
}
|
|
1177
|
+
function isUploadedChunkCheckpoint(value) {
|
|
1178
|
+
if (!value || typeof value !== "object") return false;
|
|
1179
|
+
const checkpoint = value;
|
|
1180
|
+
return typeof checkpoint.chunkStartByte === "number" && typeof checkpoint.chunkEndByte === "number" && typeof checkpoint.chunkSha256 === "string";
|
|
1181
|
+
}
|
|
1182
|
+
function isQueueEntry(value) {
|
|
1183
|
+
if (!value || typeof value !== "object") return false;
|
|
1184
|
+
const entry = value;
|
|
1185
|
+
return typeof entry.artifactId === "string" && typeof entry.artifactGenerationId === "string" && typeof entry.generationStartedAtMs === "number" && typeof entry.artifactKey === "string" && typeof entry.artifactType === "string" && typeof entry.source === "string" && typeof entry.sourcePath === "string" && (entry.identityKey === void 0 || typeof entry.identityKey === "string") && (entry.readPath === void 0 || typeof entry.readPath === "string") && entry.contentEncoding === "identity" && typeof entry.sizeBytes === "number" && typeof entry.uploadedSizeBytes === "number" && (entry.uploadedRepositoryContextRevision === void 0 || typeof entry.uploadedRepositoryContextRevision === "number" && Number.isSafeInteger(entry.uploadedRepositoryContextRevision) && entry.uploadedRepositoryContextRevision >= 1) && (entry.uploadedIdentitySignature === void 0 || typeof entry.uploadedIdentitySignature === "string") && typeof entry.uploadAttempts === "number" && typeof entry.firstSeenAtMs === "number" && typeof entry.lastSeenAtMs === "number" && (entry.uploadDestinationId === void 0 || typeof entry.uploadDestinationId === "string") && (entry.inFlightChunk === void 0 || isRawArtifactInFlightChunk(entry.inFlightChunk)) && (entry.uploadedChunkCheckpoints === void 0 || Array.isArray(entry.uploadedChunkCheckpoints) && entry.uploadedChunkCheckpoints.every(isUploadedChunkCheckpoint)) && (entry.status === "pending" || entry.status === "uploaded" || entry.status === "failed");
|
|
1186
|
+
}
|
|
1187
|
+
function isRawArtifactUploadThrottle(value) {
|
|
1188
|
+
if (!value || typeof value !== "object") return false;
|
|
1189
|
+
const throttle = value;
|
|
1190
|
+
return typeof throttle.retryAtMs === "number" && Number.isSafeInteger(throttle.retryAtMs) && throttle.retryAtMs >= 0 && throttle.status === 429 && (throttle.code === void 0 || typeof throttle.code === "string");
|
|
1191
|
+
}
|
|
1192
|
+
function readUploadThrottles(value) {
|
|
1193
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1194
|
+
return void 0;
|
|
1195
|
+
}
|
|
1196
|
+
const entries = Object.entries(value).filter(
|
|
1197
|
+
([, throttle]) => isRawArtifactUploadThrottle(throttle)
|
|
1198
|
+
);
|
|
1199
|
+
return entries.length > 0 ? Object.fromEntries(entries) : void 0;
|
|
1200
|
+
}
|
|
1201
|
+
function readArtifactQueue(queueFile) {
|
|
1202
|
+
try {
|
|
1203
|
+
const parsed = JSON.parse(fs7.readFileSync(queueFile, "utf8"));
|
|
1204
|
+
if (!parsed || typeof parsed !== "object") return emptyQueue();
|
|
1205
|
+
const record = parsed;
|
|
1206
|
+
if (record.schemaVersion !== RAW_ARTIFACT_SCHEMA_VERSION) {
|
|
1207
|
+
return emptyQueue();
|
|
1208
|
+
}
|
|
1209
|
+
if (!Array.isArray(record.artifacts)) return emptyQueue();
|
|
1210
|
+
return {
|
|
1211
|
+
schemaVersion: RAW_ARTIFACT_SCHEMA_VERSION,
|
|
1212
|
+
artifacts: record.artifacts.filter(isQueueEntry),
|
|
1213
|
+
uploadThrottles: readUploadThrottles(record.uploadThrottles)
|
|
1214
|
+
};
|
|
1215
|
+
} catch {
|
|
1216
|
+
return emptyQueue();
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
function writeArtifactQueue(queueFile, queue) {
|
|
1220
|
+
fs7.mkdirSync(path17.dirname(queueFile), { recursive: true, mode: 448 });
|
|
1221
|
+
const tempFile = `${queueFile}.${process.pid}.${Date.now()}.tmp`;
|
|
1222
|
+
fs7.writeFileSync(tempFile, `${JSON.stringify(queue, null, 2)}
|
|
1223
|
+
`, {
|
|
1224
|
+
mode: 384
|
|
1225
|
+
});
|
|
1226
|
+
fs7.renameSync(tempFile, queueFile);
|
|
1227
|
+
try {
|
|
1228
|
+
fs7.chmodSync(queueFile, 384);
|
|
1229
|
+
} catch {
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
function statMtimeMs(stat) {
|
|
1233
|
+
return Math.floor(stat.mtimeMs);
|
|
1234
|
+
}
|
|
1235
|
+
function buildInitialGenerationId(input) {
|
|
1236
|
+
return buildArtifactGenerationId({
|
|
1237
|
+
artifactId: input.artifactId,
|
|
1238
|
+
generationStartedAtMs: input.nowMs
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
function nextGenerationStartedAtMs(nowMs, existing) {
|
|
1242
|
+
return existing ? Math.max(nowMs, existing.generationStartedAtMs + 1) : nowMs;
|
|
1243
|
+
}
|
|
1244
|
+
function shouldStartNewGeneration(input) {
|
|
1245
|
+
if (input.nextSizeBytes < input.existing.uploadedSizeBytes) return true;
|
|
1246
|
+
if (input.nextSizeBytes < input.existing.sizeBytes) return true;
|
|
1247
|
+
if ((input.nextReadPath !== artifactReadPath(input.existing) || input.existing.readPath !== void 0) && input.nextSizeBytes === input.existing.sizeBytes && input.existing.uploadedSizeBytes >= input.nextSizeBytes && input.existing.contentSha256) {
|
|
1248
|
+
try {
|
|
1249
|
+
const contentSha256 = createHash4("sha256").update(fs7.readFileSync(input.nextReadPath)).digest("hex");
|
|
1250
|
+
if (contentSha256 === input.existing.contentSha256) return false;
|
|
1251
|
+
} catch {
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
return input.nextSizeBytes === input.existing.sizeBytes && input.nextMtimeMs !== input.existing.mtimeMs && input.existing.uploadedSizeBytes >= input.nextSizeBytes;
|
|
1255
|
+
}
|
|
1256
|
+
function optionalBoundedMetadataString(value, maxBytes) {
|
|
1257
|
+
return typeof value === "string" && value.length > 0 && value.trim() === value && !new RegExp("\\p{Cc}", "u").test(value) && Buffer.byteLength(value, "utf8") <= maxBytes;
|
|
1258
|
+
}
|
|
1259
|
+
function isRepositoryObservation(value) {
|
|
1260
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1261
|
+
const observation = value;
|
|
1262
|
+
const repository = observation.repository;
|
|
1263
|
+
const repositoryName = typeof repository === "string" ? repository.split("/")[1] : void 0;
|
|
1264
|
+
return typeof observation.id === "string" && /^repo_observation:[0-9a-f]{64}$/.test(observation.id) && typeof repository === "string" && /^[a-z0-9_.-]+\/[a-z0-9_.-]+$/.test(repository) && !repository.startsWith("./") && !repository.startsWith("../") && repositoryName !== "." && repositoryName !== ".." && !repositoryName?.endsWith(".git") && (observation.source === "codex_session_meta" || observation.source === "git_origin") && typeof observation.observedAtMs === "number" && Number.isSafeInteger(observation.observedAtMs) && observation.observedAtMs >= 0 && (observation.cwd === void 0 || optionalBoundedMetadataString(observation.cwd, 4096)) && (observation.branch === void 0 || optionalBoundedMetadataString(observation.branch, 1024)) && (observation.commitHash === void 0 || typeof observation.commitHash === "string" && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(observation.commitHash));
|
|
1265
|
+
}
|
|
1266
|
+
function repositoryContextFromMetadata(metadata, artifactGenerationId) {
|
|
1267
|
+
const value = metadata?.repositoryContext;
|
|
1268
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
1269
|
+
const context = value;
|
|
1270
|
+
if (context.schemaVersion !== RAW_ARTIFACT_REPOSITORY_CONTEXT_SCHEMA_VERSION || context.artifactGenerationId !== artifactGenerationId || typeof context.revision !== "number" || !Number.isSafeInteger(context.revision) || context.revision < 1 || !Array.isArray(context.observations) || context.observations.length < 1 || context.observations.length > MAX_RAW_ARTIFACT_REPOSITORY_OBSERVATIONS || !context.observations.every(isRepositoryObservation)) {
|
|
1271
|
+
return null;
|
|
1272
|
+
}
|
|
1273
|
+
const observations = [...context.observations].sort(
|
|
1274
|
+
(left, right) => left.id.localeCompare(right.id)
|
|
1275
|
+
);
|
|
1276
|
+
if (new Set(observations.map((observation) => observation.id)).size !== observations.length) {
|
|
1277
|
+
return null;
|
|
1278
|
+
}
|
|
1279
|
+
return {
|
|
1280
|
+
schemaVersion: RAW_ARTIFACT_REPOSITORY_CONTEXT_SCHEMA_VERSION,
|
|
1281
|
+
artifactGenerationId,
|
|
1282
|
+
revision: context.revision,
|
|
1283
|
+
observations
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
function repositoryContextRevision(metadata, artifactGenerationId) {
|
|
1287
|
+
return repositoryContextFromMetadata(metadata, artifactGenerationId)?.revision ?? 0;
|
|
1288
|
+
}
|
|
1289
|
+
function contextFitsMetadataBudget(context, metadataWithoutContext) {
|
|
1290
|
+
return Buffer.byteLength(JSON.stringify(context), "utf8") <= MAX_RAW_ARTIFACT_REPOSITORY_CONTEXT_BYTES && Buffer.byteLength(
|
|
1291
|
+
JSON.stringify({ ...metadataWithoutContext, repositoryContext: context }),
|
|
1292
|
+
"utf8"
|
|
1293
|
+
) <= MAX_RAW_ARTIFACT_NESTED_METADATA_BYTES;
|
|
1294
|
+
}
|
|
1295
|
+
function mergeRepositoryContext(input) {
|
|
1296
|
+
const byId = new Map(
|
|
1297
|
+
(input.previous?.observations ?? []).map((observation) => [
|
|
1298
|
+
observation.id,
|
|
1299
|
+
observation
|
|
1300
|
+
])
|
|
1301
|
+
);
|
|
1302
|
+
let added = false;
|
|
1303
|
+
const candidates = (input.observations ?? []).filter(isRepositoryObservation).sort((left, right) => left.id.localeCompare(right.id));
|
|
1304
|
+
if (!input.previous && candidates.length === 0) return null;
|
|
1305
|
+
for (const observation of candidates) {
|
|
1306
|
+
if (byId.has(observation.id) || byId.size >= MAX_RAW_ARTIFACT_REPOSITORY_OBSERVATIONS) {
|
|
1307
|
+
continue;
|
|
1308
|
+
}
|
|
1309
|
+
const nextObservations = [...byId.values(), observation].sort(
|
|
1310
|
+
(left, right) => left.id.localeCompare(right.id)
|
|
1311
|
+
);
|
|
1312
|
+
const nextContext = {
|
|
1313
|
+
schemaVersion: RAW_ARTIFACT_REPOSITORY_CONTEXT_SCHEMA_VERSION,
|
|
1314
|
+
artifactGenerationId: input.artifactGenerationId,
|
|
1315
|
+
revision: input.previous ? input.previous.revision + 1 : 1,
|
|
1316
|
+
observations: nextObservations
|
|
1317
|
+
};
|
|
1318
|
+
if (!contextFitsMetadataBudget(nextContext, input.metadataWithoutContext)) {
|
|
1319
|
+
continue;
|
|
1320
|
+
}
|
|
1321
|
+
byId.set(observation.id, observation);
|
|
1322
|
+
added = true;
|
|
1323
|
+
}
|
|
1324
|
+
if (byId.size === 0) return null;
|
|
1325
|
+
return {
|
|
1326
|
+
schemaVersion: RAW_ARTIFACT_REPOSITORY_CONTEXT_SCHEMA_VERSION,
|
|
1327
|
+
artifactGenerationId: input.artifactGenerationId,
|
|
1328
|
+
revision: input.previous ? input.previous.revision + (added ? 1 : 0) : 1,
|
|
1329
|
+
observations: [...byId.values()].sort(
|
|
1330
|
+
(left, right) => left.id.localeCompare(right.id)
|
|
1331
|
+
)
|
|
1332
|
+
};
|
|
1333
|
+
}
|
|
1334
|
+
function withoutRepositoryContext(metadata) {
|
|
1335
|
+
if (!metadata) return {};
|
|
1336
|
+
return Object.fromEntries(
|
|
1337
|
+
Object.entries(metadata).filter(([key]) => key !== "repositoryContext")
|
|
1338
|
+
);
|
|
1339
|
+
}
|
|
1340
|
+
function metadataForGeneration(input) {
|
|
1341
|
+
const previousContext = input.inheritPrevious ? repositoryContextFromMetadata(
|
|
1342
|
+
input.previousMetadata,
|
|
1343
|
+
input.artifactGenerationId
|
|
1344
|
+
) : null;
|
|
1345
|
+
const metadataWithoutContext = {
|
|
1346
|
+
...input.inheritPrevious ? withoutRepositoryContext(input.previousMetadata) : {},
|
|
1347
|
+
originalPath: input.sourcePath,
|
|
1348
|
+
...withoutRepositoryContext(input.candidateMetadata),
|
|
1349
|
+
// Identity is refreshed on every snapshot; when unavailable this pass, an
|
|
1350
|
+
// inherited value from a previous generation snapshot is kept as-is.
|
|
1351
|
+
...input.observerIdentity ? { observer: input.observerIdentity } : {}
|
|
1352
|
+
};
|
|
1353
|
+
const hasInstallIdentity = metadataWithoutContext.observer !== void 0 || metadataWithoutContext.harnessInstallId !== void 0;
|
|
1354
|
+
const repositoryContext = mergeRepositoryContext({
|
|
1355
|
+
artifactGenerationId: input.artifactGenerationId,
|
|
1356
|
+
previous: previousContext,
|
|
1357
|
+
observations: input.candidateObservations,
|
|
1358
|
+
metadataWithoutContext
|
|
1359
|
+
});
|
|
1360
|
+
return {
|
|
1361
|
+
...metadataWithoutContext,
|
|
1362
|
+
...hasInstallIdentity ? {
|
|
1363
|
+
installIdentitySchemaVersion: RAW_ARTIFACT_IDENTITY_SCHEMA_VERSION
|
|
1364
|
+
} : {},
|
|
1365
|
+
...repositoryContext ? { repositoryContext } : {}
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
function snapshotCandidate(candidate, existing, nowMs, maxFileBytes, transientStatErrors = "ignore", observerIdentity) {
|
|
1369
|
+
const sourcePath = normalizeFilePath2(candidate.sourcePath);
|
|
1370
|
+
const readPath = artifactReadPath(candidate);
|
|
1371
|
+
const stat = statCandidateSource(readPath, transientStatErrors);
|
|
1372
|
+
if (!stat?.isFile() || stat.size > maxFileBytes) return null;
|
|
1373
|
+
const artifactType = candidate.artifactType;
|
|
1374
|
+
const source = candidate.source;
|
|
1375
|
+
const artifactKey = existing?.artifactKey ?? buildArtifactKey({ ...candidate, sourcePath });
|
|
1376
|
+
const artifactId = existing?.artifactId ?? buildArtifactId({ artifactType, source, sourcePath });
|
|
1377
|
+
const identityKey = candidate.identityKey ?? existing?.identityKey;
|
|
1378
|
+
const sizeBytes = stat.size;
|
|
1379
|
+
const mtimeMs = statMtimeMs(stat);
|
|
1380
|
+
if (existing && !shouldStartNewGeneration({
|
|
1381
|
+
existing,
|
|
1382
|
+
nextReadPath: readPath,
|
|
1383
|
+
nextSizeBytes: sizeBytes,
|
|
1384
|
+
nextMtimeMs: mtimeMs
|
|
1385
|
+
})) {
|
|
1386
|
+
const metadata2 = metadataForGeneration({
|
|
1387
|
+
sourcePath,
|
|
1388
|
+
artifactGenerationId: existing.artifactGenerationId,
|
|
1389
|
+
candidateMetadata: candidate.metadata,
|
|
1390
|
+
candidateObservations: candidate.repositoryObservations,
|
|
1391
|
+
previousMetadata: existing.metadata,
|
|
1392
|
+
inheritPrevious: true,
|
|
1393
|
+
observerIdentity
|
|
1394
|
+
});
|
|
1395
|
+
const previousContextRevision = repositoryContextRevision(
|
|
1396
|
+
existing.metadata,
|
|
1397
|
+
existing.artifactGenerationId
|
|
1398
|
+
);
|
|
1399
|
+
const nextContextRevision = repositoryContextRevision(
|
|
1400
|
+
metadata2,
|
|
1401
|
+
existing.artifactGenerationId
|
|
1402
|
+
);
|
|
1403
|
+
const contextAdvanced = nextContextRevision > previousContextRevision;
|
|
1404
|
+
const contextNeedsUpload = nextContextRevision > (existing.uploadedRepositoryContextRevision ?? 0);
|
|
1405
|
+
const nextIdentitySignature = identitySignature(metadata2);
|
|
1406
|
+
const identityAdvanced = nextIdentitySignature !== void 0 && nextIdentitySignature !== identitySignature(existing.metadata);
|
|
1407
|
+
const identityUnacknowledged = nextIdentitySignature !== void 0 && nextIdentitySignature !== existing.uploadedIdentitySignature;
|
|
1408
|
+
const amendmentNeedsUpload = contextNeedsUpload || identityUnacknowledged;
|
|
1409
|
+
const amendmentAdvanced = contextAdvanced || identityAdvanced;
|
|
1410
|
+
const fullyUploaded = existing.uploadedSizeBytes >= sizeBytes;
|
|
1411
|
+
const grew = sizeBytes > existing.sizeBytes;
|
|
1412
|
+
const hasNewWork = grew || amendmentAdvanced;
|
|
1413
|
+
const status = fullyUploaded ? amendmentNeedsUpload ? amendmentAdvanced ? "pending" : existing.status === "failed" ? "failed" : "pending" : "uploaded" : grew ? "pending" : existing.status === "failed" ? "failed" : "pending";
|
|
1414
|
+
return {
|
|
1415
|
+
...existing,
|
|
1416
|
+
sourcePath,
|
|
1417
|
+
...identityKey ? { identityKey } : {},
|
|
1418
|
+
// Always taken from the candidate: a stale decoded copy paired with a
|
|
1419
|
+
// freshly discovered plain source would read the wrong bytes.
|
|
1420
|
+
readPath: candidate.readPath ? readPath : void 0,
|
|
1421
|
+
sessionId: existing.sessionId ?? candidate.sessionId ?? null,
|
|
1422
|
+
sizeBytes,
|
|
1423
|
+
mtimeMs,
|
|
1424
|
+
lastSeenAtMs: nowMs,
|
|
1425
|
+
metadata: metadata2,
|
|
1426
|
+
status,
|
|
1427
|
+
// A stored contentSha256 is only valid for the sizeBytes it was
|
|
1428
|
+
// computed at (the completed upload). Once the file grows past that,
|
|
1429
|
+
// drop it so intermediate chunks never carry a stale full-file hash.
|
|
1430
|
+
contentSha256: fullyUploaded ? existing.contentSha256 : void 0,
|
|
1431
|
+
uploadedAtMs: fullyUploaded ? existing.uploadedAtMs : void 0,
|
|
1432
|
+
// Appends are the normal transcript lifecycle. New bytes are meaningful
|
|
1433
|
+
// work and reset both an exhausted retry budget and its backoff clock.
|
|
1434
|
+
uploadAttempts: hasNewWork ? 0 : existing.uploadAttempts,
|
|
1435
|
+
lastAttemptAtMs: hasNewWork ? void 0 : existing.lastAttemptAtMs,
|
|
1436
|
+
lastError: hasNewWork || status === "pending" ? void 0 : existing.lastError
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
const generationStartedAtMs = nextGenerationStartedAtMs(nowMs, existing);
|
|
1440
|
+
const artifactGenerationId = buildInitialGenerationId({
|
|
1441
|
+
artifactId,
|
|
1442
|
+
nowMs: generationStartedAtMs
|
|
1443
|
+
});
|
|
1444
|
+
const metadata = metadataForGeneration({
|
|
1445
|
+
sourcePath,
|
|
1446
|
+
artifactGenerationId,
|
|
1447
|
+
candidateMetadata: candidate.metadata,
|
|
1448
|
+
candidateObservations: candidate.repositoryObservations,
|
|
1449
|
+
inheritPrevious: false,
|
|
1450
|
+
observerIdentity
|
|
1451
|
+
});
|
|
1452
|
+
return {
|
|
1453
|
+
artifactId,
|
|
1454
|
+
artifactGenerationId,
|
|
1455
|
+
generationStartedAtMs,
|
|
1456
|
+
artifactKey,
|
|
1457
|
+
artifactType,
|
|
1458
|
+
source,
|
|
1459
|
+
sourcePath,
|
|
1460
|
+
...identityKey ? { identityKey } : {},
|
|
1461
|
+
...candidate.readPath ? { readPath } : {},
|
|
1462
|
+
sessionId: candidate.sessionId ?? null,
|
|
1463
|
+
contentType: contentTypeForPath(readPath),
|
|
1464
|
+
contentEncoding: "identity",
|
|
1465
|
+
sizeBytes,
|
|
1466
|
+
mtimeMs,
|
|
1467
|
+
firstSeenAtMs: existing?.firstSeenAtMs ?? nowMs,
|
|
1468
|
+
lastSeenAtMs: generationStartedAtMs,
|
|
1469
|
+
metadata,
|
|
1470
|
+
status: sizeBytes === 0 ? "uploaded" : "pending",
|
|
1471
|
+
uploadedSizeBytes: 0,
|
|
1472
|
+
uploadedRepositoryContextRevision: void 0,
|
|
1473
|
+
uploadedIdentitySignature: void 0,
|
|
1474
|
+
uploadedChunkCheckpoints: [],
|
|
1475
|
+
uploadAttempts: 0
|
|
1476
|
+
};
|
|
1477
|
+
}
|
|
1478
|
+
function snapshotArtifactsLocked(options, lock) {
|
|
1479
|
+
const nowMs = options.nowMs?.() ?? Date.now();
|
|
1480
|
+
const maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_ARTIFACT_BYTES;
|
|
1481
|
+
const queue = readArtifactQueue(options.queueFile);
|
|
1482
|
+
const entriesByKey = new Map(
|
|
1483
|
+
queue.artifacts.map((entry) => [entry.artifactKey, entry])
|
|
1484
|
+
);
|
|
1485
|
+
const entriesByIdentityKey = new Map(
|
|
1486
|
+
queue.artifacts.flatMap(
|
|
1487
|
+
(entry) => entry.identityKey ? [[entry.identityKey, entry]] : []
|
|
1488
|
+
)
|
|
1489
|
+
);
|
|
1490
|
+
let queued = 0;
|
|
1491
|
+
let skipped = 0;
|
|
1492
|
+
const oversizedByPath = /* @__PURE__ */ new Map();
|
|
1493
|
+
const recordOversized = (input) => {
|
|
1494
|
+
const stat = safeStat(input.sourcePath);
|
|
1495
|
+
if (!stat?.isFile() || stat.size <= maxFileBytes) return false;
|
|
1496
|
+
oversizedByPath.set(input.sourcePath, {
|
|
1497
|
+
artifactType: input.artifactType,
|
|
1498
|
+
source: input.source,
|
|
1499
|
+
sourcePath: input.sourcePath,
|
|
1500
|
+
sizeBytes: stat.size,
|
|
1501
|
+
maxFileBytes
|
|
1502
|
+
});
|
|
1503
|
+
return true;
|
|
1504
|
+
};
|
|
1505
|
+
let candidatesSeen = 0;
|
|
1506
|
+
for (const candidate of options.candidates) {
|
|
1507
|
+
candidatesSeen += 1;
|
|
1508
|
+
if (lock && candidatesSeen % 100 === 0 && !lock.heartbeat()) {
|
|
1509
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
1510
|
+
}
|
|
1511
|
+
const existing = (candidate.identityKey ? entriesByIdentityKey.get(candidate.identityKey) : void 0) ?? entriesByKey.get(buildArtifactKey(candidate));
|
|
1512
|
+
const snapshot = snapshotCandidate(
|
|
1513
|
+
candidate,
|
|
1514
|
+
existing,
|
|
1515
|
+
nowMs,
|
|
1516
|
+
maxFileBytes,
|
|
1517
|
+
"ignore",
|
|
1518
|
+
options.observerIdentity
|
|
1519
|
+
);
|
|
1520
|
+
if (!snapshot) {
|
|
1521
|
+
skipped += 1;
|
|
1522
|
+
recordOversized(candidate);
|
|
1523
|
+
continue;
|
|
1524
|
+
}
|
|
1525
|
+
queued += 1;
|
|
1526
|
+
entriesByKey.set(snapshot.artifactKey, snapshot);
|
|
1527
|
+
if (snapshot.identityKey) {
|
|
1528
|
+
entriesByIdentityKey.set(snapshot.identityKey, snapshot);
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
const artifacts = [...entriesByKey.values()].filter((entry) => {
|
|
1532
|
+
if (!artifactSourceEnabled(entry.source)) {
|
|
1533
|
+
skipped += 1;
|
|
1534
|
+
return false;
|
|
1535
|
+
}
|
|
1536
|
+
if (entry.readPath && !safeStat(entry.sourcePath)?.isFile()) {
|
|
1537
|
+
skipped += 1;
|
|
1538
|
+
return false;
|
|
1539
|
+
}
|
|
1540
|
+
if (entry.readPath && !safeStat(entry.readPath)?.isFile()) {
|
|
1541
|
+
return true;
|
|
1542
|
+
}
|
|
1543
|
+
const stat = safeStat(artifactReadPath(entry));
|
|
1544
|
+
if (!stat?.isFile() || stat.size > maxFileBytes) {
|
|
1545
|
+
skipped += 1;
|
|
1546
|
+
recordOversized(entry);
|
|
1547
|
+
return false;
|
|
1548
|
+
}
|
|
1549
|
+
return true;
|
|
1550
|
+
});
|
|
1551
|
+
const nextQueue = {
|
|
1552
|
+
schemaVersion: RAW_ARTIFACT_SCHEMA_VERSION,
|
|
1553
|
+
artifacts: artifacts.sort(
|
|
1554
|
+
(a, b) => a.artifactKey.localeCompare(b.artifactKey)
|
|
1555
|
+
),
|
|
1556
|
+
uploadThrottles: queue.uploadThrottles
|
|
1557
|
+
};
|
|
1558
|
+
if (lock && !lock.heartbeat()) {
|
|
1559
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
1560
|
+
}
|
|
1561
|
+
writeArtifactQueue(options.queueFile, nextQueue);
|
|
1562
|
+
return {
|
|
1563
|
+
queued,
|
|
1564
|
+
skipped,
|
|
1565
|
+
oversized: [...oversizedByPath.values()],
|
|
1566
|
+
queue: nextQueue
|
|
1567
|
+
};
|
|
1568
|
+
}
|
|
1569
|
+
function sourceRevisionFromStat(stat) {
|
|
1570
|
+
return {
|
|
1571
|
+
device: stat.dev,
|
|
1572
|
+
inode: stat.ino,
|
|
1573
|
+
sizeBytes: stat.size,
|
|
1574
|
+
mtimeNanoseconds: stat.mtimeNs,
|
|
1575
|
+
ctimeNanoseconds: stat.ctimeNs
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
function sameSourceRevision(left, right) {
|
|
1579
|
+
return left.device === right.device && left.inode === right.inode && left.sizeBytes === right.sizeBytes && left.mtimeNanoseconds === right.mtimeNanoseconds && left.ctimeNanoseconds === right.ctimeNanoseconds;
|
|
1580
|
+
}
|
|
1581
|
+
function safeSourceRevision(filePath) {
|
|
1582
|
+
try {
|
|
1583
|
+
const stat = fs7.statSync(filePath, { bigint: true });
|
|
1584
|
+
return stat.isFile() ? sourceRevisionFromStat(stat) : null;
|
|
1585
|
+
} catch {
|
|
1586
|
+
return null;
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
function readExactRange(fd, startByte, endByte) {
|
|
1590
|
+
const sizeBytes = endByte - startByte;
|
|
1591
|
+
if (sizeBytes < 0) return null;
|
|
1592
|
+
const buffer = Buffer.alloc(sizeBytes);
|
|
1593
|
+
let offset = 0;
|
|
1594
|
+
while (offset < sizeBytes) {
|
|
1595
|
+
const bytesRead = fs7.readSync(
|
|
1596
|
+
fd,
|
|
1597
|
+
buffer,
|
|
1598
|
+
offset,
|
|
1599
|
+
sizeBytes - offset,
|
|
1600
|
+
startByte + offset
|
|
1601
|
+
);
|
|
1602
|
+
if (bytesRead <= 0) return null;
|
|
1603
|
+
offset += bytesRead;
|
|
1604
|
+
}
|
|
1605
|
+
return buffer;
|
|
1606
|
+
}
|
|
1607
|
+
function readFileRangeAtRevision(filePath, startByte, endByte, expectedRevision) {
|
|
1608
|
+
const pathRevisionBefore = safeSourceRevision(filePath);
|
|
1609
|
+
if (!pathRevisionBefore || expectedRevision && !sameSourceRevision(pathRevisionBefore, expectedRevision)) {
|
|
1610
|
+
return null;
|
|
1611
|
+
}
|
|
1612
|
+
let fd;
|
|
1613
|
+
try {
|
|
1614
|
+
fd = fs7.openSync(filePath, "r");
|
|
1615
|
+
} catch {
|
|
1616
|
+
return null;
|
|
1617
|
+
}
|
|
1618
|
+
try {
|
|
1619
|
+
const openedRevision = sourceRevisionFromStat(
|
|
1620
|
+
fs7.fstatSync(fd, { bigint: true })
|
|
1621
|
+
);
|
|
1622
|
+
if (!sameSourceRevision(pathRevisionBefore, openedRevision)) return null;
|
|
1623
|
+
const bytes = readExactRange(fd, startByte, endByte);
|
|
1624
|
+
if (!bytes) return null;
|
|
1625
|
+
const revisionAfterRead = sourceRevisionFromStat(
|
|
1626
|
+
fs7.fstatSync(fd, { bigint: true })
|
|
1627
|
+
);
|
|
1628
|
+
const pathRevisionAfter = safeSourceRevision(filePath);
|
|
1629
|
+
if (!sameSourceRevision(openedRevision, revisionAfterRead) || !pathRevisionAfter || !sameSourceRevision(openedRevision, pathRevisionAfter)) {
|
|
1630
|
+
return null;
|
|
1631
|
+
}
|
|
1632
|
+
return { bytes, revision: openedRevision };
|
|
1633
|
+
} catch {
|
|
1634
|
+
return null;
|
|
1635
|
+
} finally {
|
|
1636
|
+
fs7.closeSync(fd);
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
function uploadMetadata(input) {
|
|
1640
|
+
const chunkSha256 = sha256Hex(input.chunk);
|
|
1641
|
+
return {
|
|
1642
|
+
artifactId: input.entry.artifactId,
|
|
1643
|
+
artifactGenerationId: input.entry.artifactGenerationId,
|
|
1644
|
+
generationStartedAtMs: input.entry.generationStartedAtMs,
|
|
1645
|
+
artifactChunkId: buildArtifactChunkId({
|
|
1646
|
+
artifactId: input.entry.artifactId,
|
|
1647
|
+
artifactGenerationId: input.entry.artifactGenerationId,
|
|
1648
|
+
chunkStartByte: input.chunkStartByte,
|
|
1649
|
+
chunkEndByte: input.chunkEndByte,
|
|
1650
|
+
chunkSha256
|
|
1651
|
+
}),
|
|
1652
|
+
artifactKey: input.entry.artifactKey,
|
|
1653
|
+
artifactType: input.entry.artifactType,
|
|
1654
|
+
source: input.entry.source,
|
|
1655
|
+
sourcePath: input.entry.sourcePath,
|
|
1656
|
+
sessionId: input.entry.sessionId,
|
|
1657
|
+
contentType: input.entry.contentType,
|
|
1658
|
+
contentEncoding: input.entry.contentEncoding,
|
|
1659
|
+
sizeBytes: input.entry.sizeBytes,
|
|
1660
|
+
// Only the final-chunk computation produces a hash for the current
|
|
1661
|
+
// generation at the current sizeBytes; never fall back to the stored
|
|
1662
|
+
// hash of a previously completed (shorter) content.
|
|
1663
|
+
contentSha256: input.contentSha256,
|
|
1664
|
+
chunkStartByte: input.chunkStartByte,
|
|
1665
|
+
chunkEndByte: input.chunkEndByte,
|
|
1666
|
+
chunkSizeBytes: input.chunk.length,
|
|
1667
|
+
chunkSha256,
|
|
1668
|
+
mtimeMs: input.entry.mtimeMs,
|
|
1669
|
+
firstSeenAtMs: input.entry.firstSeenAtMs,
|
|
1670
|
+
lastSeenAtMs: input.entry.lastSeenAtMs,
|
|
1671
|
+
metadata: input.entry.metadata
|
|
1672
|
+
};
|
|
1673
|
+
}
|
|
1674
|
+
function repositoryContextNeedsUpload(entry) {
|
|
1675
|
+
return repositoryContextRevision(entry.metadata, entry.artifactGenerationId) > (entry.uploadedRepositoryContextRevision ?? 0);
|
|
1676
|
+
}
|
|
1677
|
+
function installIdentityAmendment(metadata) {
|
|
1678
|
+
const observer = metadata.observer !== null && typeof metadata.observer === "object" && !Array.isArray(metadata.observer) ? metadata.observer : void 0;
|
|
1679
|
+
const fmlInstallId = observer?.fmlInstallId;
|
|
1680
|
+
const validObserver = typeof fmlInstallId === "string" && fmlInstallId.trim().length > 0 ? observer : void 0;
|
|
1681
|
+
const harnessInstallId = typeof metadata.harnessInstallId === "string" && metadata.harnessInstallId.trim().length > 0 ? metadata.harnessInstallId : void 0;
|
|
1682
|
+
if (validObserver === void 0 && harnessInstallId === void 0) {
|
|
1683
|
+
return void 0;
|
|
1684
|
+
}
|
|
1685
|
+
const identity = {
|
|
1686
|
+
installIdentitySchemaVersion: RAW_ARTIFACT_IDENTITY_SCHEMA_VERSION,
|
|
1687
|
+
...validObserver === void 0 ? {} : { observer: validObserver },
|
|
1688
|
+
...harnessInstallId === void 0 ? {} : { harnessInstallId }
|
|
1689
|
+
};
|
|
1690
|
+
return Buffer.byteLength(JSON.stringify(identity), "utf8") <= MAX_RAW_ARTIFACT_NESTED_METADATA_BYTES ? identity : void 0;
|
|
1691
|
+
}
|
|
1692
|
+
function identitySignature(metadata) {
|
|
1693
|
+
if (!metadata) return void 0;
|
|
1694
|
+
const identity = installIdentityAmendment(metadata);
|
|
1695
|
+
if (identity === void 0) return void 0;
|
|
1696
|
+
return JSON.stringify({
|
|
1697
|
+
installIdentitySchemaVersion: RAW_ARTIFACT_IDENTITY_SCHEMA_VERSION,
|
|
1698
|
+
observer: identity.observer ?? null,
|
|
1699
|
+
harnessInstallId: identity.harnessInstallId ?? null
|
|
1700
|
+
});
|
|
1701
|
+
}
|
|
1702
|
+
function identityNeedsUpload(entry) {
|
|
1703
|
+
const signature = identitySignature(entry.metadata);
|
|
1704
|
+
return signature !== void 0 && signature !== entry.uploadedIdentitySignature;
|
|
1705
|
+
}
|
|
1706
|
+
var UNCONFIRMED_IDENTITY_PERSISTENCE_ERROR = "raw artifact server did not confirm install identity persistence";
|
|
1707
|
+
function rawUploadAcknowledgedIdentitySignature(entry, metadata, response) {
|
|
1708
|
+
const signature = identitySignature(metadata.metadata);
|
|
1709
|
+
if (signature === void 0) return entry.uploadedIdentitySignature;
|
|
1710
|
+
return response.status === "stored" || response.installIdentityPersisted === true ? signature : entry.uploadedIdentitySignature;
|
|
1711
|
+
}
|
|
1712
|
+
function rawUploadLeavesFinalIdentityUnconfirmed(entry, metadata, response) {
|
|
1713
|
+
return metadata.chunkEndByte === entry.sizeBytes && identityNeedsUpload(entry) && identitySignature(metadata.metadata) !== void 0 && response.status === "duplicate" && response.installIdentityPersisted !== true;
|
|
1714
|
+
}
|
|
1715
|
+
function metadataAmendmentNeedsUpload(entry) {
|
|
1716
|
+
return repositoryContextNeedsUpload(entry) || identityNeedsUpload(entry);
|
|
1717
|
+
}
|
|
1718
|
+
function acknowledgedRepositoryContextRevision(entry, metadata) {
|
|
1719
|
+
const revision = repositoryContextRevision(
|
|
1720
|
+
metadata.metadata,
|
|
1721
|
+
metadata.artifactGenerationId
|
|
1722
|
+
);
|
|
1723
|
+
return revision > 0 ? Math.max(entry.uploadedRepositoryContextRevision ?? 0, revision) : entry.uploadedRepositoryContextRevision;
|
|
1724
|
+
}
|
|
1725
|
+
function acknowledgedCheckpoints(entry) {
|
|
1726
|
+
if (entry.uploadedChunkCheckpoints !== void 0) {
|
|
1727
|
+
return entry.uploadedChunkCheckpoints;
|
|
1728
|
+
}
|
|
1729
|
+
if (entry.lastUploadedChunkStartByte !== void 0 && entry.lastUploadedChunkEndByte !== void 0 && entry.lastUploadedChunkSha256 !== void 0) {
|
|
1730
|
+
return [
|
|
1731
|
+
{
|
|
1732
|
+
chunkStartByte: entry.lastUploadedChunkStartByte,
|
|
1733
|
+
chunkEndByte: entry.lastUploadedChunkEndByte,
|
|
1734
|
+
chunkSha256: entry.lastUploadedChunkSha256
|
|
1735
|
+
}
|
|
1736
|
+
];
|
|
1737
|
+
}
|
|
1738
|
+
return [];
|
|
1739
|
+
}
|
|
1740
|
+
function contiguousCheckpoints(checkpoints, expectedEndByte) {
|
|
1741
|
+
if (!Number.isSafeInteger(expectedEndByte) || expectedEndByte < 0) {
|
|
1742
|
+
return null;
|
|
1743
|
+
}
|
|
1744
|
+
const sorted = [...checkpoints].sort(
|
|
1745
|
+
(left, right) => left.chunkStartByte - right.chunkStartByte
|
|
1746
|
+
);
|
|
1747
|
+
let nextStartByte = 0;
|
|
1748
|
+
for (const checkpoint of sorted) {
|
|
1749
|
+
if (!Number.isSafeInteger(checkpoint.chunkStartByte) || !Number.isSafeInteger(checkpoint.chunkEndByte) || checkpoint.chunkStartByte !== nextStartByte || checkpoint.chunkEndByte <= checkpoint.chunkStartByte || checkpoint.chunkEndByte > expectedEndByte) {
|
|
1750
|
+
return null;
|
|
1751
|
+
}
|
|
1752
|
+
nextStartByte = checkpoint.chunkEndByte;
|
|
1753
|
+
}
|
|
1754
|
+
return nextStartByte === expectedEndByte ? sorted : null;
|
|
1755
|
+
}
|
|
1756
|
+
function validateCheckpointPrefix(input) {
|
|
1757
|
+
const checkpoints = contiguousCheckpoints(
|
|
1758
|
+
input.checkpoints,
|
|
1759
|
+
input.expectedEndByte
|
|
1760
|
+
);
|
|
1761
|
+
if (!checkpoints) return { status: "mismatch" };
|
|
1762
|
+
const pathRevisionBefore = safeSourceRevision(input.sourcePath);
|
|
1763
|
+
if (!pathRevisionBefore || pathRevisionBefore.sizeBytes < BigInt(input.expectedEndByte) || input.expectedRevision && !sameSourceRevision(pathRevisionBefore, input.expectedRevision)) {
|
|
1764
|
+
return { status: "changed" };
|
|
1765
|
+
}
|
|
1766
|
+
let fd;
|
|
1767
|
+
try {
|
|
1768
|
+
fd = fs7.openSync(input.sourcePath, "r");
|
|
1769
|
+
} catch {
|
|
1770
|
+
return { status: "changed" };
|
|
1771
|
+
}
|
|
1772
|
+
try {
|
|
1773
|
+
const openedRevision = sourceRevisionFromStat(
|
|
1774
|
+
fs7.fstatSync(fd, { bigint: true })
|
|
1775
|
+
);
|
|
1776
|
+
if (!sameSourceRevision(pathRevisionBefore, openedRevision)) {
|
|
1777
|
+
return { status: "changed" };
|
|
1778
|
+
}
|
|
1779
|
+
const contentHash = createHash4("sha256");
|
|
1780
|
+
let checkpointMismatch = false;
|
|
1781
|
+
for (const checkpoint of checkpoints) {
|
|
1782
|
+
const chunk = readExactRange(
|
|
1783
|
+
fd,
|
|
1784
|
+
checkpoint.chunkStartByte,
|
|
1785
|
+
checkpoint.chunkEndByte
|
|
1786
|
+
);
|
|
1787
|
+
if (!chunk || sha256Hex(chunk) !== checkpoint.chunkSha256) {
|
|
1788
|
+
checkpointMismatch = true;
|
|
1789
|
+
break;
|
|
1790
|
+
}
|
|
1791
|
+
contentHash.update(chunk);
|
|
1792
|
+
}
|
|
1793
|
+
const revisionAfterRead = sourceRevisionFromStat(
|
|
1794
|
+
fs7.fstatSync(fd, { bigint: true })
|
|
1795
|
+
);
|
|
1796
|
+
const pathRevisionAfter = safeSourceRevision(input.sourcePath);
|
|
1797
|
+
if (!sameSourceRevision(openedRevision, revisionAfterRead) || !pathRevisionAfter || !sameSourceRevision(openedRevision, pathRevisionAfter)) {
|
|
1798
|
+
return { status: "changed" };
|
|
1799
|
+
}
|
|
1800
|
+
if (checkpointMismatch) return { status: "mismatch" };
|
|
1801
|
+
return {
|
|
1802
|
+
status: "valid",
|
|
1803
|
+
revision: openedRevision,
|
|
1804
|
+
contentSha256: contentHash.digest("hex")
|
|
1805
|
+
};
|
|
1806
|
+
} catch {
|
|
1807
|
+
return { status: "changed" };
|
|
1808
|
+
} finally {
|
|
1809
|
+
fs7.closeSync(fd);
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
function validatedPrefixStillApplies(prefix, entry, uploadDestinationId, revision) {
|
|
1813
|
+
return prefix?.artifactGenerationId === entry.artifactGenerationId && prefix.uploadDestinationId === uploadDestinationId && prefix.sourcePath === artifactReadPath(entry) && prefix.verifiedThroughByte === entry.uploadedSizeBytes && sameSourceRevision(prefix.revision, revision);
|
|
1814
|
+
}
|
|
1815
|
+
function withAcknowledgedCheckpoint(entry, checkpoint) {
|
|
1816
|
+
return [
|
|
1817
|
+
...acknowledgedCheckpoints(entry).filter(
|
|
1818
|
+
(existing) => existing.chunkStartByte !== checkpoint.chunkStartByte
|
|
1819
|
+
),
|
|
1820
|
+
checkpoint
|
|
1821
|
+
].sort((left, right) => left.chunkStartByte - right.chunkStartByte);
|
|
1822
|
+
}
|
|
1823
|
+
function startNewGenerationFromEntry(entry, nowMs) {
|
|
1824
|
+
const stat = safeStat(artifactReadPath(entry));
|
|
1825
|
+
if (!stat?.isFile()) return null;
|
|
1826
|
+
const sizeBytes = stat.size;
|
|
1827
|
+
const mtimeMs = statMtimeMs(stat);
|
|
1828
|
+
const generationStartedAtMs = nextGenerationStartedAtMs(nowMs, entry);
|
|
1829
|
+
const artifactGenerationId = buildInitialGenerationId({
|
|
1830
|
+
artifactId: entry.artifactId,
|
|
1831
|
+
nowMs: generationStartedAtMs
|
|
1832
|
+
});
|
|
1833
|
+
return {
|
|
1834
|
+
...entry,
|
|
1835
|
+
artifactGenerationId,
|
|
1836
|
+
metadata: metadataForGeneration({
|
|
1837
|
+
sourcePath: entry.sourcePath,
|
|
1838
|
+
artifactGenerationId,
|
|
1839
|
+
candidateMetadata: entry.metadata,
|
|
1840
|
+
inheritPrevious: false
|
|
1841
|
+
}),
|
|
1842
|
+
generationStartedAtMs,
|
|
1843
|
+
sizeBytes,
|
|
1844
|
+
mtimeMs,
|
|
1845
|
+
lastSeenAtMs: generationStartedAtMs,
|
|
1846
|
+
status: sizeBytes === 0 ? "uploaded" : "pending",
|
|
1847
|
+
uploadedSizeBytes: 0,
|
|
1848
|
+
uploadedRepositoryContextRevision: void 0,
|
|
1849
|
+
uploadedIdentitySignature: void 0,
|
|
1850
|
+
// A new generation locks its own chunk grid on first upload and computes
|
|
1851
|
+
// its own full-content hash; never inherit the previous generation's.
|
|
1852
|
+
contentSha256: void 0,
|
|
1853
|
+
uploadChunkSizeBytes: void 0,
|
|
1854
|
+
lastUploadedChunkStartByte: void 0,
|
|
1855
|
+
lastUploadedChunkEndByte: void 0,
|
|
1856
|
+
lastUploadedChunkSha256: void 0,
|
|
1857
|
+
uploadedChunkCheckpoints: [],
|
|
1858
|
+
uploadDestinationId: void 0,
|
|
1859
|
+
uploadTarget: void 0,
|
|
1860
|
+
inFlightChunk: void 0,
|
|
1861
|
+
uploadedAtMs: void 0,
|
|
1862
|
+
uploadAttempts: 0,
|
|
1863
|
+
lastAttemptAtMs: void 0,
|
|
1864
|
+
lastError: void 0
|
|
1865
|
+
};
|
|
1866
|
+
}
|
|
1867
|
+
function rawArtifactUrl(targetUrl) {
|
|
1868
|
+
return `${targetUrl.replace(/\/$/, "")}${RAW_ARTIFACT_ENDPOINT}`;
|
|
1869
|
+
}
|
|
1870
|
+
function jwtPrincipalFingerprint(authorization) {
|
|
1871
|
+
const token = authorization.trim().replace(/^Bearer\s+/, "");
|
|
1872
|
+
const parts = token.split(".");
|
|
1873
|
+
if (parts.length !== 3) return null;
|
|
1874
|
+
try {
|
|
1875
|
+
const payload = JSON.parse(
|
|
1876
|
+
Buffer.from(parts[1] ?? "", "base64url").toString("utf8")
|
|
1877
|
+
);
|
|
1878
|
+
if (!payload || typeof payload !== "object") return null;
|
|
1879
|
+
const claims = payload;
|
|
1880
|
+
if (typeof claims.sub !== "string" || claims.sub.length === 0) {
|
|
1881
|
+
return null;
|
|
1882
|
+
}
|
|
1883
|
+
const issuer = typeof claims.iss === "string" ? claims.iss : "";
|
|
1884
|
+
return sha256Hex(`jwt\0${issuer}\0${claims.sub}`);
|
|
1885
|
+
} catch {
|
|
1886
|
+
return null;
|
|
1887
|
+
}
|
|
1888
|
+
}
|
|
1889
|
+
function buildUploadDestinationId(input) {
|
|
1890
|
+
const normalizedTargetUrl = input.targetUrl.replace(/\/$/, "");
|
|
1891
|
+
const explicitIdentity = input.destinationIdentity?.trim();
|
|
1892
|
+
const credentialFingerprint = explicitIdentity ? sha256Hex(`identity\0${explicitIdentity}`) : jwtPrincipalFingerprint(input.authorization) ?? sha256Hex(input.authorization);
|
|
1893
|
+
const organizationScope = input.organization ? `\0${input.organization}` : "";
|
|
1894
|
+
return `raw_artifact_destination:${sha256Hex(
|
|
1895
|
+
`${normalizedTargetUrl}\0${input.targetName}\0${credentialFingerprint}${organizationScope}`
|
|
1896
|
+
)}`;
|
|
1897
|
+
}
|
|
1898
|
+
function rawArtifactRequestHeaders(options) {
|
|
1899
|
+
return {
|
|
1900
|
+
Authorization: options.authorization,
|
|
1901
|
+
...options.organization ? { "X-FML-Organization": options.organization } : {}
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
function canPreserveRetryStateOnDestinationActivation(entry) {
|
|
1905
|
+
return entry.uploadDestinationId === void 0 && entry.uploadedSizeBytes === 0 && entry.status !== "uploaded";
|
|
1906
|
+
}
|
|
1907
|
+
function activateUploadDestination(entry, uploadDestinationId, targetName) {
|
|
1908
|
+
if (entry.uploadDestinationId === uploadDestinationId) return entry;
|
|
1909
|
+
const canPreserveRetryState = canPreserveRetryStateOnDestinationActivation(entry);
|
|
1910
|
+
return {
|
|
1911
|
+
...entry,
|
|
1912
|
+
status: entry.sizeBytes === 0 ? "uploaded" : canPreserveRetryState ? entry.status : "pending",
|
|
1913
|
+
uploadedSizeBytes: 0,
|
|
1914
|
+
uploadedRepositoryContextRevision: void 0,
|
|
1915
|
+
uploadedIdentitySignature: void 0,
|
|
1916
|
+
contentSha256: void 0,
|
|
1917
|
+
lastUploadedChunkStartByte: void 0,
|
|
1918
|
+
lastUploadedChunkEndByte: void 0,
|
|
1919
|
+
lastUploadedChunkSha256: void 0,
|
|
1920
|
+
uploadedChunkCheckpoints: [],
|
|
1921
|
+
uploadDestinationId,
|
|
1922
|
+
uploadTarget: targetName,
|
|
1923
|
+
inFlightChunk: void 0,
|
|
1924
|
+
uploadedAtMs: void 0,
|
|
1925
|
+
uploadAttempts: canPreserveRetryState ? entry.uploadAttempts : 0,
|
|
1926
|
+
lastAttemptAtMs: canPreserveRetryState ? entry.lastAttemptAtMs : void 0,
|
|
1927
|
+
lastError: canPreserveRetryState ? entry.lastError : void 0
|
|
1928
|
+
};
|
|
1929
|
+
}
|
|
1930
|
+
function queueLockFile(queueFile) {
|
|
1931
|
+
return `${queueFile}.lock`;
|
|
1932
|
+
}
|
|
1933
|
+
function defaultIsPidAlive(pid) {
|
|
1934
|
+
try {
|
|
1935
|
+
process.kill(pid, 0);
|
|
1936
|
+
return true;
|
|
1937
|
+
} catch (error) {
|
|
1938
|
+
return error.code === "EPERM";
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
function readQueueLockHolder(lockFile) {
|
|
1942
|
+
try {
|
|
1943
|
+
const parsed = JSON.parse(fs7.readFileSync(lockFile, "utf8"));
|
|
1944
|
+
if (!parsed || typeof parsed !== "object") return null;
|
|
1945
|
+
const record = parsed;
|
|
1946
|
+
if (typeof record.pid !== "number" || typeof record.timestampMs !== "number") {
|
|
1947
|
+
return null;
|
|
1948
|
+
}
|
|
1949
|
+
const ownerToken = typeof record.ownerToken === "string" ? record.ownerToken : void 0;
|
|
1950
|
+
const timestampMs = ownerToken ? Math.max(record.timestampMs, fs7.statSync(lockFile).mtimeMs) : record.timestampMs;
|
|
1951
|
+
return { pid: record.pid, timestampMs, ownerToken };
|
|
1952
|
+
} catch {
|
|
1953
|
+
return null;
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
function queueLockFileMatchesOwner(lockFile, fd, ownerToken) {
|
|
1957
|
+
try {
|
|
1958
|
+
const openStat = fs7.fstatSync(fd);
|
|
1959
|
+
const pathStat = fs7.statSync(lockFile);
|
|
1960
|
+
if (openStat.dev !== pathStat.dev || openStat.ino !== pathStat.ino) {
|
|
1961
|
+
return false;
|
|
1962
|
+
}
|
|
1963
|
+
return readQueueLockHolder(lockFile)?.ownerToken === ownerToken;
|
|
1964
|
+
} catch {
|
|
1965
|
+
return false;
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
function acquireQueueLock(queueFile, deps) {
|
|
1969
|
+
const lockFile = queueLockFile(queueFile);
|
|
1970
|
+
fs7.mkdirSync(path17.dirname(lockFile), { recursive: true, mode: 448 });
|
|
1971
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
1972
|
+
try {
|
|
1973
|
+
const fd = fs7.openSync(lockFile, "wx+", 384);
|
|
1974
|
+
const ownerToken = randomUUID3();
|
|
1975
|
+
let released = false;
|
|
1976
|
+
try {
|
|
1977
|
+
fs7.writeSync(
|
|
1978
|
+
fd,
|
|
1979
|
+
`${JSON.stringify({
|
|
1980
|
+
pid: process.pid,
|
|
1981
|
+
ownerToken,
|
|
1982
|
+
timestampMs: deps.nowMs()
|
|
1983
|
+
})}
|
|
1984
|
+
`
|
|
1985
|
+
);
|
|
1986
|
+
const heartbeatAt = new Date(deps.nowMs());
|
|
1987
|
+
fs7.futimesSync(fd, heartbeatAt, heartbeatAt);
|
|
1988
|
+
} catch (error) {
|
|
1989
|
+
fs7.closeSync(fd);
|
|
1990
|
+
try {
|
|
1991
|
+
fs7.unlinkSync(lockFile);
|
|
1992
|
+
} catch {
|
|
1993
|
+
}
|
|
1994
|
+
throw error;
|
|
1995
|
+
}
|
|
1996
|
+
const close = () => {
|
|
1997
|
+
if (released) return;
|
|
1998
|
+
released = true;
|
|
1999
|
+
fs7.closeSync(fd);
|
|
2000
|
+
};
|
|
2001
|
+
return {
|
|
2002
|
+
heartbeat() {
|
|
2003
|
+
if (released) return false;
|
|
2004
|
+
if (!queueLockFileMatchesOwner(lockFile, fd, ownerToken))
|
|
2005
|
+
return false;
|
|
2006
|
+
try {
|
|
2007
|
+
const heartbeatAt = new Date(deps.nowMs());
|
|
2008
|
+
fs7.futimesSync(fd, heartbeatAt, heartbeatAt);
|
|
2009
|
+
return true;
|
|
2010
|
+
} catch {
|
|
2011
|
+
return false;
|
|
2012
|
+
}
|
|
2013
|
+
},
|
|
2014
|
+
release() {
|
|
2015
|
+
if (released) return;
|
|
2016
|
+
if (queueLockFileMatchesOwner(lockFile, fd, ownerToken)) {
|
|
2017
|
+
try {
|
|
2018
|
+
fs7.unlinkSync(lockFile);
|
|
2019
|
+
} catch {
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
close();
|
|
2023
|
+
}
|
|
2024
|
+
};
|
|
2025
|
+
} catch (error) {
|
|
2026
|
+
if (error.code !== "EEXIST") throw error;
|
|
2027
|
+
const holder = readQueueLockHolder(lockFile);
|
|
2028
|
+
const stale = !holder || !deps.isPidAlive(holder.pid) || deps.nowMs() - holder.timestampMs > QUEUE_LOCK_STALE_MS;
|
|
2029
|
+
if (!stale) return null;
|
|
2030
|
+
try {
|
|
2031
|
+
fs7.unlinkSync(lockFile);
|
|
2032
|
+
} catch {
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
return null;
|
|
2037
|
+
}
|
|
2038
|
+
function resetArtifactQueue(options) {
|
|
2039
|
+
const nowMs = options.nowMs ?? Date.now;
|
|
2040
|
+
const lock = acquireQueueLock(options.queueFile, {
|
|
2041
|
+
nowMs,
|
|
2042
|
+
isPidAlive: options.isPidAlive ?? defaultIsPidAlive
|
|
2043
|
+
});
|
|
2044
|
+
if (!lock) {
|
|
2045
|
+
return {
|
|
2046
|
+
reset: 0,
|
|
2047
|
+
removed: 0,
|
|
2048
|
+
skipped: true,
|
|
2049
|
+
queue: readArtifactQueue(options.queueFile)
|
|
2050
|
+
};
|
|
2051
|
+
}
|
|
2052
|
+
try {
|
|
2053
|
+
const queue = readArtifactQueue(options.queueFile);
|
|
2054
|
+
const resetAtMs = nowMs();
|
|
2055
|
+
const artifacts = queue.artifacts.flatMap((entry) => {
|
|
2056
|
+
if (!artifactSourceEnabled(entry.source)) return [];
|
|
2057
|
+
const reset = startNewGenerationFromEntry(entry, resetAtMs);
|
|
2058
|
+
return reset ? [reset] : [];
|
|
2059
|
+
});
|
|
2060
|
+
const nextQueue = {
|
|
2061
|
+
schemaVersion: RAW_ARTIFACT_SCHEMA_VERSION,
|
|
2062
|
+
artifacts,
|
|
2063
|
+
...queue.uploadThrottles ? { uploadThrottles: queue.uploadThrottles } : {}
|
|
2064
|
+
};
|
|
2065
|
+
if (!lock.heartbeat()) {
|
|
2066
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2067
|
+
}
|
|
2068
|
+
writeArtifactQueue(options.queueFile, nextQueue);
|
|
2069
|
+
return {
|
|
2070
|
+
reset: artifacts.length,
|
|
2071
|
+
removed: queue.artifacts.length - artifacts.length,
|
|
2072
|
+
skipped: false,
|
|
2073
|
+
queue: nextQueue
|
|
2074
|
+
};
|
|
2075
|
+
} finally {
|
|
2076
|
+
lock.release();
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
async function postRawArtifact(url, metadata, content, headers, fetchImpl = fetch, timeoutMs = RAW_ARTIFACT_POST_TIMEOUT_MS) {
|
|
2080
|
+
const form = new FormData();
|
|
2081
|
+
form.append(
|
|
2082
|
+
"metadata",
|
|
2083
|
+
new Blob([JSON.stringify(metadata)], { type: "application/json" }),
|
|
2084
|
+
"metadata.json"
|
|
2085
|
+
);
|
|
2086
|
+
form.append(
|
|
2087
|
+
"file",
|
|
2088
|
+
new Blob([new Uint8Array(content)], { type: metadata.contentType }),
|
|
2089
|
+
path17.basename(metadata.sourcePath) || "raw-artifact"
|
|
2090
|
+
);
|
|
2091
|
+
const abort = new AbortController();
|
|
2092
|
+
const timeout = setTimeout(() => abort.abort(), timeoutMs);
|
|
2093
|
+
let bodyText;
|
|
2094
|
+
let responseStatus;
|
|
2095
|
+
let responseOk;
|
|
2096
|
+
let retryAfter;
|
|
2097
|
+
try {
|
|
2098
|
+
const response2 = await fetchImpl(url, {
|
|
2099
|
+
method: "POST",
|
|
2100
|
+
headers,
|
|
2101
|
+
body: form,
|
|
2102
|
+
signal: abort.signal
|
|
2103
|
+
});
|
|
2104
|
+
bodyText = await response2.text().catch(() => "");
|
|
2105
|
+
responseStatus = response2.status;
|
|
2106
|
+
responseOk = response2.ok;
|
|
2107
|
+
retryAfter = response2.headers.get("Retry-After") ?? void 0;
|
|
2108
|
+
} catch (error) {
|
|
2109
|
+
if (abort.signal.aborted) {
|
|
2110
|
+
throw new Error(`raw artifact POST timed out after ${timeoutMs}ms`);
|
|
2111
|
+
}
|
|
2112
|
+
throw error;
|
|
2113
|
+
} finally {
|
|
2114
|
+
clearTimeout(timeout);
|
|
2115
|
+
}
|
|
2116
|
+
if (!responseOk) {
|
|
2117
|
+
let code;
|
|
2118
|
+
try {
|
|
2119
|
+
const errorBody = JSON.parse(bodyText);
|
|
2120
|
+
if (errorBody && typeof errorBody === "object") {
|
|
2121
|
+
const parsedCode = errorBody.code;
|
|
2122
|
+
if (typeof parsedCode === "string") code = parsedCode;
|
|
2123
|
+
}
|
|
2124
|
+
} catch {
|
|
2125
|
+
}
|
|
2126
|
+
throw new RawArtifactPostError(
|
|
2127
|
+
`HTTP ${responseStatus}${bodyText ? `: ${bodyText.slice(0, 500)}` : ""}`,
|
|
2128
|
+
responseStatus,
|
|
2129
|
+
code,
|
|
2130
|
+
retryAfter
|
|
2131
|
+
);
|
|
2132
|
+
}
|
|
2133
|
+
let parsed;
|
|
2134
|
+
try {
|
|
2135
|
+
parsed = JSON.parse(bodyText);
|
|
2136
|
+
} catch {
|
|
2137
|
+
throw new Error("raw artifact POST returned invalid JSON");
|
|
2138
|
+
}
|
|
2139
|
+
if (!parsed || typeof parsed !== "object") {
|
|
2140
|
+
throw new Error("raw artifact POST returned an invalid acknowledgement");
|
|
2141
|
+
}
|
|
2142
|
+
const response = parsed;
|
|
2143
|
+
if (response.ok !== true || response.status !== "stored" && response.status !== "duplicate") {
|
|
2144
|
+
throw new Error("raw artifact POST did not acknowledge the upload");
|
|
2145
|
+
}
|
|
2146
|
+
if (response.artifactId !== metadata.artifactId || response.artifactGenerationId !== metadata.artifactGenerationId || response.artifactChunkId !== metadata.artifactChunkId) {
|
|
2147
|
+
throw new Error(
|
|
2148
|
+
"raw artifact POST acknowledgement identifiers did not match"
|
|
2149
|
+
);
|
|
2150
|
+
}
|
|
2151
|
+
return {
|
|
2152
|
+
ok: true,
|
|
2153
|
+
status: response.status,
|
|
2154
|
+
artifactId: metadata.artifactId,
|
|
2155
|
+
artifactGenerationId: metadata.artifactGenerationId,
|
|
2156
|
+
artifactChunkId: metadata.artifactChunkId,
|
|
2157
|
+
...response.installIdentityPersisted === true ? { installIdentityPersisted: true } : {}
|
|
2158
|
+
};
|
|
2159
|
+
}
|
|
2160
|
+
function retryAtMsFromHeader(retryAfter, nowMs) {
|
|
2161
|
+
const normalized = retryAfter?.trim();
|
|
2162
|
+
const minimumRetryAtMs = Math.min(
|
|
2163
|
+
Number.MAX_SAFE_INTEGER,
|
|
2164
|
+
nowMs + MIN_RATE_LIMIT_RETRY_MS
|
|
2165
|
+
);
|
|
2166
|
+
const maximumRetryAtMs = Math.min(
|
|
2167
|
+
Number.MAX_SAFE_INTEGER,
|
|
2168
|
+
nowMs + MAX_RATE_LIMIT_RETRY_MS
|
|
2169
|
+
);
|
|
2170
|
+
if (normalized && /^\d+$/.test(normalized)) {
|
|
2171
|
+
const seconds = Number(normalized);
|
|
2172
|
+
if (Number.isSafeInteger(seconds)) {
|
|
2173
|
+
return Math.max(
|
|
2174
|
+
minimumRetryAtMs,
|
|
2175
|
+
Math.min(maximumRetryAtMs, nowMs + seconds * 1e3)
|
|
2176
|
+
);
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
if (normalized) {
|
|
2180
|
+
const parsedDate = Date.parse(normalized);
|
|
2181
|
+
if (Number.isFinite(parsedDate)) {
|
|
2182
|
+
return Math.max(minimumRetryAtMs, Math.min(maximumRetryAtMs, parsedDate));
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
return Math.min(maximumRetryAtMs, nowMs + RATE_LIMIT_RETRY_FALLBACK_MS);
|
|
2186
|
+
}
|
|
2187
|
+
function activeRetryAtMs(throttle, nowMs) {
|
|
2188
|
+
if (!throttle || throttle.retryAtMs <= nowMs) return void 0;
|
|
2189
|
+
return Math.min(
|
|
2190
|
+
throttle.retryAtMs,
|
|
2191
|
+
Math.min(Number.MAX_SAFE_INTEGER, nowMs + MAX_RATE_LIMIT_RETRY_MS)
|
|
2192
|
+
);
|
|
2193
|
+
}
|
|
2194
|
+
function retryBackoffMs(entry) {
|
|
2195
|
+
const exponent = Math.max(0, entry.uploadAttempts - 1);
|
|
2196
|
+
const baseDelay = Math.min(
|
|
2197
|
+
RETRY_BACKOFF_MAX_MS,
|
|
2198
|
+
RETRY_BACKOFF_BASE_MS * 2 ** exponent
|
|
2199
|
+
);
|
|
2200
|
+
const jitterSeed = Number.parseInt(
|
|
2201
|
+
sha256Hex(`${entry.artifactId}\0${entry.uploadAttempts}`).slice(0, 8),
|
|
2202
|
+
16
|
|
2203
|
+
);
|
|
2204
|
+
const jitter = 0.75 + jitterSeed / 4294967295 * 0.5;
|
|
2205
|
+
return Math.round(baseDelay * jitter);
|
|
2206
|
+
}
|
|
2207
|
+
function isUploadableEntry(entry, nowMs, retryExhausted) {
|
|
2208
|
+
return (entry.status === "pending" || entry.status === "failed") && (retryExhausted || entry.uploadAttempts < MAX_UPLOAD_ATTEMPTS) && (entry.status !== "failed" || entry.lastAttemptAtMs === void 0 || nowMs - entry.lastAttemptAtMs >= retryBackoffMs(entry));
|
|
2209
|
+
}
|
|
2210
|
+
function isRunnableForDestination(entry, uploadDestinationId, nowMs, retryExhausted) {
|
|
2211
|
+
if (!needsUploadForDestination(entry, uploadDestinationId)) return false;
|
|
2212
|
+
if (entry.uploadDestinationId !== uploadDestinationId && !canPreserveRetryStateOnDestinationActivation(entry)) {
|
|
2213
|
+
return true;
|
|
2214
|
+
}
|
|
2215
|
+
return isUploadableEntry(entry, nowMs, retryExhausted);
|
|
2216
|
+
}
|
|
2217
|
+
function needsUploadForDestination(entry, uploadDestinationId) {
|
|
2218
|
+
if (!artifactSourceEnabled(entry.source)) return false;
|
|
2219
|
+
if (entry.sizeBytes === 0) return false;
|
|
2220
|
+
if (entry.uploadDestinationId !== uploadDestinationId) return true;
|
|
2221
|
+
return entry.status === "pending" || entry.status === "failed";
|
|
2222
|
+
}
|
|
2223
|
+
function entryRetryAtMs(entry, retryExhausted) {
|
|
2224
|
+
if (entry.status !== "failed" || entry.lastAttemptAtMs === void 0) {
|
|
2225
|
+
return void 0;
|
|
2226
|
+
}
|
|
2227
|
+
if (!retryExhausted && entry.uploadAttempts >= MAX_UPLOAD_ATTEMPTS) {
|
|
2228
|
+
return void 0;
|
|
2229
|
+
}
|
|
2230
|
+
return entry.lastAttemptAtMs + retryBackoffMs(entry);
|
|
2231
|
+
}
|
|
2232
|
+
function nextRetryAtMsForDestination(artifacts, uploadDestinationId, nowMs, retryExhausted) {
|
|
2233
|
+
let nextRetryAtMs;
|
|
2234
|
+
for (const entry of artifacts) {
|
|
2235
|
+
if (!needsUploadForDestination(entry, uploadDestinationId)) continue;
|
|
2236
|
+
if (isRunnableForDestination(
|
|
2237
|
+
entry,
|
|
2238
|
+
uploadDestinationId,
|
|
2239
|
+
nowMs,
|
|
2240
|
+
retryExhausted
|
|
2241
|
+
)) {
|
|
2242
|
+
continue;
|
|
2243
|
+
}
|
|
2244
|
+
const retryAtMs = entryRetryAtMs(entry, retryExhausted);
|
|
2245
|
+
if (retryAtMs === void 0) continue;
|
|
2246
|
+
nextRetryAtMs = nextRetryAtMs === void 0 ? retryAtMs : Math.min(nextRetryAtMs, retryAtMs);
|
|
2247
|
+
}
|
|
2248
|
+
return nextRetryAtMs;
|
|
2249
|
+
}
|
|
2250
|
+
function isExhaustedForDestination(entry, uploadDestinationId, retryExhausted) {
|
|
2251
|
+
return artifactSourceEnabled(entry.source) && !retryExhausted && entry.sizeBytes > 0 && (entry.uploadDestinationId === uploadDestinationId || canPreserveRetryStateOnDestinationActivation(entry)) && entry.status === "failed" && entry.uploadAttempts >= MAX_UPLOAD_ATTEMPTS;
|
|
2252
|
+
}
|
|
2253
|
+
function resolveUploadRun(options, lock) {
|
|
2254
|
+
const nowMs = options.nowMs ?? Date.now;
|
|
2255
|
+
const targetName = options.targetName ?? options.targetUrl;
|
|
2256
|
+
return {
|
|
2257
|
+
nowMs,
|
|
2258
|
+
limit: options.limit ?? DEFAULT_UPLOAD_LIMIT,
|
|
2259
|
+
maxUploadBytes: Math.max(
|
|
2260
|
+
1,
|
|
2261
|
+
Math.floor(options.maxUploadBytes ?? DEFAULT_UPLOAD_BYTE_LIMIT)
|
|
2262
|
+
),
|
|
2263
|
+
chunkSizeBytes: Math.min(
|
|
2264
|
+
MAX_UPLOAD_CHUNK_BYTES,
|
|
2265
|
+
Math.max(
|
|
2266
|
+
1,
|
|
2267
|
+
Math.floor(options.chunkSizeBytes ?? DEFAULT_UPLOAD_CHUNK_BYTES)
|
|
2268
|
+
)
|
|
2269
|
+
),
|
|
2270
|
+
maxFileBytes: Math.max(
|
|
2271
|
+
1,
|
|
2272
|
+
Math.floor(options.maxFileBytes ?? DEFAULT_MAX_ARTIFACT_BYTES)
|
|
2273
|
+
),
|
|
2274
|
+
targetName,
|
|
2275
|
+
uploadDestinationId: buildUploadDestinationId({
|
|
2276
|
+
targetUrl: options.targetUrl,
|
|
2277
|
+
targetName,
|
|
2278
|
+
authorization: options.authorization,
|
|
2279
|
+
organization: options.organization,
|
|
2280
|
+
destinationIdentity: options.destinationIdentity
|
|
2281
|
+
}),
|
|
2282
|
+
retryExhausted: options.retryExhausted ?? false,
|
|
2283
|
+
lock
|
|
2284
|
+
};
|
|
2285
|
+
}
|
|
2286
|
+
function skippedUploadResult(queue, uploadDestinationId, nowMs, retryExhausted) {
|
|
2287
|
+
const retryAtMs = activeRetryAtMs(
|
|
2288
|
+
queue.uploadThrottles?.[uploadDestinationId],
|
|
2289
|
+
nowMs
|
|
2290
|
+
);
|
|
2291
|
+
return {
|
|
2292
|
+
attempted: 0,
|
|
2293
|
+
uploaded: 0,
|
|
2294
|
+
uploadedChunks: 0,
|
|
2295
|
+
uploadedBytes: 0,
|
|
2296
|
+
failed: 0,
|
|
2297
|
+
exhausted: queue.artifacts.filter(
|
|
2298
|
+
(entry) => isExhaustedForDestination(entry, uploadDestinationId, retryExhausted)
|
|
2299
|
+
).length,
|
|
2300
|
+
refreshed: 0,
|
|
2301
|
+
retryAtMs,
|
|
2302
|
+
nextRetryAtMs: nextRetryAtMsForDestination(
|
|
2303
|
+
queue.artifacts,
|
|
2304
|
+
uploadDestinationId,
|
|
2305
|
+
nowMs,
|
|
2306
|
+
retryExhausted
|
|
2307
|
+
),
|
|
2308
|
+
hasRunnable: retryAtMs === void 0 && queue.artifacts.some(
|
|
2309
|
+
(entry) => isRunnableForDestination(
|
|
2310
|
+
entry,
|
|
2311
|
+
uploadDestinationId,
|
|
2312
|
+
nowMs,
|
|
2313
|
+
retryExhausted
|
|
2314
|
+
)
|
|
2315
|
+
),
|
|
2316
|
+
hasMore: queue.artifacts.some(
|
|
2317
|
+
(entry) => needsUploadForDestination(entry, uploadDestinationId)
|
|
2318
|
+
),
|
|
2319
|
+
skipped: true,
|
|
2320
|
+
queue
|
|
2321
|
+
};
|
|
2322
|
+
}
|
|
2323
|
+
async function syncArtifacts(options) {
|
|
2324
|
+
const nowMs = options.nowMs ?? Date.now;
|
|
2325
|
+
const lock = acquireQueueLock(options.queueFile, {
|
|
2326
|
+
nowMs,
|
|
2327
|
+
isPidAlive: options.isPidAlive ?? defaultIsPidAlive
|
|
2328
|
+
});
|
|
2329
|
+
const targetName = options.targetName ?? options.targetUrl;
|
|
2330
|
+
const uploadDestinationId = buildUploadDestinationId({
|
|
2331
|
+
targetUrl: options.targetUrl,
|
|
2332
|
+
targetName,
|
|
2333
|
+
authorization: options.authorization,
|
|
2334
|
+
organization: options.organization,
|
|
2335
|
+
destinationIdentity: options.destinationIdentity
|
|
2336
|
+
});
|
|
2337
|
+
if (!lock) {
|
|
2338
|
+
const queue = readArtifactQueue(options.queueFile);
|
|
2339
|
+
return {
|
|
2340
|
+
snapshot: {
|
|
2341
|
+
queued: 0,
|
|
2342
|
+
skipped: 0,
|
|
2343
|
+
oversized: [],
|
|
2344
|
+
queue
|
|
2345
|
+
},
|
|
2346
|
+
upload: skippedUploadResult(
|
|
2347
|
+
queue,
|
|
2348
|
+
uploadDestinationId,
|
|
2349
|
+
nowMs(),
|
|
2350
|
+
options.retryExhausted ?? false
|
|
2351
|
+
)
|
|
2352
|
+
};
|
|
2353
|
+
}
|
|
2354
|
+
try {
|
|
2355
|
+
const snapshot = snapshotArtifactsLocked(
|
|
2356
|
+
{
|
|
2357
|
+
queueFile: options.queueFile,
|
|
2358
|
+
candidates: options.candidates,
|
|
2359
|
+
nowMs,
|
|
2360
|
+
maxFileBytes: options.maxFileBytes,
|
|
2361
|
+
observerIdentity: options.observerIdentity
|
|
2362
|
+
},
|
|
2363
|
+
lock
|
|
2364
|
+
);
|
|
2365
|
+
if (!lock.heartbeat()) {
|
|
2366
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2367
|
+
}
|
|
2368
|
+
const upload = await uploadPendingArtifactsLocked(
|
|
2369
|
+
options,
|
|
2370
|
+
resolveUploadRun(options, lock)
|
|
2371
|
+
);
|
|
2372
|
+
return { snapshot, upload };
|
|
2373
|
+
} finally {
|
|
2374
|
+
lock.release();
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
async function uploadPendingArtifactsLocked(options, run) {
|
|
2378
|
+
const {
|
|
2379
|
+
nowMs,
|
|
2380
|
+
limit,
|
|
2381
|
+
maxUploadBytes,
|
|
2382
|
+
chunkSizeBytes,
|
|
2383
|
+
maxFileBytes,
|
|
2384
|
+
targetName,
|
|
2385
|
+
uploadDestinationId,
|
|
2386
|
+
retryExhausted,
|
|
2387
|
+
lock
|
|
2388
|
+
} = run;
|
|
2389
|
+
const queue = readArtifactQueue(options.queueFile);
|
|
2390
|
+
const entries = queue.artifacts.filter((entry) => artifactSourceEnabled(entry.source)).map(
|
|
2391
|
+
(entry) => activateUploadDestination(entry, uploadDestinationId, targetName)
|
|
2392
|
+
);
|
|
2393
|
+
const selectionTimeMs = nowMs();
|
|
2394
|
+
const persistedThrottle = queue.uploadThrottles?.[uploadDestinationId];
|
|
2395
|
+
let retryAtMs = activeRetryAtMs(persistedThrottle, selectionTimeMs);
|
|
2396
|
+
const uploadThrottles = {};
|
|
2397
|
+
if (persistedThrottle && retryAtMs !== void 0) {
|
|
2398
|
+
uploadThrottles[uploadDestinationId] = {
|
|
2399
|
+
...persistedThrottle,
|
|
2400
|
+
retryAtMs
|
|
2401
|
+
};
|
|
2402
|
+
}
|
|
2403
|
+
const pending = retryAtMs ? [] : entries.filter(
|
|
2404
|
+
(entry) => isUploadableEntry(entry, selectionTimeMs, retryExhausted)
|
|
2405
|
+
);
|
|
2406
|
+
const selected = [
|
|
2407
|
+
...pending.filter((entry) => entry.status === "pending"),
|
|
2408
|
+
...pending.filter((entry) => entry.status === "failed")
|
|
2409
|
+
].slice(0, limit);
|
|
2410
|
+
let attempted = 0;
|
|
2411
|
+
let uploaded = 0;
|
|
2412
|
+
let uploadedChunks = 0;
|
|
2413
|
+
let uploadedBytes = 0;
|
|
2414
|
+
let failed = 0;
|
|
2415
|
+
let refreshed = 0;
|
|
2416
|
+
const persist = () => {
|
|
2417
|
+
if (!lock.heartbeat()) {
|
|
2418
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2419
|
+
}
|
|
2420
|
+
writeArtifactQueue(options.queueFile, {
|
|
2421
|
+
schemaVersion: RAW_ARTIFACT_SCHEMA_VERSION,
|
|
2422
|
+
artifacts: entries,
|
|
2423
|
+
uploadThrottles: Object.keys(uploadThrottles).length > 0 ? uploadThrottles : void 0
|
|
2424
|
+
});
|
|
2425
|
+
if (!lock.heartbeat()) {
|
|
2426
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2427
|
+
}
|
|
2428
|
+
};
|
|
2429
|
+
persist();
|
|
2430
|
+
for (const entry of selected) {
|
|
2431
|
+
if (uploadedBytes >= maxUploadBytes) break;
|
|
2432
|
+
const index = entries.findIndex(
|
|
2433
|
+
(candidate) => candidate.artifactKey === entry.artifactKey
|
|
2434
|
+
);
|
|
2435
|
+
if (index < 0) continue;
|
|
2436
|
+
const attemptAtMs = nowMs();
|
|
2437
|
+
entries[index] = {
|
|
2438
|
+
...entries[index],
|
|
2439
|
+
uploadAttempts: entries[index].uploadAttempts + 1,
|
|
2440
|
+
lastAttemptAtMs: attemptAtMs
|
|
2441
|
+
};
|
|
2442
|
+
persist();
|
|
2443
|
+
try {
|
|
2444
|
+
let current = entries[index];
|
|
2445
|
+
let sourceRefreshesThisAttempt = 0;
|
|
2446
|
+
let validatedPrefix;
|
|
2447
|
+
const refreshSourceSnapshot = () => {
|
|
2448
|
+
const snapshot = snapshotCandidate(
|
|
2449
|
+
{
|
|
2450
|
+
artifactType: current.artifactType,
|
|
2451
|
+
source: current.source,
|
|
2452
|
+
sourcePath: current.sourcePath,
|
|
2453
|
+
identityKey: current.identityKey,
|
|
2454
|
+
readPath: current.readPath,
|
|
2455
|
+
sessionId: current.sessionId,
|
|
2456
|
+
metadata: current.metadata
|
|
2457
|
+
},
|
|
2458
|
+
current,
|
|
2459
|
+
nowMs(),
|
|
2460
|
+
maxFileBytes,
|
|
2461
|
+
// Only a definitively removed or no-longer-eligible source may drop
|
|
2462
|
+
// the entry (and its watermark). A transient stat failure throws at
|
|
2463
|
+
// the stat itself — no re-stat race — and surfaces as a failed
|
|
2464
|
+
// attempt, so the retry backoff applies and progress survives.
|
|
2465
|
+
"throw"
|
|
2466
|
+
);
|
|
2467
|
+
if (!snapshot) {
|
|
2468
|
+
entries.splice(index, 1);
|
|
2469
|
+
refreshed += 1;
|
|
2470
|
+
validatedPrefix = void 0;
|
|
2471
|
+
persist();
|
|
2472
|
+
return false;
|
|
2473
|
+
}
|
|
2474
|
+
entries[index] = snapshot.artifactGenerationId === current.artifactGenerationId ? {
|
|
2475
|
+
...snapshot,
|
|
2476
|
+
uploadAttempts: current.uploadAttempts,
|
|
2477
|
+
lastAttemptAtMs: current.lastAttemptAtMs
|
|
2478
|
+
} : snapshot;
|
|
2479
|
+
current = entries[index];
|
|
2480
|
+
refreshed += 1;
|
|
2481
|
+
sourceRefreshesThisAttempt += 1;
|
|
2482
|
+
validatedPrefix = void 0;
|
|
2483
|
+
persist();
|
|
2484
|
+
return sourceRefreshesThisAttempt <= 1;
|
|
2485
|
+
};
|
|
2486
|
+
const resetCurrentGeneration = () => {
|
|
2487
|
+
const reset = startNewGenerationFromEntry(current, nowMs());
|
|
2488
|
+
if (reset) {
|
|
2489
|
+
entries[index] = activateUploadDestination(
|
|
2490
|
+
reset,
|
|
2491
|
+
uploadDestinationId,
|
|
2492
|
+
targetName
|
|
2493
|
+
);
|
|
2494
|
+
current = entries[index];
|
|
2495
|
+
} else {
|
|
2496
|
+
entries.splice(index, 1);
|
|
2497
|
+
}
|
|
2498
|
+
refreshed += 1;
|
|
2499
|
+
validatedPrefix = void 0;
|
|
2500
|
+
persist();
|
|
2501
|
+
};
|
|
2502
|
+
while (uploadedBytes < maxUploadBytes) {
|
|
2503
|
+
if (current.readPath !== void 0 && !safeStat(current.readPath)?.isFile() && statCandidateSource(current.sourcePath, "throw") !== null) {
|
|
2504
|
+
break;
|
|
2505
|
+
}
|
|
2506
|
+
const inFlight = current.inFlightChunk;
|
|
2507
|
+
if (inFlight) {
|
|
2508
|
+
const metadata2 = inFlight.metadata;
|
|
2509
|
+
if (inFlight.uploadDestinationId !== uploadDestinationId || metadata2.artifactGenerationId !== current.artifactGenerationId) {
|
|
2510
|
+
resetCurrentGeneration();
|
|
2511
|
+
break;
|
|
2512
|
+
}
|
|
2513
|
+
const stableRange2 = readFileRangeAtRevision(
|
|
2514
|
+
artifactReadPath(current),
|
|
2515
|
+
metadata2.chunkStartByte,
|
|
2516
|
+
metadata2.chunkEndByte
|
|
2517
|
+
);
|
|
2518
|
+
if (!stableRange2 || stableRange2.bytes.length !== metadata2.chunkSizeBytes || sha256Hex(stableRange2.bytes) !== metadata2.chunkSha256) {
|
|
2519
|
+
resetCurrentGeneration();
|
|
2520
|
+
break;
|
|
2521
|
+
}
|
|
2522
|
+
const chunk2 = stableRange2.bytes;
|
|
2523
|
+
attempted += 1;
|
|
2524
|
+
if (!lock.heartbeat()) {
|
|
2525
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2526
|
+
}
|
|
2527
|
+
const response2 = await postRawArtifact(
|
|
2528
|
+
rawArtifactUrl(options.targetUrl),
|
|
2529
|
+
metadata2,
|
|
2530
|
+
chunk2,
|
|
2531
|
+
rawArtifactRequestHeaders(options),
|
|
2532
|
+
options.fetch
|
|
2533
|
+
);
|
|
2534
|
+
if (!lock.heartbeat()) {
|
|
2535
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2536
|
+
}
|
|
2537
|
+
const checkpoint2 = {
|
|
2538
|
+
chunkStartByte: metadata2.chunkStartByte,
|
|
2539
|
+
chunkEndByte: metadata2.chunkEndByte,
|
|
2540
|
+
chunkSha256: metadata2.chunkSha256
|
|
2541
|
+
};
|
|
2542
|
+
const checkpoints = withAcknowledgedCheckpoint(current, checkpoint2);
|
|
2543
|
+
const sourceRevisionAfter2 = safeSourceRevision(
|
|
2544
|
+
artifactReadPath(current)
|
|
2545
|
+
);
|
|
2546
|
+
const identityPersistenceUnconfirmed2 = rawUploadLeavesFinalIdentityUnconfirmed(
|
|
2547
|
+
current,
|
|
2548
|
+
metadata2,
|
|
2549
|
+
response2
|
|
2550
|
+
);
|
|
2551
|
+
const replayedCurrentFinalChunk = metadata2.chunkEndByte === current.sizeBytes && metadata2.sizeBytes === current.sizeBytes;
|
|
2552
|
+
let complete2 = false;
|
|
2553
|
+
let completedValidation;
|
|
2554
|
+
if (replayedCurrentFinalChunk && sourceRevisionAfter2 && sameSourceRevision(stableRange2.revision, sourceRevisionAfter2)) {
|
|
2555
|
+
const validation = validateCheckpointPrefix({
|
|
2556
|
+
sourcePath: artifactReadPath(current),
|
|
2557
|
+
checkpoints,
|
|
2558
|
+
expectedEndByte: current.sizeBytes,
|
|
2559
|
+
expectedRevision: sourceRevisionAfter2
|
|
2560
|
+
});
|
|
2561
|
+
if (validation.status === "mismatch") {
|
|
2562
|
+
uploadedChunks += 1;
|
|
2563
|
+
uploadedBytes += chunk2.length;
|
|
2564
|
+
resetCurrentGeneration();
|
|
2565
|
+
break;
|
|
2566
|
+
}
|
|
2567
|
+
if (validation.status === "valid" && validation.contentSha256 === metadata2.contentSha256) {
|
|
2568
|
+
complete2 = true;
|
|
2569
|
+
completedValidation = validation;
|
|
2570
|
+
} else if (validation.status === "valid") {
|
|
2571
|
+
uploadedChunks += 1;
|
|
2572
|
+
uploadedBytes += chunk2.length;
|
|
2573
|
+
resetCurrentGeneration();
|
|
2574
|
+
break;
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
entries[index] = {
|
|
2578
|
+
...current,
|
|
2579
|
+
contentSha256: complete2 ? metadata2.contentSha256 : void 0,
|
|
2580
|
+
status: complete2 ? "uploaded" : "pending",
|
|
2581
|
+
uploadedSizeBytes: metadata2.chunkEndByte,
|
|
2582
|
+
uploadedRepositoryContextRevision: acknowledgedRepositoryContextRevision(current, metadata2),
|
|
2583
|
+
uploadedIdentitySignature: rawUploadAcknowledgedIdentitySignature(
|
|
2584
|
+
current,
|
|
2585
|
+
metadata2,
|
|
2586
|
+
response2
|
|
2587
|
+
),
|
|
2588
|
+
lastUploadedChunkStartByte: metadata2.chunkStartByte,
|
|
2589
|
+
lastUploadedChunkEndByte: metadata2.chunkEndByte,
|
|
2590
|
+
lastUploadedChunkSha256: metadata2.chunkSha256,
|
|
2591
|
+
uploadedChunkCheckpoints: checkpoints,
|
|
2592
|
+
uploadDestinationId,
|
|
2593
|
+
uploadTarget: targetName,
|
|
2594
|
+
inFlightChunk: void 0,
|
|
2595
|
+
uploadedAtMs: complete2 ? nowMs() : void 0,
|
|
2596
|
+
// Byte progress normally resets the retry budget, but a duplicate
|
|
2597
|
+
// final chunk without identity confirmation made no progress on
|
|
2598
|
+
// the remaining work and must consume the current attempt.
|
|
2599
|
+
uploadAttempts: identityPersistenceUnconfirmed2 ? current.uploadAttempts : 1,
|
|
2600
|
+
lastAttemptAtMs: entries[index].lastAttemptAtMs,
|
|
2601
|
+
lastError: void 0
|
|
2602
|
+
};
|
|
2603
|
+
current = entries[index];
|
|
2604
|
+
if (identityPersistenceUnconfirmed2) {
|
|
2605
|
+
entries[index] = {
|
|
2606
|
+
...current,
|
|
2607
|
+
status: "failed",
|
|
2608
|
+
uploadedAtMs: void 0,
|
|
2609
|
+
lastError: UNCONFIRMED_IDENTITY_PERSISTENCE_ERROR
|
|
2610
|
+
};
|
|
2611
|
+
current = entries[index];
|
|
2612
|
+
}
|
|
2613
|
+
validatedPrefix = completedValidation ? {
|
|
2614
|
+
artifactGenerationId: current.artifactGenerationId,
|
|
2615
|
+
uploadDestinationId,
|
|
2616
|
+
sourcePath: artifactReadPath(current),
|
|
2617
|
+
revision: completedValidation.revision,
|
|
2618
|
+
verifiedThroughByte: current.uploadedSizeBytes,
|
|
2619
|
+
contentSha256: completedValidation.contentSha256
|
|
2620
|
+
} : void 0;
|
|
2621
|
+
uploadedChunks += 1;
|
|
2622
|
+
uploadedBytes += chunk2.length;
|
|
2623
|
+
if (identityPersistenceUnconfirmed2) failed += 1;
|
|
2624
|
+
else if (complete2) uploaded += 1;
|
|
2625
|
+
persist();
|
|
2626
|
+
if (identityPersistenceUnconfirmed2) break;
|
|
2627
|
+
if (complete2 && !metadataAmendmentNeedsUpload(current)) break;
|
|
2628
|
+
continue;
|
|
2629
|
+
}
|
|
2630
|
+
const stat = safeStat(artifactReadPath(current));
|
|
2631
|
+
const decodedOutlivedSource = current.readPath !== void 0 && statCandidateSource(current.sourcePath, "throw") === null;
|
|
2632
|
+
if (!stat?.isFile() || decodedOutlivedSource) {
|
|
2633
|
+
entries.splice(index, 1);
|
|
2634
|
+
refreshed += 1;
|
|
2635
|
+
persist();
|
|
2636
|
+
break;
|
|
2637
|
+
}
|
|
2638
|
+
const mtimeMs = statMtimeMs(stat);
|
|
2639
|
+
if (stat.size !== current.sizeBytes || mtimeMs !== current.mtimeMs) {
|
|
2640
|
+
if (!refreshSourceSnapshot()) break;
|
|
2641
|
+
}
|
|
2642
|
+
const sourceRevision = safeSourceRevision(artifactReadPath(current));
|
|
2643
|
+
if (!sourceRevision) {
|
|
2644
|
+
entries.splice(index, 1);
|
|
2645
|
+
refreshed += 1;
|
|
2646
|
+
persist();
|
|
2647
|
+
break;
|
|
2648
|
+
}
|
|
2649
|
+
if (!validatedPrefixStillApplies(
|
|
2650
|
+
validatedPrefix,
|
|
2651
|
+
current,
|
|
2652
|
+
uploadDestinationId,
|
|
2653
|
+
sourceRevision
|
|
2654
|
+
)) {
|
|
2655
|
+
const validation = validateCheckpointPrefix({
|
|
2656
|
+
sourcePath: artifactReadPath(current),
|
|
2657
|
+
checkpoints: acknowledgedCheckpoints(current),
|
|
2658
|
+
expectedEndByte: current.uploadedSizeBytes,
|
|
2659
|
+
expectedRevision: sourceRevision
|
|
2660
|
+
});
|
|
2661
|
+
if (validation.status === "mismatch") {
|
|
2662
|
+
resetCurrentGeneration();
|
|
2663
|
+
break;
|
|
2664
|
+
}
|
|
2665
|
+
if (validation.status === "changed") {
|
|
2666
|
+
if (!refreshSourceSnapshot()) break;
|
|
2667
|
+
continue;
|
|
2668
|
+
}
|
|
2669
|
+
validatedPrefix = {
|
|
2670
|
+
artifactGenerationId: current.artifactGenerationId,
|
|
2671
|
+
uploadDestinationId,
|
|
2672
|
+
sourcePath: artifactReadPath(current),
|
|
2673
|
+
revision: validation.revision,
|
|
2674
|
+
verifiedThroughByte: current.uploadedSizeBytes,
|
|
2675
|
+
contentSha256: validation.contentSha256
|
|
2676
|
+
};
|
|
2677
|
+
}
|
|
2678
|
+
if (current.uploadedSizeBytes >= current.sizeBytes) {
|
|
2679
|
+
if (metadataAmendmentNeedsUpload(current)) {
|
|
2680
|
+
const finalCheckpoint = acknowledgedCheckpoints(current).find(
|
|
2681
|
+
(checkpoint2) => checkpoint2.chunkEndByte === current.sizeBytes
|
|
2682
|
+
);
|
|
2683
|
+
if (!finalCheckpoint || !validatedPrefix || !validatedPrefix.contentSha256) {
|
|
2684
|
+
resetCurrentGeneration();
|
|
2685
|
+
break;
|
|
2686
|
+
}
|
|
2687
|
+
const stableRange2 = readFileRangeAtRevision(
|
|
2688
|
+
artifactReadPath(current),
|
|
2689
|
+
finalCheckpoint.chunkStartByte,
|
|
2690
|
+
finalCheckpoint.chunkEndByte,
|
|
2691
|
+
validatedPrefix.revision
|
|
2692
|
+
);
|
|
2693
|
+
if (!stableRange2 || sha256Hex(stableRange2.bytes) !== finalCheckpoint.chunkSha256) {
|
|
2694
|
+
resetCurrentGeneration();
|
|
2695
|
+
break;
|
|
2696
|
+
}
|
|
2697
|
+
const metadata2 = uploadMetadata({
|
|
2698
|
+
entry: current,
|
|
2699
|
+
chunkStartByte: finalCheckpoint.chunkStartByte,
|
|
2700
|
+
chunkEndByte: finalCheckpoint.chunkEndByte,
|
|
2701
|
+
chunk: stableRange2.bytes,
|
|
2702
|
+
contentSha256: validatedPrefix.contentSha256
|
|
2703
|
+
});
|
|
2704
|
+
entries[index] = {
|
|
2705
|
+
...current,
|
|
2706
|
+
inFlightChunk: { uploadDestinationId, metadata: metadata2 }
|
|
2707
|
+
};
|
|
2708
|
+
current = entries[index];
|
|
2709
|
+
persist();
|
|
2710
|
+
continue;
|
|
2711
|
+
}
|
|
2712
|
+
entries[index] = {
|
|
2713
|
+
...current,
|
|
2714
|
+
status: "uploaded",
|
|
2715
|
+
uploadDestinationId,
|
|
2716
|
+
uploadTarget: targetName,
|
|
2717
|
+
uploadedAtMs: nowMs(),
|
|
2718
|
+
lastError: void 0
|
|
2719
|
+
};
|
|
2720
|
+
uploaded += 1;
|
|
2721
|
+
persist();
|
|
2722
|
+
break;
|
|
2723
|
+
}
|
|
2724
|
+
if (maxUploadBytes - uploadedBytes <= 0) break;
|
|
2725
|
+
const generationChunkSizeBytes = current.uploadChunkSizeBytes ?? chunkSizeBytes;
|
|
2726
|
+
if (current.uploadChunkSizeBytes === void 0) {
|
|
2727
|
+
entries[index] = {
|
|
2728
|
+
...current,
|
|
2729
|
+
uploadChunkSizeBytes: generationChunkSizeBytes
|
|
2730
|
+
};
|
|
2731
|
+
current = entries[index];
|
|
2732
|
+
persist();
|
|
2733
|
+
}
|
|
2734
|
+
const chunkStartByte = current.uploadedSizeBytes;
|
|
2735
|
+
const chunkEndByte = Math.min(
|
|
2736
|
+
current.sizeBytes,
|
|
2737
|
+
chunkStartByte + generationChunkSizeBytes
|
|
2738
|
+
);
|
|
2739
|
+
const stableRange = readFileRangeAtRevision(
|
|
2740
|
+
artifactReadPath(current),
|
|
2741
|
+
chunkStartByte,
|
|
2742
|
+
chunkEndByte,
|
|
2743
|
+
sourceRevision
|
|
2744
|
+
);
|
|
2745
|
+
if (!stableRange) {
|
|
2746
|
+
if (!refreshSourceSnapshot()) break;
|
|
2747
|
+
continue;
|
|
2748
|
+
}
|
|
2749
|
+
const chunk = stableRange.bytes;
|
|
2750
|
+
const complete = chunkEndByte >= current.sizeBytes;
|
|
2751
|
+
const checkpoint = {
|
|
2752
|
+
chunkStartByte,
|
|
2753
|
+
chunkEndByte,
|
|
2754
|
+
chunkSha256: sha256Hex(chunk)
|
|
2755
|
+
};
|
|
2756
|
+
let contentSha256;
|
|
2757
|
+
if (complete) {
|
|
2758
|
+
const validation = validateCheckpointPrefix({
|
|
2759
|
+
sourcePath: artifactReadPath(current),
|
|
2760
|
+
checkpoints: withAcknowledgedCheckpoint(current, checkpoint),
|
|
2761
|
+
expectedEndByte: current.sizeBytes,
|
|
2762
|
+
expectedRevision: stableRange.revision
|
|
2763
|
+
});
|
|
2764
|
+
if (validation.status === "mismatch") {
|
|
2765
|
+
resetCurrentGeneration();
|
|
2766
|
+
break;
|
|
2767
|
+
}
|
|
2768
|
+
if (validation.status === "changed") {
|
|
2769
|
+
if (!refreshSourceSnapshot()) break;
|
|
2770
|
+
continue;
|
|
2771
|
+
}
|
|
2772
|
+
contentSha256 = validation.contentSha256;
|
|
2773
|
+
}
|
|
2774
|
+
const metadata = uploadMetadata({
|
|
2775
|
+
entry: current,
|
|
2776
|
+
chunkStartByte,
|
|
2777
|
+
chunkEndByte,
|
|
2778
|
+
chunk,
|
|
2779
|
+
contentSha256
|
|
2780
|
+
});
|
|
2781
|
+
entries[index] = {
|
|
2782
|
+
...current,
|
|
2783
|
+
inFlightChunk: {
|
|
2784
|
+
uploadDestinationId,
|
|
2785
|
+
metadata
|
|
2786
|
+
}
|
|
2787
|
+
};
|
|
2788
|
+
current = entries[index];
|
|
2789
|
+
persist();
|
|
2790
|
+
attempted += 1;
|
|
2791
|
+
if (!lock.heartbeat()) {
|
|
2792
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2793
|
+
}
|
|
2794
|
+
const response = await postRawArtifact(
|
|
2795
|
+
rawArtifactUrl(options.targetUrl),
|
|
2796
|
+
metadata,
|
|
2797
|
+
chunk,
|
|
2798
|
+
rawArtifactRequestHeaders(options),
|
|
2799
|
+
options.fetch
|
|
2800
|
+
);
|
|
2801
|
+
if (!lock.heartbeat()) {
|
|
2802
|
+
throw new Error("artifact queue lock ownership was lost");
|
|
2803
|
+
}
|
|
2804
|
+
const sourceRevisionAfter = safeSourceRevision(
|
|
2805
|
+
artifactReadPath(current)
|
|
2806
|
+
);
|
|
2807
|
+
const acknowledgedComplete = complete && sourceRevisionAfter !== null && sameSourceRevision(stableRange.revision, sourceRevisionAfter);
|
|
2808
|
+
const identityPersistenceUnconfirmed = rawUploadLeavesFinalIdentityUnconfirmed(current, metadata, response);
|
|
2809
|
+
entries[index] = {
|
|
2810
|
+
...current,
|
|
2811
|
+
contentSha256: acknowledgedComplete ? contentSha256 : void 0,
|
|
2812
|
+
status: acknowledgedComplete ? "uploaded" : "pending",
|
|
2813
|
+
uploadChunkSizeBytes: generationChunkSizeBytes,
|
|
2814
|
+
uploadedSizeBytes: chunkEndByte,
|
|
2815
|
+
uploadedRepositoryContextRevision: acknowledgedRepositoryContextRevision(current, metadata),
|
|
2816
|
+
uploadedIdentitySignature: rawUploadAcknowledgedIdentitySignature(
|
|
2817
|
+
current,
|
|
2818
|
+
metadata,
|
|
2819
|
+
response
|
|
2820
|
+
),
|
|
2821
|
+
lastUploadedChunkStartByte: chunkStartByte,
|
|
2822
|
+
lastUploadedChunkEndByte: chunkEndByte,
|
|
2823
|
+
lastUploadedChunkSha256: metadata.chunkSha256,
|
|
2824
|
+
uploadedChunkCheckpoints: withAcknowledgedCheckpoint(
|
|
2825
|
+
current,
|
|
2826
|
+
checkpoint
|
|
2827
|
+
),
|
|
2828
|
+
uploadDestinationId,
|
|
2829
|
+
uploadTarget: targetName,
|
|
2830
|
+
inFlightChunk: void 0,
|
|
2831
|
+
uploadedAtMs: acknowledgedComplete ? nowMs() : void 0,
|
|
2832
|
+
// Byte progress normally resets the retry budget, but a duplicate
|
|
2833
|
+
// final chunk without identity confirmation made no progress on
|
|
2834
|
+
// the remaining work and must consume the current attempt.
|
|
2835
|
+
uploadAttempts: identityPersistenceUnconfirmed ? current.uploadAttempts : 1,
|
|
2836
|
+
lastAttemptAtMs: entries[index].lastAttemptAtMs,
|
|
2837
|
+
lastError: void 0
|
|
2838
|
+
};
|
|
2839
|
+
current = entries[index];
|
|
2840
|
+
if (identityPersistenceUnconfirmed) {
|
|
2841
|
+
entries[index] = {
|
|
2842
|
+
...current,
|
|
2843
|
+
status: "failed",
|
|
2844
|
+
uploadedAtMs: void 0,
|
|
2845
|
+
lastError: UNCONFIRMED_IDENTITY_PERSISTENCE_ERROR
|
|
2846
|
+
};
|
|
2847
|
+
current = entries[index];
|
|
2848
|
+
}
|
|
2849
|
+
validatedPrefix = sourceRevisionAfter && sameSourceRevision(stableRange.revision, sourceRevisionAfter) ? {
|
|
2850
|
+
artifactGenerationId: current.artifactGenerationId,
|
|
2851
|
+
uploadDestinationId,
|
|
2852
|
+
sourcePath: artifactReadPath(current),
|
|
2853
|
+
revision: sourceRevisionAfter,
|
|
2854
|
+
verifiedThroughByte: current.uploadedSizeBytes,
|
|
2855
|
+
contentSha256
|
|
2856
|
+
} : void 0;
|
|
2857
|
+
uploadedChunks += 1;
|
|
2858
|
+
uploadedBytes += chunk.length;
|
|
2859
|
+
if (identityPersistenceUnconfirmed) failed += 1;
|
|
2860
|
+
else if (acknowledgedComplete) uploaded += 1;
|
|
2861
|
+
persist();
|
|
2862
|
+
if (identityPersistenceUnconfirmed) break;
|
|
2863
|
+
if (acknowledgedComplete) break;
|
|
2864
|
+
}
|
|
2865
|
+
} catch (error) {
|
|
2866
|
+
const failureIndex = entries.findIndex(
|
|
2867
|
+
(candidate) => candidate.artifactKey === entry.artifactKey
|
|
2868
|
+
);
|
|
2869
|
+
if (isAbortError(error)) {
|
|
2870
|
+
if (failureIndex >= 0) {
|
|
2871
|
+
const current = entries[failureIndex];
|
|
2872
|
+
entries[failureIndex] = {
|
|
2873
|
+
...current,
|
|
2874
|
+
// Cancellation is a lifecycle event, not an upload failure. Undo
|
|
2875
|
+
// this run's attempt bookkeeping while retaining the current
|
|
2876
|
+
// persisted request descriptor for an exact replay. Restoring the
|
|
2877
|
+
// selected entry also preserves a prior real failure and backoff.
|
|
2878
|
+
status: entry.status,
|
|
2879
|
+
uploadAttempts: entry.uploadAttempts,
|
|
2880
|
+
lastAttemptAtMs: entry.lastAttemptAtMs,
|
|
2881
|
+
lastError: entry.lastError
|
|
2882
|
+
};
|
|
2883
|
+
persist();
|
|
2884
|
+
}
|
|
2885
|
+
throw error;
|
|
2886
|
+
}
|
|
2887
|
+
if (failureIndex < 0) {
|
|
2888
|
+
failed += 1;
|
|
2889
|
+
persist();
|
|
2890
|
+
continue;
|
|
2891
|
+
}
|
|
2892
|
+
const identityReplayConflict = entry.uploadedSizeBytes >= entry.sizeBytes && identityNeedsUpload(entry);
|
|
2893
|
+
if (error instanceof RawArtifactPostError && error.status === 409 && error.code === "CONFLICT" && !identityReplayConflict) {
|
|
2894
|
+
const reset = startNewGenerationFromEntry(
|
|
2895
|
+
entries[failureIndex],
|
|
2896
|
+
nowMs()
|
|
2897
|
+
);
|
|
2898
|
+
if (reset) {
|
|
2899
|
+
entries[failureIndex] = activateUploadDestination(
|
|
2900
|
+
reset,
|
|
2901
|
+
uploadDestinationId,
|
|
2902
|
+
targetName
|
|
2903
|
+
);
|
|
2904
|
+
} else {
|
|
2905
|
+
entries.splice(failureIndex, 1);
|
|
2906
|
+
}
|
|
2907
|
+
refreshed += 1;
|
|
2908
|
+
persist();
|
|
2909
|
+
continue;
|
|
2910
|
+
}
|
|
2911
|
+
if (error instanceof RawArtifactPostError && error.status === 429) {
|
|
2912
|
+
const throttledAtMs = nowMs();
|
|
2913
|
+
retryAtMs = retryAtMsFromHeader(error.retryAfter, throttledAtMs);
|
|
2914
|
+
uploadThrottles[uploadDestinationId] = {
|
|
2915
|
+
retryAtMs,
|
|
2916
|
+
status: 429,
|
|
2917
|
+
code: error.code
|
|
2918
|
+
};
|
|
2919
|
+
const current = entries[failureIndex];
|
|
2920
|
+
if (current.artifactGenerationId === entry.artifactGenerationId && current.uploadedSizeBytes === entry.uploadedSizeBytes) {
|
|
2921
|
+
entries[failureIndex] = {
|
|
2922
|
+
...current,
|
|
2923
|
+
status: entry.status,
|
|
2924
|
+
uploadAttempts: entry.uploadAttempts,
|
|
2925
|
+
lastAttemptAtMs: entry.lastAttemptAtMs,
|
|
2926
|
+
lastError: entry.lastError
|
|
2927
|
+
};
|
|
2928
|
+
}
|
|
2929
|
+
persist();
|
|
2930
|
+
break;
|
|
2931
|
+
}
|
|
2932
|
+
entries[failureIndex] = {
|
|
2933
|
+
...entries[failureIndex],
|
|
2934
|
+
status: "failed",
|
|
2935
|
+
lastError: error instanceof Error ? error.message : String(error)
|
|
2936
|
+
};
|
|
2937
|
+
failed += 1;
|
|
2938
|
+
persist();
|
|
2939
|
+
}
|
|
2940
|
+
}
|
|
2941
|
+
const nextQueue = readArtifactQueue(options.queueFile);
|
|
2942
|
+
const completedAtMs = nowMs();
|
|
2943
|
+
const exhausted = nextQueue.artifacts.filter(
|
|
2944
|
+
(entry) => isExhaustedForDestination(entry, uploadDestinationId, retryExhausted)
|
|
2945
|
+
).length;
|
|
2946
|
+
return {
|
|
2947
|
+
attempted,
|
|
2948
|
+
uploaded,
|
|
2949
|
+
uploadedChunks,
|
|
2950
|
+
uploadedBytes,
|
|
2951
|
+
failed,
|
|
2952
|
+
exhausted,
|
|
2953
|
+
refreshed,
|
|
2954
|
+
retryAtMs,
|
|
2955
|
+
nextRetryAtMs: nextRetryAtMsForDestination(
|
|
2956
|
+
nextQueue.artifacts,
|
|
2957
|
+
uploadDestinationId,
|
|
2958
|
+
completedAtMs,
|
|
2959
|
+
retryExhausted
|
|
2960
|
+
),
|
|
2961
|
+
hasRunnable: retryAtMs === void 0 && nextQueue.artifacts.some(
|
|
2962
|
+
(entry) => isRunnableForDestination(
|
|
2963
|
+
entry,
|
|
2964
|
+
uploadDestinationId,
|
|
2965
|
+
completedAtMs,
|
|
2966
|
+
retryExhausted
|
|
2967
|
+
)
|
|
2968
|
+
),
|
|
2969
|
+
hasMore: nextQueue.artifacts.some(
|
|
2970
|
+
(entry) => needsUploadForDestination(entry, uploadDestinationId)
|
|
2971
|
+
),
|
|
2972
|
+
skipped: false,
|
|
2973
|
+
queue: nextQueue
|
|
2974
|
+
};
|
|
2975
|
+
}
|
|
2976
|
+
|
|
2977
|
+
// ../../packages/panopticon/src/raw-sync.ts
|
|
2978
|
+
import path18 from "path";
|
|
2979
|
+
|
|
2980
|
+
// ../../packages/panopticon/src/artifact-materializer.ts
|
|
2981
|
+
import fs8 from "fs";
|
|
2982
|
+
function entryNeedsBytes(entry, amendmentPending) {
|
|
2983
|
+
if (entry.uploadedSizeBytes < entry.sizeBytes) return true;
|
|
2984
|
+
return amendmentPending(entry);
|
|
2985
|
+
}
|
|
2986
|
+
function createArtifactMaterializer(input) {
|
|
2987
|
+
const maxTotalBytes = input.maxTotalBytes ?? DEFAULT_MATERIALIZED_CACHE_BYTES;
|
|
2988
|
+
const satisfiedSources = new Set(
|
|
2989
|
+
input.entries.filter((entry) => !entryNeedsBytes(entry, input.amendmentPending)).map((entry) => entry.sourcePath)
|
|
2990
|
+
);
|
|
2991
|
+
const retained = /* @__PURE__ */ new Set();
|
|
2992
|
+
let spentBytes = 0;
|
|
2993
|
+
const unsupportedReason = (input.supportsCompression ?? supportsCompressedArtifacts)() ? void 0 : `compressed artifacts need Node 22.15 or newer (running ${process.version})`;
|
|
2994
|
+
const chargeExisting = (cachePath) => {
|
|
2995
|
+
try {
|
|
2996
|
+
const stat = fs8.statSync(cachePath);
|
|
2997
|
+
if (!stat.isFile()) return false;
|
|
2998
|
+
if (!retained.has(cachePath)) {
|
|
2999
|
+
retained.add(cachePath);
|
|
3000
|
+
spentBytes += stat.size;
|
|
3001
|
+
}
|
|
3002
|
+
return true;
|
|
3003
|
+
} catch {
|
|
3004
|
+
return false;
|
|
3005
|
+
}
|
|
3006
|
+
};
|
|
3007
|
+
return {
|
|
3008
|
+
unsupportedReason,
|
|
3009
|
+
materialize: (sourcePath) => {
|
|
3010
|
+
if (unsupportedReason) return null;
|
|
3011
|
+
if (satisfiedSources.has(sourcePath)) return null;
|
|
3012
|
+
const cachePath = materializedArtifactPath({
|
|
3013
|
+
sourcePath,
|
|
3014
|
+
cacheDir: input.cacheDir
|
|
3015
|
+
});
|
|
3016
|
+
if (cachePath && chargeExisting(cachePath)) return cachePath;
|
|
3017
|
+
if (spentBytes >= maxTotalBytes) return null;
|
|
3018
|
+
const materialized = materializeCompressedArtifact({
|
|
3019
|
+
sourcePath,
|
|
3020
|
+
cacheDir: input.cacheDir
|
|
3021
|
+
});
|
|
3022
|
+
if (!materialized) return null;
|
|
3023
|
+
chargeExisting(materialized);
|
|
3024
|
+
return materialized;
|
|
3025
|
+
},
|
|
3026
|
+
retained: () => retained
|
|
3027
|
+
};
|
|
3028
|
+
}
|
|
3029
|
+
|
|
3030
|
+
// ../../packages/panopticon/src/raw-sync.ts
|
|
3031
|
+
var DEFAULT_RAW_SYNC_MAX_DRAIN_PASSES = 10;
|
|
3032
|
+
var DEFAULT_RAW_SYNC_MAX_DRAIN_BYTES = 100 * 1024 * 1024;
|
|
3033
|
+
var DEFAULT_RAW_SYNC_MAX_DRAIN_MS = 6e4;
|
|
3034
|
+
function getRawArtifactSyncPaths(stateDir) {
|
|
3035
|
+
if (stateDir.length === 0) {
|
|
3036
|
+
throw new Error("raw artifact sync state directory must not be empty");
|
|
3037
|
+
}
|
|
3038
|
+
const root = path18.resolve(stateDir);
|
|
3039
|
+
const managed = getManagedClientPaths({ PANOPTICON_HOME: root });
|
|
3040
|
+
return {
|
|
3041
|
+
root,
|
|
3042
|
+
settingsFile: managed.syncSettingsFile,
|
|
3043
|
+
queueFile: managed.artifactQueue,
|
|
3044
|
+
artifactCacheDir: managed.artifactCache
|
|
3045
|
+
};
|
|
3046
|
+
}
|
|
3047
|
+
function summarizeArtifactQueue(queue) {
|
|
3048
|
+
let pending = 0;
|
|
3049
|
+
let uploaded = 0;
|
|
3050
|
+
let failed = 0;
|
|
3051
|
+
let rawPending = 0;
|
|
3052
|
+
let metadataPending = 0;
|
|
3053
|
+
let uploadedBytes = 0;
|
|
3054
|
+
let totalBytes = 0;
|
|
3055
|
+
for (const artifact of queue.artifacts) {
|
|
3056
|
+
if (artifact.status === "pending") pending += 1;
|
|
3057
|
+
if (artifact.status === "uploaded") uploaded += 1;
|
|
3058
|
+
if (artifact.status === "failed") failed += 1;
|
|
3059
|
+
if (artifact.uploadedSizeBytes < artifact.sizeBytes) rawPending += 1;
|
|
3060
|
+
if (artifact.uploadedSizeBytes >= artifact.sizeBytes && metadataAmendmentNeedsUpload(artifact)) {
|
|
3061
|
+
metadataPending += 1;
|
|
3062
|
+
}
|
|
3063
|
+
uploadedBytes += Math.min(artifact.uploadedSizeBytes, artifact.sizeBytes);
|
|
3064
|
+
totalBytes += artifact.sizeBytes;
|
|
3065
|
+
}
|
|
3066
|
+
return {
|
|
3067
|
+
tracked: queue.artifacts.length,
|
|
3068
|
+
pending,
|
|
3069
|
+
uploaded,
|
|
3070
|
+
failed,
|
|
3071
|
+
rawPending,
|
|
3072
|
+
metadataPending,
|
|
3073
|
+
uploadedBytes,
|
|
3074
|
+
totalBytes,
|
|
3075
|
+
remainingBytes: Math.max(0, totalBytes - uploadedBytes)
|
|
3076
|
+
};
|
|
3077
|
+
}
|
|
3078
|
+
function mergeOversizedArtifacts(...groups) {
|
|
3079
|
+
const byPath = /* @__PURE__ */ new Map();
|
|
3080
|
+
for (const group of groups) {
|
|
3081
|
+
for (const artifact of group) byPath.set(artifact.sourcePath, artifact);
|
|
3082
|
+
}
|
|
3083
|
+
return [...byPath.values()];
|
|
3084
|
+
}
|
|
3085
|
+
function uploadOutcome(upload) {
|
|
3086
|
+
if (upload.skipped) return "busy";
|
|
3087
|
+
if (upload.failed > 0) return "failed";
|
|
3088
|
+
if (upload.exhausted > 0) return "exhausted";
|
|
3089
|
+
if (upload.retryAtMs !== void 0) return "throttled";
|
|
3090
|
+
if (!upload.hasMore) return "drained";
|
|
3091
|
+
return upload.hasRunnable ? "pending" : "backing_off";
|
|
3092
|
+
}
|
|
3093
|
+
function positiveInteger(value, fallback) {
|
|
3094
|
+
return value === void 0 || !Number.isFinite(value) ? fallback : Math.max(1, Math.floor(value));
|
|
3095
|
+
}
|
|
3096
|
+
function createRawArtifactSync(options) {
|
|
3097
|
+
const paths = getRawArtifactSyncPaths(options.stateDir);
|
|
3098
|
+
const maxDrainPasses = positiveInteger(
|
|
3099
|
+
options.maxDrainPasses,
|
|
3100
|
+
DEFAULT_RAW_SYNC_MAX_DRAIN_PASSES
|
|
3101
|
+
);
|
|
3102
|
+
const maxDrainBytes = positiveInteger(
|
|
3103
|
+
options.maxDrainBytes,
|
|
3104
|
+
DEFAULT_RAW_SYNC_MAX_DRAIN_BYTES
|
|
3105
|
+
);
|
|
3106
|
+
const maxDrainMs = positiveInteger(
|
|
3107
|
+
options.maxDrainMs,
|
|
3108
|
+
DEFAULT_RAW_SYNC_MAX_DRAIN_MS
|
|
3109
|
+
);
|
|
3110
|
+
let lastMaterializer;
|
|
3111
|
+
const getSettings = () => readSyncSettings(paths.settingsFile);
|
|
3112
|
+
const status = () => ({
|
|
3113
|
+
settings: getSettings(),
|
|
3114
|
+
queue: summarizeArtifactQueue(readArtifactQueue(paths.queueFile))
|
|
3115
|
+
});
|
|
3116
|
+
const discover = () => {
|
|
3117
|
+
const materializer = createArtifactMaterializer({
|
|
3118
|
+
cacheDir: paths.artifactCacheDir,
|
|
3119
|
+
entries: readArtifactQueue(paths.queueFile).artifacts,
|
|
3120
|
+
amendmentPending: metadataAmendmentNeedsUpload,
|
|
3121
|
+
maxTotalBytes: options.maxArtifactCacheBytes
|
|
3122
|
+
});
|
|
3123
|
+
lastMaterializer = materializer;
|
|
3124
|
+
const discovery = discoverArtifacts({
|
|
3125
|
+
homeDir: options.homeDir,
|
|
3126
|
+
projectRoots: options.projectRoots,
|
|
3127
|
+
maxFileBytes: options.maxFileBytes,
|
|
3128
|
+
materializeArtifact: materializer.materialize
|
|
3129
|
+
});
|
|
3130
|
+
return materializer.unsupportedReason ? {
|
|
3131
|
+
...discovery,
|
|
3132
|
+
compressedArtifactsUnsupportedReason: materializer.unsupportedReason
|
|
3133
|
+
} : discovery;
|
|
3134
|
+
};
|
|
3135
|
+
return {
|
|
3136
|
+
paths,
|
|
3137
|
+
getSettings,
|
|
3138
|
+
setEnabled: (enabled) => setSyncEnabled(paths.settingsFile, enabled),
|
|
3139
|
+
status,
|
|
3140
|
+
discover,
|
|
3141
|
+
reset: () => {
|
|
3142
|
+
const result = resetArtifactQueue({
|
|
3143
|
+
queueFile: paths.queueFile,
|
|
3144
|
+
nowMs: options.nowMs,
|
|
3145
|
+
isPidAlive: options.isPidAlive
|
|
3146
|
+
});
|
|
3147
|
+
return {
|
|
3148
|
+
reset: result.reset,
|
|
3149
|
+
removed: result.removed,
|
|
3150
|
+
skipped: result.skipped,
|
|
3151
|
+
status: {
|
|
3152
|
+
settings: getSettings(),
|
|
3153
|
+
queue: summarizeArtifactQueue(result.queue)
|
|
3154
|
+
}
|
|
3155
|
+
};
|
|
3156
|
+
},
|
|
3157
|
+
run: async (transport) => {
|
|
3158
|
+
if (!getSettings().enabled) {
|
|
3159
|
+
return {
|
|
3160
|
+
outcome: "disabled",
|
|
3161
|
+
enabled: false,
|
|
3162
|
+
syncPasses: 0,
|
|
3163
|
+
drained: false,
|
|
3164
|
+
status: status()
|
|
3165
|
+
};
|
|
3166
|
+
}
|
|
3167
|
+
const discovery = discover();
|
|
3168
|
+
const syncOptions = {
|
|
3169
|
+
queueFile: paths.queueFile,
|
|
3170
|
+
candidates: discovery.candidates,
|
|
3171
|
+
maxFileBytes: options.maxFileBytes,
|
|
3172
|
+
observerIdentity: options.observerIdentity,
|
|
3173
|
+
retryExhausted: options.retryExhausted,
|
|
3174
|
+
nowMs: options.nowMs,
|
|
3175
|
+
isPidAlive: options.isPidAlive,
|
|
3176
|
+
targetUrl: transport.targetUrl,
|
|
3177
|
+
targetName: transport.targetName ?? transport.targetUrl,
|
|
3178
|
+
authorization: transport.authorization,
|
|
3179
|
+
organization: transport.organization,
|
|
3180
|
+
destinationIdentity: transport.destinationIdentity,
|
|
3181
|
+
fetch: transport.fetch
|
|
3182
|
+
};
|
|
3183
|
+
const startedAtMs = (options.nowMs ?? Date.now)();
|
|
3184
|
+
const runPass = async (uploadedSoFar) => await syncArtifacts({
|
|
3185
|
+
...syncOptions,
|
|
3186
|
+
maxUploadBytes: Math.min(
|
|
3187
|
+
DEFAULT_UPLOAD_BYTE_LIMIT,
|
|
3188
|
+
Math.max(1, maxDrainBytes - uploadedSoFar)
|
|
3189
|
+
)
|
|
3190
|
+
});
|
|
3191
|
+
let sync = await runPass(0);
|
|
3192
|
+
let syncPasses = 1;
|
|
3193
|
+
let snapshotQueued = sync.snapshot.queued;
|
|
3194
|
+
let snapshotSkipped = sync.snapshot.skipped;
|
|
3195
|
+
let attempted = sync.upload.attempted;
|
|
3196
|
+
let uploaded = sync.upload.uploaded;
|
|
3197
|
+
let uploadedChunks = sync.upload.uploadedChunks;
|
|
3198
|
+
let uploadedBytes = sync.upload.uploadedBytes;
|
|
3199
|
+
let failed = sync.upload.failed;
|
|
3200
|
+
let exhausted = sync.upload.exhausted;
|
|
3201
|
+
let refreshed = sync.upload.refreshed;
|
|
3202
|
+
let oversized = mergeOversizedArtifacts(
|
|
3203
|
+
discovery.oversized,
|
|
3204
|
+
sync.snapshot.oversized
|
|
3205
|
+
);
|
|
3206
|
+
while (sync.upload.hasMore && sync.upload.hasRunnable && !sync.upload.skipped && sync.upload.failed === 0 && sync.upload.exhausted === 0 && sync.upload.retryAtMs === void 0 && syncPasses < maxDrainPasses && uploadedBytes < maxDrainBytes && (options.nowMs ?? Date.now)() - startedAtMs < maxDrainMs && getSettings().enabled && (sync.upload.attempted > 0 || sync.upload.uploadedChunks > 0 || sync.upload.refreshed > 0)) {
|
|
3207
|
+
sync = await runPass(uploadedBytes);
|
|
3208
|
+
syncPasses += 1;
|
|
3209
|
+
snapshotQueued += sync.snapshot.queued;
|
|
3210
|
+
snapshotSkipped += sync.snapshot.skipped;
|
|
3211
|
+
attempted += sync.upload.attempted;
|
|
3212
|
+
uploaded += sync.upload.uploaded;
|
|
3213
|
+
uploadedChunks += sync.upload.uploadedChunks;
|
|
3214
|
+
uploadedBytes += sync.upload.uploadedBytes;
|
|
3215
|
+
failed += sync.upload.failed;
|
|
3216
|
+
exhausted = sync.upload.exhausted;
|
|
3217
|
+
refreshed += sync.upload.refreshed;
|
|
3218
|
+
oversized = mergeOversizedArtifacts(oversized, sync.snapshot.oversized);
|
|
3219
|
+
}
|
|
3220
|
+
pruneMaterializedCache({
|
|
3221
|
+
cacheDir: paths.artifactCacheDir,
|
|
3222
|
+
maxTotalBytes: options.maxArtifactCacheBytes,
|
|
3223
|
+
keepPaths: lastMaterializer?.retained()
|
|
3224
|
+
});
|
|
3225
|
+
const upload = {
|
|
3226
|
+
attempted,
|
|
3227
|
+
uploaded,
|
|
3228
|
+
uploadedChunks,
|
|
3229
|
+
uploadedBytes,
|
|
3230
|
+
failed,
|
|
3231
|
+
exhausted,
|
|
3232
|
+
refreshed,
|
|
3233
|
+
retryAtMs: sync.upload.retryAtMs,
|
|
3234
|
+
nextRetryAtMs: sync.upload.nextRetryAtMs,
|
|
3235
|
+
hasRunnable: sync.upload.hasRunnable,
|
|
3236
|
+
hasMore: sync.upload.hasMore,
|
|
3237
|
+
skipped: sync.upload.skipped
|
|
3238
|
+
};
|
|
3239
|
+
const outcome = uploadOutcome(upload);
|
|
3240
|
+
return {
|
|
3241
|
+
outcome,
|
|
3242
|
+
enabled: true,
|
|
3243
|
+
syncPasses,
|
|
3244
|
+
drained: outcome === "drained",
|
|
3245
|
+
discovery,
|
|
3246
|
+
oversized,
|
|
3247
|
+
snapshot: {
|
|
3248
|
+
queued: snapshotQueued,
|
|
3249
|
+
skipped: snapshotSkipped
|
|
3250
|
+
},
|
|
3251
|
+
upload,
|
|
3252
|
+
status: {
|
|
3253
|
+
settings: getSettings(),
|
|
3254
|
+
queue: summarizeArtifactQueue(sync.upload.queue)
|
|
3255
|
+
}
|
|
3256
|
+
};
|
|
3257
|
+
}
|
|
3258
|
+
};
|
|
3259
|
+
}
|
|
3260
|
+
|
|
3261
|
+
// src/sync/runner.ts
|
|
3262
|
+
function createCliRawSync(opts = {}) {
|
|
3263
|
+
return createRawArtifactSync({
|
|
3264
|
+
stateDir: FML_OBSERVER_DIR,
|
|
3265
|
+
retryExhausted: opts.continuous ?? false,
|
|
3266
|
+
observerIdentity: collectObserverIdentity(FML_INSTALLATION_ID_FILE) ?? void 0
|
|
3267
|
+
});
|
|
3268
|
+
}
|
|
3269
|
+
async function runRawSyncOnce(opts = {}) {
|
|
3270
|
+
const sync = createCliRawSync({ continuous: opts.continuous });
|
|
3271
|
+
if (!sync.getSettings().enabled) {
|
|
3272
|
+
return { kind: "disabled", status: sync.status() };
|
|
3273
|
+
}
|
|
3274
|
+
const environment = getActiveEnv();
|
|
3275
|
+
const token = await getValidToken({ env: environment.name });
|
|
3276
|
+
if (!token) {
|
|
3277
|
+
return {
|
|
3278
|
+
kind: "unauthenticated",
|
|
3279
|
+
environment: environment.name,
|
|
3280
|
+
status: sync.status()
|
|
3281
|
+
};
|
|
3282
|
+
}
|
|
3283
|
+
const targetUrl = getSiteUrlForConvexUrl(environment.convexUrl);
|
|
3284
|
+
const organization = process.env.FML_ORG ?? getSelectedOrg(environment.name) ?? void 0;
|
|
3285
|
+
const result = await sync.run({
|
|
3286
|
+
targetUrl,
|
|
3287
|
+
targetName: targetUrl,
|
|
3288
|
+
authorization: token.startsWith("Bearer ") ? token : `Bearer ${token}`,
|
|
3289
|
+
organization,
|
|
3290
|
+
destinationIdentity: getSyncDestinationIdentity(environment.name) ?? void 0,
|
|
3291
|
+
...opts.fetch ? { fetch: opts.fetch } : {}
|
|
3292
|
+
});
|
|
3293
|
+
if (result.outcome === "disabled") {
|
|
3294
|
+
return { kind: "disabled", status: result.status };
|
|
3295
|
+
}
|
|
3296
|
+
return {
|
|
3297
|
+
kind: "completed",
|
|
3298
|
+
environment: environment.name,
|
|
3299
|
+
organization,
|
|
3300
|
+
targetUrl,
|
|
3301
|
+
result
|
|
3302
|
+
};
|
|
3303
|
+
}
|
|
3304
|
+
|
|
3305
|
+
// src/package-root.ts
|
|
3306
|
+
import fs9 from "fs";
|
|
3307
|
+
import path19 from "path";
|
|
3308
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
3309
|
+
function resolvePackageRoot(startDir = path19.dirname(fileURLToPath2(import.meta.url))) {
|
|
3310
|
+
let directory = startDir;
|
|
3311
|
+
while (true) {
|
|
3312
|
+
if (fs9.existsSync(path19.join(directory, "package.json")) && fs9.existsSync(path19.join(directory, "bin"))) {
|
|
3313
|
+
return directory;
|
|
3314
|
+
}
|
|
3315
|
+
const parent = path19.dirname(directory);
|
|
3316
|
+
if (parent === directory) {
|
|
3317
|
+
throw new Error(`Could not resolve FML package root from ${startDir}`);
|
|
3318
|
+
}
|
|
3319
|
+
directory = parent;
|
|
3320
|
+
}
|
|
3321
|
+
}
|
|
3322
|
+
|
|
3323
|
+
export {
|
|
3324
|
+
claudeConfigDirectory,
|
|
3325
|
+
geminiConfigDirectory,
|
|
3326
|
+
initializeSyncSettings,
|
|
3327
|
+
createRawArtifactSync,
|
|
3328
|
+
resolvePackageRoot,
|
|
3329
|
+
createCliRawSync,
|
|
3330
|
+
runRawSyncOnce
|
|
3331
|
+
};
|