@plune-ai/cli 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/dist/cli.cjs +293 -29
- package/dist/reporter-core.cjs +278 -22
- package/dist/reporter-core.js +274 -18
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -324,8 +324,13 @@ without closing.
|
|
|
324
324
|
|
|
325
325
|
### Many screenshots: upload concurrency and retries
|
|
326
326
|
|
|
327
|
-
|
|
328
|
-
|
|
327
|
+
Files go to Plune up to 100 to a request: `uploads-batch` announces them (with the SHA-256 of each, read as a stream,
|
|
328
|
+
so a file the platform already has is answered `exists` and is not sent again), the bytes go straight to your bucket
|
|
329
|
+
(or, for a project without one, 100 files or 6 MiB at a time through `direct-batch`), and `complete-batch` confirms
|
|
330
|
+
them. Every file has its own verdict: one that is refused is reported and the rest of its request goes through. A
|
|
331
|
+
report of 2760 screenshots is about 56 requests to Plune instead of 5500. The bytes themselves still go one request each,
|
|
332
|
+
so the PUTs to a bucket are bound by the round trip, not by bandwidth: four at a time by default, raise it on a runner
|
|
333
|
+
with a slow link:
|
|
329
334
|
|
|
330
335
|
```bash
|
|
331
336
|
PLUNE_UPLOAD_CONCURRENCY=16 plune run import ./playwright.json # 1 to 32, default 4
|
|
@@ -336,11 +341,14 @@ When the platform answers 429 with a `Retry-After` (seconds or an HTTP date, nev
|
|
|
336
341
|
the request is tried again after that time plus up to half as much, and every other request of the import
|
|
337
342
|
waits through the same pause. `PLUNE_UPLOAD_RETRIES` (0 to 20, default 6) says how many times; a 429 that
|
|
338
343
|
names no time is the quota and is not retried. A platform that is down (a 5xx or no answer) is still tried
|
|
339
|
-
twice (three times for a bucket). A
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
the same
|
|
344
|
+
twice (three times for a bucket). A 429 on a whole request waits the same way; a file that the bucket did not
|
|
345
|
+
answer for at `complete` is confirmed again, on its own. A file that stays refused is reported without it, never fails
|
|
346
|
+
the import, and is counted in the line printed before the summary:
|
|
347
|
+
`plune: files — 2758 uploaded, 0 skipped, 0 already there (skipped), 14 retried, 0 dropped, in 41.2 s (16 at once, 56 request(s) to Plune)`.
|
|
348
|
+
`already there` are the files Plune had from an earlier import of the same report (same result, name, type and bytes):
|
|
349
|
+
none of them is sent again. Against a platform older than the bulk routes (it answers 404 to `uploads-batch`) the CLI
|
|
350
|
+
sends one request per file, as before; it decides by that answer, not by a version. A value outside the range is
|
|
351
|
+
ignored with a warning, and the default is used. The Playwright reporter reads the same variables.
|
|
344
352
|
|
|
345
353
|
### When a test is deleted
|
|
346
354
|
|
package/dist/cli.cjs
CHANGED
|
@@ -2789,6 +2789,34 @@ var init_failure_detail = __esm({
|
|
|
2789
2789
|
}
|
|
2790
2790
|
});
|
|
2791
2791
|
|
|
2792
|
+
// src/reporter-core/files-batch.ts
|
|
2793
|
+
async function sha256File(path17) {
|
|
2794
|
+
const hash = (0, import_node_crypto2.createHash)("sha256");
|
|
2795
|
+
for await (const part of (0, import_node_fs4.createReadStream)(path17)) hash.update(part);
|
|
2796
|
+
return hash.digest("hex");
|
|
2797
|
+
}
|
|
2798
|
+
var import_node_crypto2, import_node_fs4, FILES_BATCH_MAX, DIRECT_BATCH_BYTES, BATCH_PIPELINES, sha256Bytes, BUCKET_CODES;
|
|
2799
|
+
var init_files_batch = __esm({
|
|
2800
|
+
"src/reporter-core/files-batch.ts"() {
|
|
2801
|
+
"use strict";
|
|
2802
|
+
init_cjs_shims();
|
|
2803
|
+
import_node_crypto2 = require("crypto");
|
|
2804
|
+
import_node_fs4 = require("fs");
|
|
2805
|
+
FILES_BATCH_MAX = 100;
|
|
2806
|
+
DIRECT_BATCH_BYTES = 6 * 1024 * 1024;
|
|
2807
|
+
BATCH_PIPELINES = 2;
|
|
2808
|
+
sha256Bytes = (bytes) => (0, import_node_crypto2.createHash)("sha256").update(bytes).digest("hex");
|
|
2809
|
+
BUCKET_CODES = /* @__PURE__ */ new Set([
|
|
2810
|
+
"bucket_unreachable",
|
|
2811
|
+
"not_connected",
|
|
2812
|
+
"platform_not_configured",
|
|
2813
|
+
"upload_expired",
|
|
2814
|
+
"file_mismatch",
|
|
2815
|
+
"file_missing"
|
|
2816
|
+
]);
|
|
2817
|
+
}
|
|
2818
|
+
});
|
|
2819
|
+
|
|
2792
2820
|
// src/reporter-core/session.ts
|
|
2793
2821
|
function nameOf(file) {
|
|
2794
2822
|
const own = file.name.split(/[\\/]/).pop()?.trim() || (file.path === void 0 ? "" : (0, import_node_path3.basename)(file.path));
|
|
@@ -2840,7 +2868,7 @@ function tally(counts, noun) {
|
|
|
2840
2868
|
}
|
|
2841
2869
|
function boundKey(key) {
|
|
2842
2870
|
if (key.value.length <= KEY_MAX) return key;
|
|
2843
|
-
const digest = (0,
|
|
2871
|
+
const digest = (0, import_node_crypto3.createHash)("sha256").update(key.value).digest("hex").slice(0, 16);
|
|
2844
2872
|
let head = key.value.slice(0, KEY_MAX - digest.length - 1);
|
|
2845
2873
|
if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
|
|
2846
2874
|
return { ...key, value: `${head}#${digest}` };
|
|
@@ -2892,7 +2920,7 @@ async function startRun(passed, declared2) {
|
|
|
2892
2920
|
`));
|
|
2893
2921
|
const fallbackPath = cfg.fallbackPath ?? DEFAULT_FALLBACK_PATH;
|
|
2894
2922
|
const externalKey = cfg.externalKey ?? null;
|
|
2895
|
-
const sessionMarker = (0,
|
|
2923
|
+
const sessionMarker = (0, import_node_crypto3.randomUUID)();
|
|
2896
2924
|
const batchSize = Math.min(cfg.batchSize ?? BATCH_DEFAULT, BATCH_MAX);
|
|
2897
2925
|
const offerDiscovered = cfg.offerDiscovered ?? false;
|
|
2898
2926
|
const token = cfg.token ?? loadToken() ?? "";
|
|
@@ -2900,6 +2928,9 @@ async function startRun(passed, declared2) {
|
|
|
2900
2928
|
const uploadRetries = Math.min(Math.max(Math.floor(cfg.uploadRetries ?? UPLOAD_RETRIES.default), UPLOAD_RETRIES.min), UPLOAD_RETRIES.max);
|
|
2901
2929
|
let retried = 0;
|
|
2902
2930
|
let uploadMs = 0;
|
|
2931
|
+
let requests = 0;
|
|
2932
|
+
let existed = 0;
|
|
2933
|
+
let batchMissing = false;
|
|
2903
2934
|
const client = createClient({
|
|
2904
2935
|
apiUrl: resolveApiUrl(cfg.apiUrl),
|
|
2905
2936
|
token,
|
|
@@ -3083,6 +3114,7 @@ async function startRun(passed, declared2) {
|
|
|
3083
3114
|
else log(`plune: ${KINDS[kind].label} "${name}" not uploaded \u2014 ${SKIPPED[why]}.`);
|
|
3084
3115
|
}
|
|
3085
3116
|
function ask(job, size) {
|
|
3117
|
+
requests += 1;
|
|
3086
3118
|
return client.post(`/v1/results/${encodeURIComponent(job.id)}/files/uploads`, {
|
|
3087
3119
|
files: [{ name: job.name, contentType: job.type, size }]
|
|
3088
3120
|
}, uploadRetries);
|
|
@@ -3117,13 +3149,14 @@ async function startRun(passed, declared2) {
|
|
|
3117
3149
|
if (job.body !== void 0) bytes = job.body;
|
|
3118
3150
|
else {
|
|
3119
3151
|
try {
|
|
3120
|
-
bytes = await
|
|
3152
|
+
bytes = await import_node_fs5.promises.readFile(job.path);
|
|
3121
3153
|
} catch {
|
|
3122
3154
|
return skip(kind, job.name, "missing");
|
|
3123
3155
|
}
|
|
3124
3156
|
}
|
|
3125
3157
|
const refusal = kind === "text" ? textRefusal(bytes) : void 0;
|
|
3126
3158
|
if (refusal !== void 0) return skip(kind, job.name, refusal);
|
|
3159
|
+
requests += 1;
|
|
3127
3160
|
const out = await client.upload(
|
|
3128
3161
|
`/v1/results/${encodeURIComponent(job.id)}/files?name=${encodeURIComponent(job.name)}`,
|
|
3129
3162
|
bytes,
|
|
@@ -3146,6 +3179,7 @@ async function startRun(passed, declared2) {
|
|
|
3146
3179
|
if (fresh === void 0) return failedToBucket(job, "bad_answer");
|
|
3147
3180
|
link = fresh;
|
|
3148
3181
|
}
|
|
3182
|
+
requests += 1;
|
|
3149
3183
|
const done2 = await client.post(`/v1/files/${encodeURIComponent(link.id)}/complete`, {}, uploadRetries);
|
|
3150
3184
|
if (!done2.ok) return failedToBucket(job, codeOf2(done2));
|
|
3151
3185
|
countsOf(job.kind).uploaded += 1;
|
|
@@ -3170,7 +3204,7 @@ async function startRun(passed, declared2) {
|
|
|
3170
3204
|
else if (job.path === void 0) return void skip(kind, job.name, "missing");
|
|
3171
3205
|
else {
|
|
3172
3206
|
try {
|
|
3173
|
-
size = (await
|
|
3207
|
+
size = (await import_node_fs5.promises.stat(job.path)).size;
|
|
3174
3208
|
} catch {
|
|
3175
3209
|
return void skip(kind, job.name, "missing");
|
|
3176
3210
|
}
|
|
@@ -3206,6 +3240,245 @@ async function startRun(passed, declared2) {
|
|
|
3206
3240
|
}
|
|
3207
3241
|
return () => uploadToBucket(job, size, link);
|
|
3208
3242
|
}
|
|
3243
|
+
async function uploadEach(jobs) {
|
|
3244
|
+
const byResult = /* @__PURE__ */ new Map();
|
|
3245
|
+
for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
|
|
3246
|
+
await Promise.all(
|
|
3247
|
+
[...byResult.values()].map(async (group) => {
|
|
3248
|
+
const work = [];
|
|
3249
|
+
for (const job of group) {
|
|
3250
|
+
const act = await plan(job);
|
|
3251
|
+
if (act !== void 0) work.push(gated(act));
|
|
3252
|
+
}
|
|
3253
|
+
await Promise.all(work);
|
|
3254
|
+
})
|
|
3255
|
+
);
|
|
3256
|
+
}
|
|
3257
|
+
async function measure(job) {
|
|
3258
|
+
let size;
|
|
3259
|
+
if (job.body !== void 0) size = job.body.length;
|
|
3260
|
+
else if (job.path === void 0) return void skip(job.kind, job.name, "missing");
|
|
3261
|
+
else {
|
|
3262
|
+
try {
|
|
3263
|
+
size = (await import_node_fs5.promises.stat(job.path)).size;
|
|
3264
|
+
} catch {
|
|
3265
|
+
return void skip(job.kind, job.name, "missing");
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
if (size === 0) return void skip(job.kind, job.name, "empty");
|
|
3269
|
+
return size;
|
|
3270
|
+
}
|
|
3271
|
+
const routeMissing = (out) => !out.ok && out.status === 404 && out.detail === "" && out.code === void 0;
|
|
3272
|
+
function takeRoom(job) {
|
|
3273
|
+
if (job.kind === "file") {
|
|
3274
|
+
skip(job.kind, job.name, "noStorage");
|
|
3275
|
+
return false;
|
|
3276
|
+
}
|
|
3277
|
+
const roomKey = `${job.id}/${job.kind}`;
|
|
3278
|
+
const left = rooms.get(roomKey) ?? KINDS[job.kind].room;
|
|
3279
|
+
if (left === 0) {
|
|
3280
|
+
skip(job.kind, job.name, KINDS[job.kind].limit);
|
|
3281
|
+
return false;
|
|
3282
|
+
}
|
|
3283
|
+
rooms.set(roomKey, left - 1);
|
|
3284
|
+
return true;
|
|
3285
|
+
}
|
|
3286
|
+
function refusedItem(job, size, item) {
|
|
3287
|
+
const code = item.code ?? "refused";
|
|
3288
|
+
if (code === "duplicate") existed += 1;
|
|
3289
|
+
else if (code === "no_storage") skip(job.kind, job.name, refusedHere(job, size) ?? "noStorage");
|
|
3290
|
+
else if (code === "too_many_files") skip(job.kind, job.name, KINDS[job.kind].limit);
|
|
3291
|
+
else if (BUCKET_CODES.has(code)) failedToBucket(job, code);
|
|
3292
|
+
else failed(job, code);
|
|
3293
|
+
}
|
|
3294
|
+
const itemPause = (round) => new Promise((r) => setTimeout(r, Math.min(250 * 2 ** round, 4e3)));
|
|
3295
|
+
async function batchOf(jobs) {
|
|
3296
|
+
if (batchMissing) return uploadEach(jobs);
|
|
3297
|
+
const sized = [];
|
|
3298
|
+
for (const job of jobs) {
|
|
3299
|
+
const size = await measure(job);
|
|
3300
|
+
if (size === void 0) continue;
|
|
3301
|
+
try {
|
|
3302
|
+
sized.push({ job, size, sha256: job.body !== void 0 ? sha256Bytes(job.body) : await sha256File(job.path) });
|
|
3303
|
+
} catch {
|
|
3304
|
+
skip(job.kind, job.name, "missing");
|
|
3305
|
+
}
|
|
3306
|
+
}
|
|
3307
|
+
if (sized.length === 0) return;
|
|
3308
|
+
const announced = await gated(() => {
|
|
3309
|
+
requests += 1;
|
|
3310
|
+
return client.post(
|
|
3311
|
+
"/v1/files/uploads-batch",
|
|
3312
|
+
{ items: sized.map((s) => ({ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 })) },
|
|
3313
|
+
uploadRetries
|
|
3314
|
+
);
|
|
3315
|
+
});
|
|
3316
|
+
if (routeMissing(announced)) {
|
|
3317
|
+
batchMissing = true;
|
|
3318
|
+
return uploadEach(sized.map((s) => s.job));
|
|
3319
|
+
}
|
|
3320
|
+
const verdicts = announced.ok ? announced.body?.items : void 0;
|
|
3321
|
+
if (!announced.ok || !Array.isArray(verdicts) || verdicts.length !== sized.length) {
|
|
3322
|
+
for (const s of sized) {
|
|
3323
|
+
if (announced.ok) failed(s.job, "bad_answer");
|
|
3324
|
+
else if (announced.code !== void 0) failedToBucket(s.job, announced.code);
|
|
3325
|
+
else failed(s.job, announced.detail || announced.kind);
|
|
3326
|
+
}
|
|
3327
|
+
return;
|
|
3328
|
+
}
|
|
3329
|
+
const puts = [];
|
|
3330
|
+
const directs = [];
|
|
3331
|
+
sized.forEach((s, i) => {
|
|
3332
|
+
const v = verdicts[i] ?? {};
|
|
3333
|
+
if (v.status === "exists") existed += 1;
|
|
3334
|
+
else if (v.status === "direct") {
|
|
3335
|
+
if (takeRoom(s.job)) directs.push(s);
|
|
3336
|
+
} else if (v.status === "upload") {
|
|
3337
|
+
const link = linkOf(v);
|
|
3338
|
+
if (link === void 0) return failed(s.job, "bad_answer");
|
|
3339
|
+
if (!bucketSeen) {
|
|
3340
|
+
bucketSeen = true;
|
|
3341
|
+
log(BUCKET_FOUND);
|
|
3342
|
+
}
|
|
3343
|
+
puts.push({ s, link });
|
|
3344
|
+
} else if (v.status === "refused") refusedItem(s.job, s.size, v);
|
|
3345
|
+
else failed(s.job, "bad_answer");
|
|
3346
|
+
});
|
|
3347
|
+
const toComplete = [];
|
|
3348
|
+
await Promise.all(
|
|
3349
|
+
puts.map(
|
|
3350
|
+
({ s, link: first }) => gated(async () => {
|
|
3351
|
+
const source = s.job.body !== void 0 ? { body: s.job.body } : { path: s.job.path, size: s.size };
|
|
3352
|
+
let link = first;
|
|
3353
|
+
for (let renewed = false; ; ) {
|
|
3354
|
+
const put = await client.put(link.url, link.headers, source);
|
|
3355
|
+
if (put.ok) break;
|
|
3356
|
+
if (put.status !== 403 || renewed) return failedToBucket(s.job, put.code);
|
|
3357
|
+
renewed = true;
|
|
3358
|
+
requests += 1;
|
|
3359
|
+
const again = await client.post(
|
|
3360
|
+
"/v1/files/uploads-batch",
|
|
3361
|
+
{ items: [{ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 }] },
|
|
3362
|
+
uploadRetries
|
|
3363
|
+
);
|
|
3364
|
+
const fresh = again.ok ? linkOf(again.body?.items?.[0]) : void 0;
|
|
3365
|
+
if (fresh === void 0) return failedToBucket(s.job, again.ok ? "bad_answer" : codeOf2(again));
|
|
3366
|
+
link = fresh;
|
|
3367
|
+
}
|
|
3368
|
+
toComplete.push({ job: s.job, id: link.id });
|
|
3369
|
+
})
|
|
3370
|
+
)
|
|
3371
|
+
);
|
|
3372
|
+
await complete(toComplete);
|
|
3373
|
+
await sendDirect(directs);
|
|
3374
|
+
}
|
|
3375
|
+
async function complete(items) {
|
|
3376
|
+
for (let round = 0; items.length > 0; round += 1) {
|
|
3377
|
+
const out = await gated(() => {
|
|
3378
|
+
requests += 1;
|
|
3379
|
+
return client.post("/v1/files/complete-batch", { ids: items.map((i) => i.id) }, uploadRetries);
|
|
3380
|
+
});
|
|
3381
|
+
if (routeMissing(out)) {
|
|
3382
|
+
await Promise.all(
|
|
3383
|
+
items.map(
|
|
3384
|
+
(i) => gated(async () => {
|
|
3385
|
+
requests += 1;
|
|
3386
|
+
const one = await client.post(`/v1/files/${encodeURIComponent(i.id)}/complete`, {}, uploadRetries);
|
|
3387
|
+
if (one.ok) countsOf(i.job.kind).uploaded += 1;
|
|
3388
|
+
else failedToBucket(i.job, codeOf2(one));
|
|
3389
|
+
})
|
|
3390
|
+
)
|
|
3391
|
+
);
|
|
3392
|
+
return;
|
|
3393
|
+
}
|
|
3394
|
+
const verdicts = out.ok ? out.body?.items : void 0;
|
|
3395
|
+
if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== items.length) {
|
|
3396
|
+
for (const i of items) failedToBucket(i.job, out.ok ? "bad_answer" : codeOf2(out));
|
|
3397
|
+
return;
|
|
3398
|
+
}
|
|
3399
|
+
const again = [];
|
|
3400
|
+
items.forEach((i, at) => {
|
|
3401
|
+
const v = verdicts[at] ?? {};
|
|
3402
|
+
if (v.status === "ready") countsOf(i.job.kind).uploaded += 1;
|
|
3403
|
+
else if (v.retryable === true && round < Math.min(uploadRetries, 3)) again.push(i);
|
|
3404
|
+
else refusedItem(i.job, 0, v.status === "refused" ? v : { code: "bad_answer" });
|
|
3405
|
+
});
|
|
3406
|
+
items = again;
|
|
3407
|
+
if (items.length > 0) {
|
|
3408
|
+
retried += items.length;
|
|
3409
|
+
await itemPause(round);
|
|
3410
|
+
}
|
|
3411
|
+
}
|
|
3412
|
+
}
|
|
3413
|
+
async function sendDirect(items) {
|
|
3414
|
+
const groups = [];
|
|
3415
|
+
let bytes = 0;
|
|
3416
|
+
for (const item of items) {
|
|
3417
|
+
const group = groups[groups.length - 1];
|
|
3418
|
+
if (group === void 0 || group.length >= FILES_BATCH_MAX || bytes + item.size > DIRECT_BATCH_BYTES) {
|
|
3419
|
+
groups.push([item]);
|
|
3420
|
+
bytes = item.size;
|
|
3421
|
+
} else {
|
|
3422
|
+
group.push(item);
|
|
3423
|
+
bytes += item.size;
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
for (const group of groups) {
|
|
3427
|
+
const body = [];
|
|
3428
|
+
const sent = [];
|
|
3429
|
+
for (const { job, size } of group) {
|
|
3430
|
+
const { kind } = job;
|
|
3431
|
+
if (size > KINDS[kind].bytes) {
|
|
3432
|
+
skip(kind, job.name, KINDS[kind].size);
|
|
3433
|
+
continue;
|
|
3434
|
+
}
|
|
3435
|
+
let data;
|
|
3436
|
+
try {
|
|
3437
|
+
data = job.body ?? await import_node_fs5.promises.readFile(job.path);
|
|
3438
|
+
} catch {
|
|
3439
|
+
skip(kind, job.name, "missing");
|
|
3440
|
+
continue;
|
|
3441
|
+
}
|
|
3442
|
+
const refusal = kind === "text" ? textRefusal(data) : void 0;
|
|
3443
|
+
if (refusal !== void 0) {
|
|
3444
|
+
skip(kind, job.name, refusal);
|
|
3445
|
+
continue;
|
|
3446
|
+
}
|
|
3447
|
+
body.push({ resultId: job.id, name: job.name, contentType: job.type, data: data.toString("base64") });
|
|
3448
|
+
sent.push({ job, size });
|
|
3449
|
+
}
|
|
3450
|
+
if (body.length === 0) continue;
|
|
3451
|
+
const out = await gated(() => {
|
|
3452
|
+
requests += 1;
|
|
3453
|
+
return client.post("/v1/files/direct-batch", { items: body }, uploadRetries);
|
|
3454
|
+
});
|
|
3455
|
+
if (routeMissing(out)) {
|
|
3456
|
+
batchMissing = true;
|
|
3457
|
+
for (const { job, size } of sent) await gated(() => uploadDirect(job, size));
|
|
3458
|
+
continue;
|
|
3459
|
+
}
|
|
3460
|
+
const verdicts = out.ok ? out.body?.items : void 0;
|
|
3461
|
+
if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== sent.length) {
|
|
3462
|
+
for (const { job } of sent) failed(job, out.ok ? "bad_answer" : out.detail || out.kind);
|
|
3463
|
+
continue;
|
|
3464
|
+
}
|
|
3465
|
+
sent.forEach(({ job }, i) => {
|
|
3466
|
+
const v = verdicts[i] ?? {};
|
|
3467
|
+
if (v.status === "stored") countsOf(job.kind).uploaded += 1;
|
|
3468
|
+
else if (v.status === "exists") existed += 1;
|
|
3469
|
+
else refusedItem(job, 0, v.status === "refused" ? v : { code: "bad_answer" });
|
|
3470
|
+
});
|
|
3471
|
+
}
|
|
3472
|
+
}
|
|
3473
|
+
async function uploadBatched(jobs) {
|
|
3474
|
+
const batches = chunk(jobs, FILES_BATCH_MAX);
|
|
3475
|
+
let next = 0;
|
|
3476
|
+
await Promise.all(
|
|
3477
|
+
Array.from({ length: Math.min(BATCH_PIPELINES, batches.length) }, async () => {
|
|
3478
|
+
for (let at = next++; at < batches.length; at = next++) await batchOf(batches[at]);
|
|
3479
|
+
})
|
|
3480
|
+
);
|
|
3481
|
+
}
|
|
3209
3482
|
async function upload(items, filesByKey) {
|
|
3210
3483
|
if (filesByKey.size === 0 || !Array.isArray(items)) return;
|
|
3211
3484
|
const jobs = [];
|
|
@@ -3231,18 +3504,8 @@ async function startRun(passed, declared2) {
|
|
|
3231
3504
|
}
|
|
3232
3505
|
}
|
|
3233
3506
|
const started = Date.now();
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
await Promise.all(
|
|
3237
|
-
[...byResult.values()].map(async (group) => {
|
|
3238
|
-
const work = [];
|
|
3239
|
-
for (const job of group) {
|
|
3240
|
-
const act = await plan(job);
|
|
3241
|
-
if (act !== void 0) work.push(gated(act));
|
|
3242
|
-
}
|
|
3243
|
-
await Promise.all(work);
|
|
3244
|
-
})
|
|
3245
|
-
);
|
|
3507
|
+
if (batchMissing) await uploadEach(jobs);
|
|
3508
|
+
else await uploadBatched(jobs);
|
|
3246
3509
|
uploadMs += Date.now() - started;
|
|
3247
3510
|
}
|
|
3248
3511
|
async function flush() {
|
|
@@ -3338,9 +3601,9 @@ async function startRun(passed, declared2) {
|
|
|
3338
3601
|
if (shortened.size > 0) parts.push(`${shortened.size} test name(s) shortened to fit a key`);
|
|
3339
3602
|
const files = [stats.screenshots, stats.textFiles, stats.otherFiles];
|
|
3340
3603
|
const total = (what) => files.reduce((n, counts) => n + counts[what], 0);
|
|
3341
|
-
if (total("uploaded") + total("skipped") + total("failed") > 0) {
|
|
3604
|
+
if (total("uploaded") + total("skipped") + total("failed") + existed > 0) {
|
|
3342
3605
|
log(
|
|
3343
|
-
`plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once)`
|
|
3606
|
+
`plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${existed} already there (skipped), ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once, ${requests} request(s) to Plune)`
|
|
3344
3607
|
);
|
|
3345
3608
|
}
|
|
3346
3609
|
log(`plune: ${parts.join(" \xB7 ")}`);
|
|
@@ -3397,13 +3660,13 @@ async function startRun(passed, declared2) {
|
|
|
3397
3660
|
}
|
|
3398
3661
|
};
|
|
3399
3662
|
}
|
|
3400
|
-
var
|
|
3663
|
+
var import_node_crypto3, import_node_fs5, import_node_path3, import_node_util2, RESOLVE_CHUNK, BATCH_MAX, DISCOVER_MAX, BATCH_DEFAULT, EXPECTED_MAX, KEY_MAX, TITLE_MAX, COUNTED, SCREENSHOT_TYPES, SCREENSHOT_BYTES, SCREENSHOTS_PER_RESULT, TEXT_BYTES, TEXTS_PER_RESULT, UTF8, TYPE_BY_EXTENSION, METADATA_TYPE2, NAME_MAX, SKIPPED, BUCKET_FOUND, KINDS, codeOf2, BATCH_BYTES, ENVELOPE_BYTES;
|
|
3401
3664
|
var init_session = __esm({
|
|
3402
3665
|
"src/reporter-core/session.ts"() {
|
|
3403
3666
|
"use strict";
|
|
3404
3667
|
init_cjs_shims();
|
|
3405
|
-
|
|
3406
|
-
|
|
3668
|
+
import_node_crypto3 = require("crypto");
|
|
3669
|
+
import_node_fs5 = require("fs");
|
|
3407
3670
|
import_node_path3 = require("path");
|
|
3408
3671
|
import_node_util2 = require("util");
|
|
3409
3672
|
init_api_url();
|
|
@@ -3411,6 +3674,7 @@ var init_session = __esm({
|
|
|
3411
3674
|
init_client();
|
|
3412
3675
|
init_env2();
|
|
3413
3676
|
init_failure_detail();
|
|
3677
|
+
init_files_batch();
|
|
3414
3678
|
init_fallback();
|
|
3415
3679
|
RESOLVE_CHUNK = 500;
|
|
3416
3680
|
BATCH_MAX = 500;
|
|
@@ -4878,7 +5142,7 @@ function readPlaywrightJson(source, file) {
|
|
|
4878
5142
|
const { config, suites } = parsed;
|
|
4879
5143
|
const rootDir = typeof config?.rootDir === "string" ? config.rootDir : void 0;
|
|
4880
5144
|
const buildHref = config?.metadata?.ci?.buildHref;
|
|
4881
|
-
const repoRoot = rootDir !== void 0 && (0,
|
|
5145
|
+
const repoRoot = rootDir !== void 0 && (0, import_node_fs6.existsSync)(rootDir) ? repoRootOf(rootDir) : void 0;
|
|
4882
5146
|
const ctx = {
|
|
4883
5147
|
...rootDir !== void 0 ? { rootDir } : {},
|
|
4884
5148
|
...repoRoot !== void 0 ? { repoRoot } : {},
|
|
@@ -4897,12 +5161,12 @@ function readPlaywrightJson(source, file) {
|
|
|
4897
5161
|
function looksLikePlaywrightJson(source) {
|
|
4898
5162
|
return /^\s*\{/.test(source) && source.includes('"suites"');
|
|
4899
5163
|
}
|
|
4900
|
-
var
|
|
5164
|
+
var import_node_fs6, SOURCE3, TOKEN3, NOT_FAILED, JsonReportError;
|
|
4901
5165
|
var init_playwright_json = __esm({
|
|
4902
5166
|
"src/importers/playwright-json.ts"() {
|
|
4903
5167
|
"use strict";
|
|
4904
5168
|
init_cjs_shims();
|
|
4905
|
-
|
|
5169
|
+
import_node_fs6 = require("fs");
|
|
4906
5170
|
init_failure_detail();
|
|
4907
5171
|
init_result_key();
|
|
4908
5172
|
SOURCE3 = "playwright";
|
|
@@ -5930,11 +6194,11 @@ module.exports = __toCommonJS(cli_exports);
|
|
|
5930
6194
|
init_cjs_shims();
|
|
5931
6195
|
init_api_url();
|
|
5932
6196
|
var import_commander = require("commander");
|
|
5933
|
-
var
|
|
6197
|
+
var import_node_fs7 = require("fs");
|
|
5934
6198
|
var import_node_path4 = require("path");
|
|
5935
6199
|
var import_node_url = require("url");
|
|
5936
6200
|
var __dirname = (0, import_node_path4.dirname)((0, import_node_url.fileURLToPath)(importMetaUrl));
|
|
5937
|
-
var pkg = JSON.parse((0,
|
|
6201
|
+
var pkg = JSON.parse((0, import_node_fs7.readFileSync)((0, import_node_path4.join)(__dirname, "..", "package.json"), "utf-8"));
|
|
5938
6202
|
function toFormat(s) {
|
|
5939
6203
|
return s === "json" || s === "markdown" ? s : "console";
|
|
5940
6204
|
}
|
|
@@ -5943,7 +6207,7 @@ function shouldColor(outputPath) {
|
|
|
5943
6207
|
}
|
|
5944
6208
|
function writeReport(text2, outputPath) {
|
|
5945
6209
|
if (outputPath !== void 0) {
|
|
5946
|
-
(0,
|
|
6210
|
+
(0, import_node_fs7.writeFileSync)(outputPath, text2);
|
|
5947
6211
|
} else {
|
|
5948
6212
|
process.stdout.write(text2 + "\n");
|
|
5949
6213
|
}
|
|
@@ -6170,7 +6434,7 @@ function createProgram() {
|
|
|
6170
6434
|
false
|
|
6171
6435
|
).option(
|
|
6172
6436
|
"--upload-concurrency <n>",
|
|
6173
|
-
"
|
|
6437
|
+
"File uploads in flight at once, 1 to 32 (default 4; PLUNE_UPLOAD_CONCURRENCY sets it too). Files are announced and confirmed up to 100 to a request; a 429 is retried PLUNE_UPLOAD_RETRIES times (default 6)",
|
|
6174
6438
|
(v) => {
|
|
6175
6439
|
if (!/^\d+$/.test(v) || Number(v) < 1 || Number(v) > 32) throw new import_commander.InvalidArgumentError("a whole number from 1 to 32.");
|
|
6176
6440
|
return Number(v);
|
|
@@ -6442,7 +6706,7 @@ function isMainModule() {
|
|
|
6442
6706
|
const entry = process.argv[1];
|
|
6443
6707
|
if (entry === void 0) return false;
|
|
6444
6708
|
try {
|
|
6445
|
-
return (0,
|
|
6709
|
+
return (0, import_node_fs7.realpathSync)(entry) === (0, import_node_fs7.realpathSync)((0, import_node_url.fileURLToPath)(importMetaUrl));
|
|
6446
6710
|
} catch {
|
|
6447
6711
|
return false;
|
|
6448
6712
|
}
|
package/dist/reporter-core.cjs
CHANGED
|
@@ -48,8 +48,8 @@ __export(reporter_core_exports, {
|
|
|
48
48
|
module.exports = __toCommonJS(reporter_core_exports);
|
|
49
49
|
|
|
50
50
|
// src/reporter-core/session.ts
|
|
51
|
-
var
|
|
52
|
-
var
|
|
51
|
+
var import_node_crypto2 = require("crypto");
|
|
52
|
+
var import_node_fs5 = require("fs");
|
|
53
53
|
var import_node_path3 = require("path");
|
|
54
54
|
var import_node_util2 = require("util");
|
|
55
55
|
|
|
@@ -519,6 +519,27 @@ function webLink(href) {
|
|
|
519
519
|
return href !== void 0 && /^https?:\/\//i.test(href) && URL.canParse(href) ? href : void 0;
|
|
520
520
|
}
|
|
521
521
|
|
|
522
|
+
// src/reporter-core/files-batch.ts
|
|
523
|
+
var import_node_crypto = require("crypto");
|
|
524
|
+
var import_node_fs4 = require("fs");
|
|
525
|
+
var FILES_BATCH_MAX = 100;
|
|
526
|
+
var DIRECT_BATCH_BYTES = 6 * 1024 * 1024;
|
|
527
|
+
var BATCH_PIPELINES = 2;
|
|
528
|
+
async function sha256File(path3) {
|
|
529
|
+
const hash = (0, import_node_crypto.createHash)("sha256");
|
|
530
|
+
for await (const part of (0, import_node_fs4.createReadStream)(path3)) hash.update(part);
|
|
531
|
+
return hash.digest("hex");
|
|
532
|
+
}
|
|
533
|
+
var sha256Bytes = (bytes) => (0, import_node_crypto.createHash)("sha256").update(bytes).digest("hex");
|
|
534
|
+
var BUCKET_CODES = /* @__PURE__ */ new Set([
|
|
535
|
+
"bucket_unreachable",
|
|
536
|
+
"not_connected",
|
|
537
|
+
"platform_not_configured",
|
|
538
|
+
"upload_expired",
|
|
539
|
+
"file_mismatch",
|
|
540
|
+
"file_missing"
|
|
541
|
+
]);
|
|
542
|
+
|
|
522
543
|
// src/reporter-core/fallback.ts
|
|
523
544
|
var fs = __toESM(require("fs"), 1);
|
|
524
545
|
var path2 = __toESM(require("path"), 1);
|
|
@@ -631,7 +652,7 @@ function tally(counts, noun) {
|
|
|
631
652
|
}
|
|
632
653
|
function boundKey(key) {
|
|
633
654
|
if (key.value.length <= KEY_MAX) return key;
|
|
634
|
-
const digest = (0,
|
|
655
|
+
const digest = (0, import_node_crypto2.createHash)("sha256").update(key.value).digest("hex").slice(0, 16);
|
|
635
656
|
let head = key.value.slice(0, KEY_MAX - digest.length - 1);
|
|
636
657
|
if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
|
|
637
658
|
return { ...key, value: `${head}#${digest}` };
|
|
@@ -685,7 +706,7 @@ async function startRun(passed, declared) {
|
|
|
685
706
|
`));
|
|
686
707
|
const fallbackPath = cfg.fallbackPath ?? DEFAULT_FALLBACK_PATH;
|
|
687
708
|
const externalKey = cfg.externalKey ?? null;
|
|
688
|
-
const sessionMarker = (0,
|
|
709
|
+
const sessionMarker = (0, import_node_crypto2.randomUUID)();
|
|
689
710
|
const batchSize = Math.min(cfg.batchSize ?? BATCH_DEFAULT, BATCH_MAX);
|
|
690
711
|
const offerDiscovered = cfg.offerDiscovered ?? false;
|
|
691
712
|
const token = cfg.token ?? loadToken() ?? "";
|
|
@@ -693,6 +714,9 @@ async function startRun(passed, declared) {
|
|
|
693
714
|
const uploadRetries = Math.min(Math.max(Math.floor(cfg.uploadRetries ?? UPLOAD_RETRIES.default), UPLOAD_RETRIES.min), UPLOAD_RETRIES.max);
|
|
694
715
|
let retried = 0;
|
|
695
716
|
let uploadMs = 0;
|
|
717
|
+
let requests = 0;
|
|
718
|
+
let existed = 0;
|
|
719
|
+
let batchMissing = false;
|
|
696
720
|
const client = createClient({
|
|
697
721
|
apiUrl: resolveApiUrl(cfg.apiUrl),
|
|
698
722
|
token,
|
|
@@ -876,6 +900,7 @@ async function startRun(passed, declared) {
|
|
|
876
900
|
else log(`plune: ${KINDS[kind].label} "${name}" not uploaded \u2014 ${SKIPPED[why]}.`);
|
|
877
901
|
}
|
|
878
902
|
function ask(job, size) {
|
|
903
|
+
requests += 1;
|
|
879
904
|
return client.post(`/v1/results/${encodeURIComponent(job.id)}/files/uploads`, {
|
|
880
905
|
files: [{ name: job.name, contentType: job.type, size }]
|
|
881
906
|
}, uploadRetries);
|
|
@@ -910,13 +935,14 @@ async function startRun(passed, declared) {
|
|
|
910
935
|
if (job.body !== void 0) bytes = job.body;
|
|
911
936
|
else {
|
|
912
937
|
try {
|
|
913
|
-
bytes = await
|
|
938
|
+
bytes = await import_node_fs5.promises.readFile(job.path);
|
|
914
939
|
} catch {
|
|
915
940
|
return skip(kind, job.name, "missing");
|
|
916
941
|
}
|
|
917
942
|
}
|
|
918
943
|
const refusal = kind === "text" ? textRefusal(bytes) : void 0;
|
|
919
944
|
if (refusal !== void 0) return skip(kind, job.name, refusal);
|
|
945
|
+
requests += 1;
|
|
920
946
|
const out = await client.upload(
|
|
921
947
|
`/v1/results/${encodeURIComponent(job.id)}/files?name=${encodeURIComponent(job.name)}`,
|
|
922
948
|
bytes,
|
|
@@ -939,6 +965,7 @@ async function startRun(passed, declared) {
|
|
|
939
965
|
if (fresh === void 0) return failedToBucket(job, "bad_answer");
|
|
940
966
|
link = fresh;
|
|
941
967
|
}
|
|
968
|
+
requests += 1;
|
|
942
969
|
const done2 = await client.post(`/v1/files/${encodeURIComponent(link.id)}/complete`, {}, uploadRetries);
|
|
943
970
|
if (!done2.ok) return failedToBucket(job, codeOf(done2));
|
|
944
971
|
countsOf(job.kind).uploaded += 1;
|
|
@@ -963,7 +990,7 @@ async function startRun(passed, declared) {
|
|
|
963
990
|
else if (job.path === void 0) return void skip(kind, job.name, "missing");
|
|
964
991
|
else {
|
|
965
992
|
try {
|
|
966
|
-
size = (await
|
|
993
|
+
size = (await import_node_fs5.promises.stat(job.path)).size;
|
|
967
994
|
} catch {
|
|
968
995
|
return void skip(kind, job.name, "missing");
|
|
969
996
|
}
|
|
@@ -999,6 +1026,245 @@ async function startRun(passed, declared) {
|
|
|
999
1026
|
}
|
|
1000
1027
|
return () => uploadToBucket(job, size, link);
|
|
1001
1028
|
}
|
|
1029
|
+
async function uploadEach(jobs) {
|
|
1030
|
+
const byResult = /* @__PURE__ */ new Map();
|
|
1031
|
+
for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
|
|
1032
|
+
await Promise.all(
|
|
1033
|
+
[...byResult.values()].map(async (group) => {
|
|
1034
|
+
const work = [];
|
|
1035
|
+
for (const job of group) {
|
|
1036
|
+
const act = await plan(job);
|
|
1037
|
+
if (act !== void 0) work.push(gated(act));
|
|
1038
|
+
}
|
|
1039
|
+
await Promise.all(work);
|
|
1040
|
+
})
|
|
1041
|
+
);
|
|
1042
|
+
}
|
|
1043
|
+
async function measure(job) {
|
|
1044
|
+
let size;
|
|
1045
|
+
if (job.body !== void 0) size = job.body.length;
|
|
1046
|
+
else if (job.path === void 0) return void skip(job.kind, job.name, "missing");
|
|
1047
|
+
else {
|
|
1048
|
+
try {
|
|
1049
|
+
size = (await import_node_fs5.promises.stat(job.path)).size;
|
|
1050
|
+
} catch {
|
|
1051
|
+
return void skip(job.kind, job.name, "missing");
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
if (size === 0) return void skip(job.kind, job.name, "empty");
|
|
1055
|
+
return size;
|
|
1056
|
+
}
|
|
1057
|
+
const routeMissing = (out) => !out.ok && out.status === 404 && out.detail === "" && out.code === void 0;
|
|
1058
|
+
function takeRoom(job) {
|
|
1059
|
+
if (job.kind === "file") {
|
|
1060
|
+
skip(job.kind, job.name, "noStorage");
|
|
1061
|
+
return false;
|
|
1062
|
+
}
|
|
1063
|
+
const roomKey = `${job.id}/${job.kind}`;
|
|
1064
|
+
const left = rooms.get(roomKey) ?? KINDS[job.kind].room;
|
|
1065
|
+
if (left === 0) {
|
|
1066
|
+
skip(job.kind, job.name, KINDS[job.kind].limit);
|
|
1067
|
+
return false;
|
|
1068
|
+
}
|
|
1069
|
+
rooms.set(roomKey, left - 1);
|
|
1070
|
+
return true;
|
|
1071
|
+
}
|
|
1072
|
+
function refusedItem(job, size, item) {
|
|
1073
|
+
const code = item.code ?? "refused";
|
|
1074
|
+
if (code === "duplicate") existed += 1;
|
|
1075
|
+
else if (code === "no_storage") skip(job.kind, job.name, refusedHere(job, size) ?? "noStorage");
|
|
1076
|
+
else if (code === "too_many_files") skip(job.kind, job.name, KINDS[job.kind].limit);
|
|
1077
|
+
else if (BUCKET_CODES.has(code)) failedToBucket(job, code);
|
|
1078
|
+
else failed(job, code);
|
|
1079
|
+
}
|
|
1080
|
+
const itemPause = (round) => new Promise((r) => setTimeout(r, Math.min(250 * 2 ** round, 4e3)));
|
|
1081
|
+
async function batchOf(jobs) {
|
|
1082
|
+
if (batchMissing) return uploadEach(jobs);
|
|
1083
|
+
const sized = [];
|
|
1084
|
+
for (const job of jobs) {
|
|
1085
|
+
const size = await measure(job);
|
|
1086
|
+
if (size === void 0) continue;
|
|
1087
|
+
try {
|
|
1088
|
+
sized.push({ job, size, sha256: job.body !== void 0 ? sha256Bytes(job.body) : await sha256File(job.path) });
|
|
1089
|
+
} catch {
|
|
1090
|
+
skip(job.kind, job.name, "missing");
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
if (sized.length === 0) return;
|
|
1094
|
+
const announced = await gated(() => {
|
|
1095
|
+
requests += 1;
|
|
1096
|
+
return client.post(
|
|
1097
|
+
"/v1/files/uploads-batch",
|
|
1098
|
+
{ items: sized.map((s) => ({ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 })) },
|
|
1099
|
+
uploadRetries
|
|
1100
|
+
);
|
|
1101
|
+
});
|
|
1102
|
+
if (routeMissing(announced)) {
|
|
1103
|
+
batchMissing = true;
|
|
1104
|
+
return uploadEach(sized.map((s) => s.job));
|
|
1105
|
+
}
|
|
1106
|
+
const verdicts = announced.ok ? announced.body?.items : void 0;
|
|
1107
|
+
if (!announced.ok || !Array.isArray(verdicts) || verdicts.length !== sized.length) {
|
|
1108
|
+
for (const s of sized) {
|
|
1109
|
+
if (announced.ok) failed(s.job, "bad_answer");
|
|
1110
|
+
else if (announced.code !== void 0) failedToBucket(s.job, announced.code);
|
|
1111
|
+
else failed(s.job, announced.detail || announced.kind);
|
|
1112
|
+
}
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
1115
|
+
const puts = [];
|
|
1116
|
+
const directs = [];
|
|
1117
|
+
sized.forEach((s, i) => {
|
|
1118
|
+
const v = verdicts[i] ?? {};
|
|
1119
|
+
if (v.status === "exists") existed += 1;
|
|
1120
|
+
else if (v.status === "direct") {
|
|
1121
|
+
if (takeRoom(s.job)) directs.push(s);
|
|
1122
|
+
} else if (v.status === "upload") {
|
|
1123
|
+
const link = linkOf(v);
|
|
1124
|
+
if (link === void 0) return failed(s.job, "bad_answer");
|
|
1125
|
+
if (!bucketSeen) {
|
|
1126
|
+
bucketSeen = true;
|
|
1127
|
+
log(BUCKET_FOUND);
|
|
1128
|
+
}
|
|
1129
|
+
puts.push({ s, link });
|
|
1130
|
+
} else if (v.status === "refused") refusedItem(s.job, s.size, v);
|
|
1131
|
+
else failed(s.job, "bad_answer");
|
|
1132
|
+
});
|
|
1133
|
+
const toComplete = [];
|
|
1134
|
+
await Promise.all(
|
|
1135
|
+
puts.map(
|
|
1136
|
+
({ s, link: first }) => gated(async () => {
|
|
1137
|
+
const source = s.job.body !== void 0 ? { body: s.job.body } : { path: s.job.path, size: s.size };
|
|
1138
|
+
let link = first;
|
|
1139
|
+
for (let renewed = false; ; ) {
|
|
1140
|
+
const put = await client.put(link.url, link.headers, source);
|
|
1141
|
+
if (put.ok) break;
|
|
1142
|
+
if (put.status !== 403 || renewed) return failedToBucket(s.job, put.code);
|
|
1143
|
+
renewed = true;
|
|
1144
|
+
requests += 1;
|
|
1145
|
+
const again = await client.post(
|
|
1146
|
+
"/v1/files/uploads-batch",
|
|
1147
|
+
{ items: [{ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 }] },
|
|
1148
|
+
uploadRetries
|
|
1149
|
+
);
|
|
1150
|
+
const fresh = again.ok ? linkOf(again.body?.items?.[0]) : void 0;
|
|
1151
|
+
if (fresh === void 0) return failedToBucket(s.job, again.ok ? "bad_answer" : codeOf(again));
|
|
1152
|
+
link = fresh;
|
|
1153
|
+
}
|
|
1154
|
+
toComplete.push({ job: s.job, id: link.id });
|
|
1155
|
+
})
|
|
1156
|
+
)
|
|
1157
|
+
);
|
|
1158
|
+
await complete(toComplete);
|
|
1159
|
+
await sendDirect(directs);
|
|
1160
|
+
}
|
|
1161
|
+
async function complete(items) {
|
|
1162
|
+
for (let round = 0; items.length > 0; round += 1) {
|
|
1163
|
+
const out = await gated(() => {
|
|
1164
|
+
requests += 1;
|
|
1165
|
+
return client.post("/v1/files/complete-batch", { ids: items.map((i) => i.id) }, uploadRetries);
|
|
1166
|
+
});
|
|
1167
|
+
if (routeMissing(out)) {
|
|
1168
|
+
await Promise.all(
|
|
1169
|
+
items.map(
|
|
1170
|
+
(i) => gated(async () => {
|
|
1171
|
+
requests += 1;
|
|
1172
|
+
const one = await client.post(`/v1/files/${encodeURIComponent(i.id)}/complete`, {}, uploadRetries);
|
|
1173
|
+
if (one.ok) countsOf(i.job.kind).uploaded += 1;
|
|
1174
|
+
else failedToBucket(i.job, codeOf(one));
|
|
1175
|
+
})
|
|
1176
|
+
)
|
|
1177
|
+
);
|
|
1178
|
+
return;
|
|
1179
|
+
}
|
|
1180
|
+
const verdicts = out.ok ? out.body?.items : void 0;
|
|
1181
|
+
if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== items.length) {
|
|
1182
|
+
for (const i of items) failedToBucket(i.job, out.ok ? "bad_answer" : codeOf(out));
|
|
1183
|
+
return;
|
|
1184
|
+
}
|
|
1185
|
+
const again = [];
|
|
1186
|
+
items.forEach((i, at) => {
|
|
1187
|
+
const v = verdicts[at] ?? {};
|
|
1188
|
+
if (v.status === "ready") countsOf(i.job.kind).uploaded += 1;
|
|
1189
|
+
else if (v.retryable === true && round < Math.min(uploadRetries, 3)) again.push(i);
|
|
1190
|
+
else refusedItem(i.job, 0, v.status === "refused" ? v : { code: "bad_answer" });
|
|
1191
|
+
});
|
|
1192
|
+
items = again;
|
|
1193
|
+
if (items.length > 0) {
|
|
1194
|
+
retried += items.length;
|
|
1195
|
+
await itemPause(round);
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
async function sendDirect(items) {
|
|
1200
|
+
const groups = [];
|
|
1201
|
+
let bytes = 0;
|
|
1202
|
+
for (const item of items) {
|
|
1203
|
+
const group = groups[groups.length - 1];
|
|
1204
|
+
if (group === void 0 || group.length >= FILES_BATCH_MAX || bytes + item.size > DIRECT_BATCH_BYTES) {
|
|
1205
|
+
groups.push([item]);
|
|
1206
|
+
bytes = item.size;
|
|
1207
|
+
} else {
|
|
1208
|
+
group.push(item);
|
|
1209
|
+
bytes += item.size;
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
for (const group of groups) {
|
|
1213
|
+
const body = [];
|
|
1214
|
+
const sent = [];
|
|
1215
|
+
for (const { job, size } of group) {
|
|
1216
|
+
const { kind } = job;
|
|
1217
|
+
if (size > KINDS[kind].bytes) {
|
|
1218
|
+
skip(kind, job.name, KINDS[kind].size);
|
|
1219
|
+
continue;
|
|
1220
|
+
}
|
|
1221
|
+
let data;
|
|
1222
|
+
try {
|
|
1223
|
+
data = job.body ?? await import_node_fs5.promises.readFile(job.path);
|
|
1224
|
+
} catch {
|
|
1225
|
+
skip(kind, job.name, "missing");
|
|
1226
|
+
continue;
|
|
1227
|
+
}
|
|
1228
|
+
const refusal = kind === "text" ? textRefusal(data) : void 0;
|
|
1229
|
+
if (refusal !== void 0) {
|
|
1230
|
+
skip(kind, job.name, refusal);
|
|
1231
|
+
continue;
|
|
1232
|
+
}
|
|
1233
|
+
body.push({ resultId: job.id, name: job.name, contentType: job.type, data: data.toString("base64") });
|
|
1234
|
+
sent.push({ job, size });
|
|
1235
|
+
}
|
|
1236
|
+
if (body.length === 0) continue;
|
|
1237
|
+
const out = await gated(() => {
|
|
1238
|
+
requests += 1;
|
|
1239
|
+
return client.post("/v1/files/direct-batch", { items: body }, uploadRetries);
|
|
1240
|
+
});
|
|
1241
|
+
if (routeMissing(out)) {
|
|
1242
|
+
batchMissing = true;
|
|
1243
|
+
for (const { job, size } of sent) await gated(() => uploadDirect(job, size));
|
|
1244
|
+
continue;
|
|
1245
|
+
}
|
|
1246
|
+
const verdicts = out.ok ? out.body?.items : void 0;
|
|
1247
|
+
if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== sent.length) {
|
|
1248
|
+
for (const { job } of sent) failed(job, out.ok ? "bad_answer" : out.detail || out.kind);
|
|
1249
|
+
continue;
|
|
1250
|
+
}
|
|
1251
|
+
sent.forEach(({ job }, i) => {
|
|
1252
|
+
const v = verdicts[i] ?? {};
|
|
1253
|
+
if (v.status === "stored") countsOf(job.kind).uploaded += 1;
|
|
1254
|
+
else if (v.status === "exists") existed += 1;
|
|
1255
|
+
else refusedItem(job, 0, v.status === "refused" ? v : { code: "bad_answer" });
|
|
1256
|
+
});
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
async function uploadBatched(jobs) {
|
|
1260
|
+
const batches = chunk(jobs, FILES_BATCH_MAX);
|
|
1261
|
+
let next = 0;
|
|
1262
|
+
await Promise.all(
|
|
1263
|
+
Array.from({ length: Math.min(BATCH_PIPELINES, batches.length) }, async () => {
|
|
1264
|
+
for (let at = next++; at < batches.length; at = next++) await batchOf(batches[at]);
|
|
1265
|
+
})
|
|
1266
|
+
);
|
|
1267
|
+
}
|
|
1002
1268
|
async function upload(items, filesByKey) {
|
|
1003
1269
|
if (filesByKey.size === 0 || !Array.isArray(items)) return;
|
|
1004
1270
|
const jobs = [];
|
|
@@ -1024,18 +1290,8 @@ async function startRun(passed, declared) {
|
|
|
1024
1290
|
}
|
|
1025
1291
|
}
|
|
1026
1292
|
const started = Date.now();
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
await Promise.all(
|
|
1030
|
-
[...byResult.values()].map(async (group) => {
|
|
1031
|
-
const work = [];
|
|
1032
|
-
for (const job of group) {
|
|
1033
|
-
const act = await plan(job);
|
|
1034
|
-
if (act !== void 0) work.push(gated(act));
|
|
1035
|
-
}
|
|
1036
|
-
await Promise.all(work);
|
|
1037
|
-
})
|
|
1038
|
-
);
|
|
1293
|
+
if (batchMissing) await uploadEach(jobs);
|
|
1294
|
+
else await uploadBatched(jobs);
|
|
1039
1295
|
uploadMs += Date.now() - started;
|
|
1040
1296
|
}
|
|
1041
1297
|
async function flush() {
|
|
@@ -1131,9 +1387,9 @@ async function startRun(passed, declared) {
|
|
|
1131
1387
|
if (shortened.size > 0) parts.push(`${shortened.size} test name(s) shortened to fit a key`);
|
|
1132
1388
|
const files = [stats.screenshots, stats.textFiles, stats.otherFiles];
|
|
1133
1389
|
const total = (what) => files.reduce((n, counts) => n + counts[what], 0);
|
|
1134
|
-
if (total("uploaded") + total("skipped") + total("failed") > 0) {
|
|
1390
|
+
if (total("uploaded") + total("skipped") + total("failed") + existed > 0) {
|
|
1135
1391
|
log(
|
|
1136
|
-
`plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once)`
|
|
1392
|
+
`plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${existed} already there (skipped), ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once, ${requests} request(s) to Plune)`
|
|
1137
1393
|
);
|
|
1138
1394
|
}
|
|
1139
1395
|
log(`plune: ${parts.join(" \xB7 ")}`);
|
|
@@ -1192,9 +1448,9 @@ async function startRun(passed, declared) {
|
|
|
1192
1448
|
}
|
|
1193
1449
|
|
|
1194
1450
|
// src/util/hash.ts
|
|
1195
|
-
var
|
|
1451
|
+
var import_node_crypto3 = require("crypto");
|
|
1196
1452
|
function sha256(input) {
|
|
1197
|
-
return (0,
|
|
1453
|
+
return (0, import_node_crypto3.createHash)("sha256").update(input, "utf8").digest("hex");
|
|
1198
1454
|
}
|
|
1199
1455
|
|
|
1200
1456
|
// src/reporter-core/result-key.ts
|
package/dist/reporter-core.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/reporter-core/session.ts
|
|
2
|
-
import { createHash, randomUUID } from "crypto";
|
|
2
|
+
import { createHash as createHash2, randomUUID } from "crypto";
|
|
3
3
|
import { promises as disk } from "fs";
|
|
4
4
|
import { basename as basename2, extname } from "path";
|
|
5
5
|
import { TextDecoder } from "util";
|
|
@@ -470,6 +470,27 @@ function webLink(href) {
|
|
|
470
470
|
return href !== void 0 && /^https?:\/\//i.test(href) && URL.canParse(href) ? href : void 0;
|
|
471
471
|
}
|
|
472
472
|
|
|
473
|
+
// src/reporter-core/files-batch.ts
|
|
474
|
+
import { createHash } from "crypto";
|
|
475
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
476
|
+
var FILES_BATCH_MAX = 100;
|
|
477
|
+
var DIRECT_BATCH_BYTES = 6 * 1024 * 1024;
|
|
478
|
+
var BATCH_PIPELINES = 2;
|
|
479
|
+
async function sha256File(path3) {
|
|
480
|
+
const hash = createHash("sha256");
|
|
481
|
+
for await (const part of createReadStream2(path3)) hash.update(part);
|
|
482
|
+
return hash.digest("hex");
|
|
483
|
+
}
|
|
484
|
+
var sha256Bytes = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
485
|
+
var BUCKET_CODES = /* @__PURE__ */ new Set([
|
|
486
|
+
"bucket_unreachable",
|
|
487
|
+
"not_connected",
|
|
488
|
+
"platform_not_configured",
|
|
489
|
+
"upload_expired",
|
|
490
|
+
"file_mismatch",
|
|
491
|
+
"file_missing"
|
|
492
|
+
]);
|
|
493
|
+
|
|
473
494
|
// src/reporter-core/fallback.ts
|
|
474
495
|
import * as fs from "fs";
|
|
475
496
|
import * as path2 from "path";
|
|
@@ -582,7 +603,7 @@ function tally(counts, noun) {
|
|
|
582
603
|
}
|
|
583
604
|
function boundKey(key) {
|
|
584
605
|
if (key.value.length <= KEY_MAX) return key;
|
|
585
|
-
const digest =
|
|
606
|
+
const digest = createHash2("sha256").update(key.value).digest("hex").slice(0, 16);
|
|
586
607
|
let head = key.value.slice(0, KEY_MAX - digest.length - 1);
|
|
587
608
|
if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
|
|
588
609
|
return { ...key, value: `${head}#${digest}` };
|
|
@@ -644,6 +665,9 @@ async function startRun(passed, declared) {
|
|
|
644
665
|
const uploadRetries = Math.min(Math.max(Math.floor(cfg.uploadRetries ?? UPLOAD_RETRIES.default), UPLOAD_RETRIES.min), UPLOAD_RETRIES.max);
|
|
645
666
|
let retried = 0;
|
|
646
667
|
let uploadMs = 0;
|
|
668
|
+
let requests = 0;
|
|
669
|
+
let existed = 0;
|
|
670
|
+
let batchMissing = false;
|
|
647
671
|
const client = createClient({
|
|
648
672
|
apiUrl: resolveApiUrl(cfg.apiUrl),
|
|
649
673
|
token,
|
|
@@ -827,6 +851,7 @@ async function startRun(passed, declared) {
|
|
|
827
851
|
else log(`plune: ${KINDS[kind].label} "${name}" not uploaded \u2014 ${SKIPPED[why]}.`);
|
|
828
852
|
}
|
|
829
853
|
function ask(job, size) {
|
|
854
|
+
requests += 1;
|
|
830
855
|
return client.post(`/v1/results/${encodeURIComponent(job.id)}/files/uploads`, {
|
|
831
856
|
files: [{ name: job.name, contentType: job.type, size }]
|
|
832
857
|
}, uploadRetries);
|
|
@@ -868,6 +893,7 @@ async function startRun(passed, declared) {
|
|
|
868
893
|
}
|
|
869
894
|
const refusal = kind === "text" ? textRefusal(bytes) : void 0;
|
|
870
895
|
if (refusal !== void 0) return skip(kind, job.name, refusal);
|
|
896
|
+
requests += 1;
|
|
871
897
|
const out = await client.upload(
|
|
872
898
|
`/v1/results/${encodeURIComponent(job.id)}/files?name=${encodeURIComponent(job.name)}`,
|
|
873
899
|
bytes,
|
|
@@ -890,6 +916,7 @@ async function startRun(passed, declared) {
|
|
|
890
916
|
if (fresh === void 0) return failedToBucket(job, "bad_answer");
|
|
891
917
|
link = fresh;
|
|
892
918
|
}
|
|
919
|
+
requests += 1;
|
|
893
920
|
const done2 = await client.post(`/v1/files/${encodeURIComponent(link.id)}/complete`, {}, uploadRetries);
|
|
894
921
|
if (!done2.ok) return failedToBucket(job, codeOf(done2));
|
|
895
922
|
countsOf(job.kind).uploaded += 1;
|
|
@@ -950,6 +977,245 @@ async function startRun(passed, declared) {
|
|
|
950
977
|
}
|
|
951
978
|
return () => uploadToBucket(job, size, link);
|
|
952
979
|
}
|
|
980
|
+
async function uploadEach(jobs) {
|
|
981
|
+
const byResult = /* @__PURE__ */ new Map();
|
|
982
|
+
for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
|
|
983
|
+
await Promise.all(
|
|
984
|
+
[...byResult.values()].map(async (group) => {
|
|
985
|
+
const work = [];
|
|
986
|
+
for (const job of group) {
|
|
987
|
+
const act = await plan(job);
|
|
988
|
+
if (act !== void 0) work.push(gated(act));
|
|
989
|
+
}
|
|
990
|
+
await Promise.all(work);
|
|
991
|
+
})
|
|
992
|
+
);
|
|
993
|
+
}
|
|
994
|
+
async function measure(job) {
|
|
995
|
+
let size;
|
|
996
|
+
if (job.body !== void 0) size = job.body.length;
|
|
997
|
+
else if (job.path === void 0) return void skip(job.kind, job.name, "missing");
|
|
998
|
+
else {
|
|
999
|
+
try {
|
|
1000
|
+
size = (await disk.stat(job.path)).size;
|
|
1001
|
+
} catch {
|
|
1002
|
+
return void skip(job.kind, job.name, "missing");
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
if (size === 0) return void skip(job.kind, job.name, "empty");
|
|
1006
|
+
return size;
|
|
1007
|
+
}
|
|
1008
|
+
const routeMissing = (out) => !out.ok && out.status === 404 && out.detail === "" && out.code === void 0;
|
|
1009
|
+
function takeRoom(job) {
|
|
1010
|
+
if (job.kind === "file") {
|
|
1011
|
+
skip(job.kind, job.name, "noStorage");
|
|
1012
|
+
return false;
|
|
1013
|
+
}
|
|
1014
|
+
const roomKey = `${job.id}/${job.kind}`;
|
|
1015
|
+
const left = rooms.get(roomKey) ?? KINDS[job.kind].room;
|
|
1016
|
+
if (left === 0) {
|
|
1017
|
+
skip(job.kind, job.name, KINDS[job.kind].limit);
|
|
1018
|
+
return false;
|
|
1019
|
+
}
|
|
1020
|
+
rooms.set(roomKey, left - 1);
|
|
1021
|
+
return true;
|
|
1022
|
+
}
|
|
1023
|
+
function refusedItem(job, size, item) {
|
|
1024
|
+
const code = item.code ?? "refused";
|
|
1025
|
+
if (code === "duplicate") existed += 1;
|
|
1026
|
+
else if (code === "no_storage") skip(job.kind, job.name, refusedHere(job, size) ?? "noStorage");
|
|
1027
|
+
else if (code === "too_many_files") skip(job.kind, job.name, KINDS[job.kind].limit);
|
|
1028
|
+
else if (BUCKET_CODES.has(code)) failedToBucket(job, code);
|
|
1029
|
+
else failed(job, code);
|
|
1030
|
+
}
|
|
1031
|
+
const itemPause = (round) => new Promise((r) => setTimeout(r, Math.min(250 * 2 ** round, 4e3)));
|
|
1032
|
+
async function batchOf(jobs) {
|
|
1033
|
+
if (batchMissing) return uploadEach(jobs);
|
|
1034
|
+
const sized = [];
|
|
1035
|
+
for (const job of jobs) {
|
|
1036
|
+
const size = await measure(job);
|
|
1037
|
+
if (size === void 0) continue;
|
|
1038
|
+
try {
|
|
1039
|
+
sized.push({ job, size, sha256: job.body !== void 0 ? sha256Bytes(job.body) : await sha256File(job.path) });
|
|
1040
|
+
} catch {
|
|
1041
|
+
skip(job.kind, job.name, "missing");
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
if (sized.length === 0) return;
|
|
1045
|
+
const announced = await gated(() => {
|
|
1046
|
+
requests += 1;
|
|
1047
|
+
return client.post(
|
|
1048
|
+
"/v1/files/uploads-batch",
|
|
1049
|
+
{ items: sized.map((s) => ({ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 })) },
|
|
1050
|
+
uploadRetries
|
|
1051
|
+
);
|
|
1052
|
+
});
|
|
1053
|
+
if (routeMissing(announced)) {
|
|
1054
|
+
batchMissing = true;
|
|
1055
|
+
return uploadEach(sized.map((s) => s.job));
|
|
1056
|
+
}
|
|
1057
|
+
const verdicts = announced.ok ? announced.body?.items : void 0;
|
|
1058
|
+
if (!announced.ok || !Array.isArray(verdicts) || verdicts.length !== sized.length) {
|
|
1059
|
+
for (const s of sized) {
|
|
1060
|
+
if (announced.ok) failed(s.job, "bad_answer");
|
|
1061
|
+
else if (announced.code !== void 0) failedToBucket(s.job, announced.code);
|
|
1062
|
+
else failed(s.job, announced.detail || announced.kind);
|
|
1063
|
+
}
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
const puts = [];
|
|
1067
|
+
const directs = [];
|
|
1068
|
+
sized.forEach((s, i) => {
|
|
1069
|
+
const v = verdicts[i] ?? {};
|
|
1070
|
+
if (v.status === "exists") existed += 1;
|
|
1071
|
+
else if (v.status === "direct") {
|
|
1072
|
+
if (takeRoom(s.job)) directs.push(s);
|
|
1073
|
+
} else if (v.status === "upload") {
|
|
1074
|
+
const link = linkOf(v);
|
|
1075
|
+
if (link === void 0) return failed(s.job, "bad_answer");
|
|
1076
|
+
if (!bucketSeen) {
|
|
1077
|
+
bucketSeen = true;
|
|
1078
|
+
log(BUCKET_FOUND);
|
|
1079
|
+
}
|
|
1080
|
+
puts.push({ s, link });
|
|
1081
|
+
} else if (v.status === "refused") refusedItem(s.job, s.size, v);
|
|
1082
|
+
else failed(s.job, "bad_answer");
|
|
1083
|
+
});
|
|
1084
|
+
const toComplete = [];
|
|
1085
|
+
await Promise.all(
|
|
1086
|
+
puts.map(
|
|
1087
|
+
({ s, link: first }) => gated(async () => {
|
|
1088
|
+
const source = s.job.body !== void 0 ? { body: s.job.body } : { path: s.job.path, size: s.size };
|
|
1089
|
+
let link = first;
|
|
1090
|
+
for (let renewed = false; ; ) {
|
|
1091
|
+
const put = await client.put(link.url, link.headers, source);
|
|
1092
|
+
if (put.ok) break;
|
|
1093
|
+
if (put.status !== 403 || renewed) return failedToBucket(s.job, put.code);
|
|
1094
|
+
renewed = true;
|
|
1095
|
+
requests += 1;
|
|
1096
|
+
const again = await client.post(
|
|
1097
|
+
"/v1/files/uploads-batch",
|
|
1098
|
+
{ items: [{ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 }] },
|
|
1099
|
+
uploadRetries
|
|
1100
|
+
);
|
|
1101
|
+
const fresh = again.ok ? linkOf(again.body?.items?.[0]) : void 0;
|
|
1102
|
+
if (fresh === void 0) return failedToBucket(s.job, again.ok ? "bad_answer" : codeOf(again));
|
|
1103
|
+
link = fresh;
|
|
1104
|
+
}
|
|
1105
|
+
toComplete.push({ job: s.job, id: link.id });
|
|
1106
|
+
})
|
|
1107
|
+
)
|
|
1108
|
+
);
|
|
1109
|
+
await complete(toComplete);
|
|
1110
|
+
await sendDirect(directs);
|
|
1111
|
+
}
|
|
1112
|
+
async function complete(items) {
|
|
1113
|
+
for (let round = 0; items.length > 0; round += 1) {
|
|
1114
|
+
const out = await gated(() => {
|
|
1115
|
+
requests += 1;
|
|
1116
|
+
return client.post("/v1/files/complete-batch", { ids: items.map((i) => i.id) }, uploadRetries);
|
|
1117
|
+
});
|
|
1118
|
+
if (routeMissing(out)) {
|
|
1119
|
+
await Promise.all(
|
|
1120
|
+
items.map(
|
|
1121
|
+
(i) => gated(async () => {
|
|
1122
|
+
requests += 1;
|
|
1123
|
+
const one = await client.post(`/v1/files/${encodeURIComponent(i.id)}/complete`, {}, uploadRetries);
|
|
1124
|
+
if (one.ok) countsOf(i.job.kind).uploaded += 1;
|
|
1125
|
+
else failedToBucket(i.job, codeOf(one));
|
|
1126
|
+
})
|
|
1127
|
+
)
|
|
1128
|
+
);
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
const verdicts = out.ok ? out.body?.items : void 0;
|
|
1132
|
+
if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== items.length) {
|
|
1133
|
+
for (const i of items) failedToBucket(i.job, out.ok ? "bad_answer" : codeOf(out));
|
|
1134
|
+
return;
|
|
1135
|
+
}
|
|
1136
|
+
const again = [];
|
|
1137
|
+
items.forEach((i, at) => {
|
|
1138
|
+
const v = verdicts[at] ?? {};
|
|
1139
|
+
if (v.status === "ready") countsOf(i.job.kind).uploaded += 1;
|
|
1140
|
+
else if (v.retryable === true && round < Math.min(uploadRetries, 3)) again.push(i);
|
|
1141
|
+
else refusedItem(i.job, 0, v.status === "refused" ? v : { code: "bad_answer" });
|
|
1142
|
+
});
|
|
1143
|
+
items = again;
|
|
1144
|
+
if (items.length > 0) {
|
|
1145
|
+
retried += items.length;
|
|
1146
|
+
await itemPause(round);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
async function sendDirect(items) {
|
|
1151
|
+
const groups = [];
|
|
1152
|
+
let bytes = 0;
|
|
1153
|
+
for (const item of items) {
|
|
1154
|
+
const group = groups[groups.length - 1];
|
|
1155
|
+
if (group === void 0 || group.length >= FILES_BATCH_MAX || bytes + item.size > DIRECT_BATCH_BYTES) {
|
|
1156
|
+
groups.push([item]);
|
|
1157
|
+
bytes = item.size;
|
|
1158
|
+
} else {
|
|
1159
|
+
group.push(item);
|
|
1160
|
+
bytes += item.size;
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
for (const group of groups) {
|
|
1164
|
+
const body = [];
|
|
1165
|
+
const sent = [];
|
|
1166
|
+
for (const { job, size } of group) {
|
|
1167
|
+
const { kind } = job;
|
|
1168
|
+
if (size > KINDS[kind].bytes) {
|
|
1169
|
+
skip(kind, job.name, KINDS[kind].size);
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
let data;
|
|
1173
|
+
try {
|
|
1174
|
+
data = job.body ?? await disk.readFile(job.path);
|
|
1175
|
+
} catch {
|
|
1176
|
+
skip(kind, job.name, "missing");
|
|
1177
|
+
continue;
|
|
1178
|
+
}
|
|
1179
|
+
const refusal = kind === "text" ? textRefusal(data) : void 0;
|
|
1180
|
+
if (refusal !== void 0) {
|
|
1181
|
+
skip(kind, job.name, refusal);
|
|
1182
|
+
continue;
|
|
1183
|
+
}
|
|
1184
|
+
body.push({ resultId: job.id, name: job.name, contentType: job.type, data: data.toString("base64") });
|
|
1185
|
+
sent.push({ job, size });
|
|
1186
|
+
}
|
|
1187
|
+
if (body.length === 0) continue;
|
|
1188
|
+
const out = await gated(() => {
|
|
1189
|
+
requests += 1;
|
|
1190
|
+
return client.post("/v1/files/direct-batch", { items: body }, uploadRetries);
|
|
1191
|
+
});
|
|
1192
|
+
if (routeMissing(out)) {
|
|
1193
|
+
batchMissing = true;
|
|
1194
|
+
for (const { job, size } of sent) await gated(() => uploadDirect(job, size));
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
const verdicts = out.ok ? out.body?.items : void 0;
|
|
1198
|
+
if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== sent.length) {
|
|
1199
|
+
for (const { job } of sent) failed(job, out.ok ? "bad_answer" : out.detail || out.kind);
|
|
1200
|
+
continue;
|
|
1201
|
+
}
|
|
1202
|
+
sent.forEach(({ job }, i) => {
|
|
1203
|
+
const v = verdicts[i] ?? {};
|
|
1204
|
+
if (v.status === "stored") countsOf(job.kind).uploaded += 1;
|
|
1205
|
+
else if (v.status === "exists") existed += 1;
|
|
1206
|
+
else refusedItem(job, 0, v.status === "refused" ? v : { code: "bad_answer" });
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
async function uploadBatched(jobs) {
|
|
1211
|
+
const batches = chunk(jobs, FILES_BATCH_MAX);
|
|
1212
|
+
let next = 0;
|
|
1213
|
+
await Promise.all(
|
|
1214
|
+
Array.from({ length: Math.min(BATCH_PIPELINES, batches.length) }, async () => {
|
|
1215
|
+
for (let at = next++; at < batches.length; at = next++) await batchOf(batches[at]);
|
|
1216
|
+
})
|
|
1217
|
+
);
|
|
1218
|
+
}
|
|
953
1219
|
async function upload(items, filesByKey) {
|
|
954
1220
|
if (filesByKey.size === 0 || !Array.isArray(items)) return;
|
|
955
1221
|
const jobs = [];
|
|
@@ -975,18 +1241,8 @@ async function startRun(passed, declared) {
|
|
|
975
1241
|
}
|
|
976
1242
|
}
|
|
977
1243
|
const started = Date.now();
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
await Promise.all(
|
|
981
|
-
[...byResult.values()].map(async (group) => {
|
|
982
|
-
const work = [];
|
|
983
|
-
for (const job of group) {
|
|
984
|
-
const act = await plan(job);
|
|
985
|
-
if (act !== void 0) work.push(gated(act));
|
|
986
|
-
}
|
|
987
|
-
await Promise.all(work);
|
|
988
|
-
})
|
|
989
|
-
);
|
|
1244
|
+
if (batchMissing) await uploadEach(jobs);
|
|
1245
|
+
else await uploadBatched(jobs);
|
|
990
1246
|
uploadMs += Date.now() - started;
|
|
991
1247
|
}
|
|
992
1248
|
async function flush() {
|
|
@@ -1082,9 +1338,9 @@ async function startRun(passed, declared) {
|
|
|
1082
1338
|
if (shortened.size > 0) parts.push(`${shortened.size} test name(s) shortened to fit a key`);
|
|
1083
1339
|
const files = [stats.screenshots, stats.textFiles, stats.otherFiles];
|
|
1084
1340
|
const total = (what) => files.reduce((n, counts) => n + counts[what], 0);
|
|
1085
|
-
if (total("uploaded") + total("skipped") + total("failed") > 0) {
|
|
1341
|
+
if (total("uploaded") + total("skipped") + total("failed") + existed > 0) {
|
|
1086
1342
|
log(
|
|
1087
|
-
`plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once)`
|
|
1343
|
+
`plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${existed} already there (skipped), ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once, ${requests} request(s) to Plune)`
|
|
1088
1344
|
);
|
|
1089
1345
|
}
|
|
1090
1346
|
log(`plune: ${parts.join(" \xB7 ")}`);
|
|
@@ -1143,9 +1399,9 @@ async function startRun(passed, declared) {
|
|
|
1143
1399
|
}
|
|
1144
1400
|
|
|
1145
1401
|
// src/util/hash.ts
|
|
1146
|
-
import { createHash as
|
|
1402
|
+
import { createHash as createHash3 } from "crypto";
|
|
1147
1403
|
function sha256(input) {
|
|
1148
|
-
return
|
|
1404
|
+
return createHash3("sha256").update(input, "utf8").digest("hex");
|
|
1149
1405
|
}
|
|
1150
1406
|
|
|
1151
1407
|
// src/reporter-core/result-key.ts
|
package/package.json
CHANGED