@plune-ai/cli 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -324,8 +324,13 @@ without closing.
324
324
 
325
325
  ### Many screenshots: upload concurrency and retries
326
326
 
327
- Every file is its own request, so a report with thousands of screenshots is bound by the round trip, not
328
- by bandwidth. Files go four at a time by default; raise it on a runner with a slow link:
327
+ Files go to Plune up to 100 to a request: `uploads-batch` announces them (with the SHA-256 of each, read as a stream,
328
+ so a file the platform already has is answered `exists` and is not sent again), the bytes go straight to your bucket
329
+ (or, for a project without one, 100 files or 6 MiB at a time through `direct-batch`), and `complete-batch` confirms
330
+ them. Every file has its own verdict: one that is refused is reported and the rest of its request goes through. A
331
+ report of 2760 screenshots is about 56 requests to Plune instead of 5500. The bytes themselves still go one request each,
332
+ so the PUTs to a bucket are bound by the round trip, not by bandwidth: four at a time by default, raise it on a runner
333
+ with a slow link:
329
334
 
330
335
  ```bash
331
336
  PLUNE_UPLOAD_CONCURRENCY=16 plune run import ./playwright.json # 1 to 32, default 4
@@ -336,11 +341,14 @@ When the platform answers 429 with a `Retry-After` (seconds or an HTTP date, nev
336
341
  the request is tried again after that time plus up to half as much, and every other request of the import
337
342
  waits through the same pause. `PLUNE_UPLOAD_RETRIES` (0 to 20, default 6) says how many times; a 429 that
338
343
  names no time is the quota and is not retried. A platform that is down (a 5xx or no answer) is still tried
339
- twice (three times for a bucket). A file that stays refused is reported without it, never fails the import,
340
- and is counted in the line printed before the summary:
341
- `plune: files — 2758 uploaded, 0 skipped, 14 retried, 0 dropped, in 41.2 s (16 at once)`.
342
- A value outside the range is ignored with a warning, and the default is used. The Playwright reporter reads
343
- the same variables.
344
+ twice (three times for a bucket). A 429 on a whole request waits the same way; a file that the bucket did not
345
+ answer for at `complete` is confirmed again, on its own. A file that stays refused is reported without it, never fails
346
+ the import, and is counted in the line printed before the summary:
347
+ `plune: files — 2758 uploaded, 0 skipped, 0 already there (skipped), 14 retried, 0 dropped, in 41.2 s (16 at once, 56 request(s) to Plune)`.
348
+ `already there` are the files Plune had from an earlier import of the same report (same result, name, type and bytes):
349
+ none of them is sent again. Against a platform older than the bulk routes (it answers 404 to `uploads-batch`) the CLI
350
+ sends one request per file, as before; it decides by that answer, not by a version. A value outside the range is
351
+ ignored with a warning, and the default is used. The Playwright reporter reads the same variables.
344
352
 
345
353
  ### When a test is deleted
346
354
 
package/dist/cli.cjs CHANGED
@@ -2789,6 +2789,34 @@ var init_failure_detail = __esm({
2789
2789
  }
2790
2790
  });
2791
2791
 
2792
+ // src/reporter-core/files-batch.ts
2793
+ async function sha256File(path17) {
2794
+ const hash = (0, import_node_crypto2.createHash)("sha256");
2795
+ for await (const part of (0, import_node_fs4.createReadStream)(path17)) hash.update(part);
2796
+ return hash.digest("hex");
2797
+ }
2798
+ var import_node_crypto2, import_node_fs4, FILES_BATCH_MAX, DIRECT_BATCH_BYTES, BATCH_PIPELINES, sha256Bytes, BUCKET_CODES;
2799
+ var init_files_batch = __esm({
2800
+ "src/reporter-core/files-batch.ts"() {
2801
+ "use strict";
2802
+ init_cjs_shims();
2803
+ import_node_crypto2 = require("crypto");
2804
+ import_node_fs4 = require("fs");
2805
+ FILES_BATCH_MAX = 100;
2806
+ DIRECT_BATCH_BYTES = 6 * 1024 * 1024;
2807
+ BATCH_PIPELINES = 2;
2808
+ sha256Bytes = (bytes) => (0, import_node_crypto2.createHash)("sha256").update(bytes).digest("hex");
2809
+ BUCKET_CODES = /* @__PURE__ */ new Set([
2810
+ "bucket_unreachable",
2811
+ "not_connected",
2812
+ "platform_not_configured",
2813
+ "upload_expired",
2814
+ "file_mismatch",
2815
+ "file_missing"
2816
+ ]);
2817
+ }
2818
+ });
2819
+
2792
2820
  // src/reporter-core/session.ts
2793
2821
  function nameOf(file) {
2794
2822
  const own = file.name.split(/[\\/]/).pop()?.trim() || (file.path === void 0 ? "" : (0, import_node_path3.basename)(file.path));
@@ -2840,7 +2868,7 @@ function tally(counts, noun) {
2840
2868
  }
2841
2869
  function boundKey(key) {
2842
2870
  if (key.value.length <= KEY_MAX) return key;
2843
- const digest = (0, import_node_crypto2.createHash)("sha256").update(key.value).digest("hex").slice(0, 16);
2871
+ const digest = (0, import_node_crypto3.createHash)("sha256").update(key.value).digest("hex").slice(0, 16);
2844
2872
  let head = key.value.slice(0, KEY_MAX - digest.length - 1);
2845
2873
  if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
2846
2874
  return { ...key, value: `${head}#${digest}` };
@@ -2892,7 +2920,7 @@ async function startRun(passed, declared2) {
2892
2920
  `));
2893
2921
  const fallbackPath = cfg.fallbackPath ?? DEFAULT_FALLBACK_PATH;
2894
2922
  const externalKey = cfg.externalKey ?? null;
2895
- const sessionMarker = (0, import_node_crypto2.randomUUID)();
2923
+ const sessionMarker = (0, import_node_crypto3.randomUUID)();
2896
2924
  const batchSize = Math.min(cfg.batchSize ?? BATCH_DEFAULT, BATCH_MAX);
2897
2925
  const offerDiscovered = cfg.offerDiscovered ?? false;
2898
2926
  const token = cfg.token ?? loadToken() ?? "";
@@ -2900,6 +2928,9 @@ async function startRun(passed, declared2) {
2900
2928
  const uploadRetries = Math.min(Math.max(Math.floor(cfg.uploadRetries ?? UPLOAD_RETRIES.default), UPLOAD_RETRIES.min), UPLOAD_RETRIES.max);
2901
2929
  let retried = 0;
2902
2930
  let uploadMs = 0;
2931
+ let requests = 0;
2932
+ let existed = 0;
2933
+ let batchMissing = false;
2903
2934
  const client = createClient({
2904
2935
  apiUrl: resolveApiUrl(cfg.apiUrl),
2905
2936
  token,
@@ -3083,6 +3114,7 @@ async function startRun(passed, declared2) {
3083
3114
  else log(`plune: ${KINDS[kind].label} "${name}" not uploaded \u2014 ${SKIPPED[why]}.`);
3084
3115
  }
3085
3116
  function ask(job, size) {
3117
+ requests += 1;
3086
3118
  return client.post(`/v1/results/${encodeURIComponent(job.id)}/files/uploads`, {
3087
3119
  files: [{ name: job.name, contentType: job.type, size }]
3088
3120
  }, uploadRetries);
@@ -3117,13 +3149,14 @@ async function startRun(passed, declared2) {
3117
3149
  if (job.body !== void 0) bytes = job.body;
3118
3150
  else {
3119
3151
  try {
3120
- bytes = await import_node_fs4.promises.readFile(job.path);
3152
+ bytes = await import_node_fs5.promises.readFile(job.path);
3121
3153
  } catch {
3122
3154
  return skip(kind, job.name, "missing");
3123
3155
  }
3124
3156
  }
3125
3157
  const refusal = kind === "text" ? textRefusal(bytes) : void 0;
3126
3158
  if (refusal !== void 0) return skip(kind, job.name, refusal);
3159
+ requests += 1;
3127
3160
  const out = await client.upload(
3128
3161
  `/v1/results/${encodeURIComponent(job.id)}/files?name=${encodeURIComponent(job.name)}`,
3129
3162
  bytes,
@@ -3146,6 +3179,7 @@ async function startRun(passed, declared2) {
3146
3179
  if (fresh === void 0) return failedToBucket(job, "bad_answer");
3147
3180
  link = fresh;
3148
3181
  }
3182
+ requests += 1;
3149
3183
  const done2 = await client.post(`/v1/files/${encodeURIComponent(link.id)}/complete`, {}, uploadRetries);
3150
3184
  if (!done2.ok) return failedToBucket(job, codeOf2(done2));
3151
3185
  countsOf(job.kind).uploaded += 1;
@@ -3170,7 +3204,7 @@ async function startRun(passed, declared2) {
3170
3204
  else if (job.path === void 0) return void skip(kind, job.name, "missing");
3171
3205
  else {
3172
3206
  try {
3173
- size = (await import_node_fs4.promises.stat(job.path)).size;
3207
+ size = (await import_node_fs5.promises.stat(job.path)).size;
3174
3208
  } catch {
3175
3209
  return void skip(kind, job.name, "missing");
3176
3210
  }
@@ -3206,6 +3240,245 @@ async function startRun(passed, declared2) {
3206
3240
  }
3207
3241
  return () => uploadToBucket(job, size, link);
3208
3242
  }
3243
+ async function uploadEach(jobs) {
3244
+ const byResult = /* @__PURE__ */ new Map();
3245
+ for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
3246
+ await Promise.all(
3247
+ [...byResult.values()].map(async (group) => {
3248
+ const work = [];
3249
+ for (const job of group) {
3250
+ const act = await plan(job);
3251
+ if (act !== void 0) work.push(gated(act));
3252
+ }
3253
+ await Promise.all(work);
3254
+ })
3255
+ );
3256
+ }
3257
+ async function measure(job) {
3258
+ let size;
3259
+ if (job.body !== void 0) size = job.body.length;
3260
+ else if (job.path === void 0) return void skip(job.kind, job.name, "missing");
3261
+ else {
3262
+ try {
3263
+ size = (await import_node_fs5.promises.stat(job.path)).size;
3264
+ } catch {
3265
+ return void skip(job.kind, job.name, "missing");
3266
+ }
3267
+ }
3268
+ if (size === 0) return void skip(job.kind, job.name, "empty");
3269
+ return size;
3270
+ }
3271
+ const routeMissing = (out) => !out.ok && out.status === 404 && out.detail === "" && out.code === void 0;
3272
+ function takeRoom(job) {
3273
+ if (job.kind === "file") {
3274
+ skip(job.kind, job.name, "noStorage");
3275
+ return false;
3276
+ }
3277
+ const roomKey = `${job.id}/${job.kind}`;
3278
+ const left = rooms.get(roomKey) ?? KINDS[job.kind].room;
3279
+ if (left === 0) {
3280
+ skip(job.kind, job.name, KINDS[job.kind].limit);
3281
+ return false;
3282
+ }
3283
+ rooms.set(roomKey, left - 1);
3284
+ return true;
3285
+ }
3286
+ function refusedItem(job, size, item) {
3287
+ const code = item.code ?? "refused";
3288
+ if (code === "duplicate") existed += 1;
3289
+ else if (code === "no_storage") skip(job.kind, job.name, refusedHere(job, size) ?? "noStorage");
3290
+ else if (code === "too_many_files") skip(job.kind, job.name, KINDS[job.kind].limit);
3291
+ else if (BUCKET_CODES.has(code)) failedToBucket(job, code);
3292
+ else failed(job, code);
3293
+ }
3294
+ const itemPause = (round) => new Promise((r) => setTimeout(r, Math.min(250 * 2 ** round, 4e3)));
3295
+ async function batchOf(jobs) {
3296
+ if (batchMissing) return uploadEach(jobs);
3297
+ const sized = [];
3298
+ for (const job of jobs) {
3299
+ const size = await measure(job);
3300
+ if (size === void 0) continue;
3301
+ try {
3302
+ sized.push({ job, size, sha256: job.body !== void 0 ? sha256Bytes(job.body) : await sha256File(job.path) });
3303
+ } catch {
3304
+ skip(job.kind, job.name, "missing");
3305
+ }
3306
+ }
3307
+ if (sized.length === 0) return;
3308
+ const announced = await gated(() => {
3309
+ requests += 1;
3310
+ return client.post(
3311
+ "/v1/files/uploads-batch",
3312
+ { items: sized.map((s) => ({ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 })) },
3313
+ uploadRetries
3314
+ );
3315
+ });
3316
+ if (routeMissing(announced)) {
3317
+ batchMissing = true;
3318
+ return uploadEach(sized.map((s) => s.job));
3319
+ }
3320
+ const verdicts = announced.ok ? announced.body?.items : void 0;
3321
+ if (!announced.ok || !Array.isArray(verdicts) || verdicts.length !== sized.length) {
3322
+ for (const s of sized) {
3323
+ if (announced.ok) failed(s.job, "bad_answer");
3324
+ else if (announced.code !== void 0) failedToBucket(s.job, announced.code);
3325
+ else failed(s.job, announced.detail || announced.kind);
3326
+ }
3327
+ return;
3328
+ }
3329
+ const puts = [];
3330
+ const directs = [];
3331
+ sized.forEach((s, i) => {
3332
+ const v = verdicts[i] ?? {};
3333
+ if (v.status === "exists") existed += 1;
3334
+ else if (v.status === "direct") {
3335
+ if (takeRoom(s.job)) directs.push(s);
3336
+ } else if (v.status === "upload") {
3337
+ const link = linkOf(v);
3338
+ if (link === void 0) return failed(s.job, "bad_answer");
3339
+ if (!bucketSeen) {
3340
+ bucketSeen = true;
3341
+ log(BUCKET_FOUND);
3342
+ }
3343
+ puts.push({ s, link });
3344
+ } else if (v.status === "refused") refusedItem(s.job, s.size, v);
3345
+ else failed(s.job, "bad_answer");
3346
+ });
3347
+ const toComplete = [];
3348
+ await Promise.all(
3349
+ puts.map(
3350
+ ({ s, link: first }) => gated(async () => {
3351
+ const source = s.job.body !== void 0 ? { body: s.job.body } : { path: s.job.path, size: s.size };
3352
+ let link = first;
3353
+ for (let renewed = false; ; ) {
3354
+ const put = await client.put(link.url, link.headers, source);
3355
+ if (put.ok) break;
3356
+ if (put.status !== 403 || renewed) return failedToBucket(s.job, put.code);
3357
+ renewed = true;
3358
+ requests += 1;
3359
+ const again = await client.post(
3360
+ "/v1/files/uploads-batch",
3361
+ { items: [{ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 }] },
3362
+ uploadRetries
3363
+ );
3364
+ const fresh = again.ok ? linkOf(again.body?.items?.[0]) : void 0;
3365
+ if (fresh === void 0) return failedToBucket(s.job, again.ok ? "bad_answer" : codeOf2(again));
3366
+ link = fresh;
3367
+ }
3368
+ toComplete.push({ job: s.job, id: link.id });
3369
+ })
3370
+ )
3371
+ );
3372
+ await complete(toComplete);
3373
+ await sendDirect(directs);
3374
+ }
3375
+ async function complete(items) {
3376
+ for (let round = 0; items.length > 0; round += 1) {
3377
+ const out = await gated(() => {
3378
+ requests += 1;
3379
+ return client.post("/v1/files/complete-batch", { ids: items.map((i) => i.id) }, uploadRetries);
3380
+ });
3381
+ if (routeMissing(out)) {
3382
+ await Promise.all(
3383
+ items.map(
3384
+ (i) => gated(async () => {
3385
+ requests += 1;
3386
+ const one = await client.post(`/v1/files/${encodeURIComponent(i.id)}/complete`, {}, uploadRetries);
3387
+ if (one.ok) countsOf(i.job.kind).uploaded += 1;
3388
+ else failedToBucket(i.job, codeOf2(one));
3389
+ })
3390
+ )
3391
+ );
3392
+ return;
3393
+ }
3394
+ const verdicts = out.ok ? out.body?.items : void 0;
3395
+ if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== items.length) {
3396
+ for (const i of items) failedToBucket(i.job, out.ok ? "bad_answer" : codeOf2(out));
3397
+ return;
3398
+ }
3399
+ const again = [];
3400
+ items.forEach((i, at) => {
3401
+ const v = verdicts[at] ?? {};
3402
+ if (v.status === "ready") countsOf(i.job.kind).uploaded += 1;
3403
+ else if (v.retryable === true && round < Math.min(uploadRetries, 3)) again.push(i);
3404
+ else refusedItem(i.job, 0, v.status === "refused" ? v : { code: "bad_answer" });
3405
+ });
3406
+ items = again;
3407
+ if (items.length > 0) {
3408
+ retried += items.length;
3409
+ await itemPause(round);
3410
+ }
3411
+ }
3412
+ }
3413
+ async function sendDirect(items) {
3414
+ const groups = [];
3415
+ let bytes = 0;
3416
+ for (const item of items) {
3417
+ const group = groups[groups.length - 1];
3418
+ if (group === void 0 || group.length >= FILES_BATCH_MAX || bytes + item.size > DIRECT_BATCH_BYTES) {
3419
+ groups.push([item]);
3420
+ bytes = item.size;
3421
+ } else {
3422
+ group.push(item);
3423
+ bytes += item.size;
3424
+ }
3425
+ }
3426
+ for (const group of groups) {
3427
+ const body = [];
3428
+ const sent = [];
3429
+ for (const { job, size } of group) {
3430
+ const { kind } = job;
3431
+ if (size > KINDS[kind].bytes) {
3432
+ skip(kind, job.name, KINDS[kind].size);
3433
+ continue;
3434
+ }
3435
+ let data;
3436
+ try {
3437
+ data = job.body ?? await import_node_fs5.promises.readFile(job.path);
3438
+ } catch {
3439
+ skip(kind, job.name, "missing");
3440
+ continue;
3441
+ }
3442
+ const refusal = kind === "text" ? textRefusal(data) : void 0;
3443
+ if (refusal !== void 0) {
3444
+ skip(kind, job.name, refusal);
3445
+ continue;
3446
+ }
3447
+ body.push({ resultId: job.id, name: job.name, contentType: job.type, data: data.toString("base64") });
3448
+ sent.push({ job, size });
3449
+ }
3450
+ if (body.length === 0) continue;
3451
+ const out = await gated(() => {
3452
+ requests += 1;
3453
+ return client.post("/v1/files/direct-batch", { items: body }, uploadRetries);
3454
+ });
3455
+ if (routeMissing(out)) {
3456
+ batchMissing = true;
3457
+ for (const { job, size } of sent) await gated(() => uploadDirect(job, size));
3458
+ continue;
3459
+ }
3460
+ const verdicts = out.ok ? out.body?.items : void 0;
3461
+ if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== sent.length) {
3462
+ for (const { job } of sent) failed(job, out.ok ? "bad_answer" : out.detail || out.kind);
3463
+ continue;
3464
+ }
3465
+ sent.forEach(({ job }, i) => {
3466
+ const v = verdicts[i] ?? {};
3467
+ if (v.status === "stored") countsOf(job.kind).uploaded += 1;
3468
+ else if (v.status === "exists") existed += 1;
3469
+ else refusedItem(job, 0, v.status === "refused" ? v : { code: "bad_answer" });
3470
+ });
3471
+ }
3472
+ }
3473
+ async function uploadBatched(jobs) {
3474
+ const batches = chunk(jobs, FILES_BATCH_MAX);
3475
+ let next = 0;
3476
+ await Promise.all(
3477
+ Array.from({ length: Math.min(BATCH_PIPELINES, batches.length) }, async () => {
3478
+ for (let at = next++; at < batches.length; at = next++) await batchOf(batches[at]);
3479
+ })
3480
+ );
3481
+ }
3209
3482
  async function upload(items, filesByKey) {
3210
3483
  if (filesByKey.size === 0 || !Array.isArray(items)) return;
3211
3484
  const jobs = [];
@@ -3231,18 +3504,8 @@ async function startRun(passed, declared2) {
3231
3504
  }
3232
3505
  }
3233
3506
  const started = Date.now();
3234
- const byResult = /* @__PURE__ */ new Map();
3235
- for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
3236
- await Promise.all(
3237
- [...byResult.values()].map(async (group) => {
3238
- const work = [];
3239
- for (const job of group) {
3240
- const act = await plan(job);
3241
- if (act !== void 0) work.push(gated(act));
3242
- }
3243
- await Promise.all(work);
3244
- })
3245
- );
3507
+ if (batchMissing) await uploadEach(jobs);
3508
+ else await uploadBatched(jobs);
3246
3509
  uploadMs += Date.now() - started;
3247
3510
  }
3248
3511
  async function flush() {
@@ -3338,9 +3601,9 @@ async function startRun(passed, declared2) {
3338
3601
  if (shortened.size > 0) parts.push(`${shortened.size} test name(s) shortened to fit a key`);
3339
3602
  const files = [stats.screenshots, stats.textFiles, stats.otherFiles];
3340
3603
  const total = (what) => files.reduce((n, counts) => n + counts[what], 0);
3341
- if (total("uploaded") + total("skipped") + total("failed") > 0) {
3604
+ if (total("uploaded") + total("skipped") + total("failed") + existed > 0) {
3342
3605
  log(
3343
- `plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once)`
3606
+ `plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${existed} already there (skipped), ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once, ${requests} request(s) to Plune)`
3344
3607
  );
3345
3608
  }
3346
3609
  log(`plune: ${parts.join(" \xB7 ")}`);
@@ -3397,13 +3660,13 @@ async function startRun(passed, declared2) {
3397
3660
  }
3398
3661
  };
3399
3662
  }
3400
- var import_node_crypto2, import_node_fs4, import_node_path3, import_node_util2, RESOLVE_CHUNK, BATCH_MAX, DISCOVER_MAX, BATCH_DEFAULT, EXPECTED_MAX, KEY_MAX, TITLE_MAX, COUNTED, SCREENSHOT_TYPES, SCREENSHOT_BYTES, SCREENSHOTS_PER_RESULT, TEXT_BYTES, TEXTS_PER_RESULT, UTF8, TYPE_BY_EXTENSION, METADATA_TYPE2, NAME_MAX, SKIPPED, BUCKET_FOUND, KINDS, codeOf2, BATCH_BYTES, ENVELOPE_BYTES;
3663
+ var import_node_crypto3, import_node_fs5, import_node_path3, import_node_util2, RESOLVE_CHUNK, BATCH_MAX, DISCOVER_MAX, BATCH_DEFAULT, EXPECTED_MAX, KEY_MAX, TITLE_MAX, COUNTED, SCREENSHOT_TYPES, SCREENSHOT_BYTES, SCREENSHOTS_PER_RESULT, TEXT_BYTES, TEXTS_PER_RESULT, UTF8, TYPE_BY_EXTENSION, METADATA_TYPE2, NAME_MAX, SKIPPED, BUCKET_FOUND, KINDS, codeOf2, BATCH_BYTES, ENVELOPE_BYTES;
3401
3664
  var init_session = __esm({
3402
3665
  "src/reporter-core/session.ts"() {
3403
3666
  "use strict";
3404
3667
  init_cjs_shims();
3405
- import_node_crypto2 = require("crypto");
3406
- import_node_fs4 = require("fs");
3668
+ import_node_crypto3 = require("crypto");
3669
+ import_node_fs5 = require("fs");
3407
3670
  import_node_path3 = require("path");
3408
3671
  import_node_util2 = require("util");
3409
3672
  init_api_url();
@@ -3411,6 +3674,7 @@ var init_session = __esm({
3411
3674
  init_client();
3412
3675
  init_env2();
3413
3676
  init_failure_detail();
3677
+ init_files_batch();
3414
3678
  init_fallback();
3415
3679
  RESOLVE_CHUNK = 500;
3416
3680
  BATCH_MAX = 500;
@@ -4878,7 +5142,7 @@ function readPlaywrightJson(source, file) {
4878
5142
  const { config, suites } = parsed;
4879
5143
  const rootDir = typeof config?.rootDir === "string" ? config.rootDir : void 0;
4880
5144
  const buildHref = config?.metadata?.ci?.buildHref;
4881
- const repoRoot = rootDir !== void 0 && (0, import_node_fs5.existsSync)(rootDir) ? repoRootOf(rootDir) : void 0;
5145
+ const repoRoot = rootDir !== void 0 && (0, import_node_fs6.existsSync)(rootDir) ? repoRootOf(rootDir) : void 0;
4882
5146
  const ctx = {
4883
5147
  ...rootDir !== void 0 ? { rootDir } : {},
4884
5148
  ...repoRoot !== void 0 ? { repoRoot } : {},
@@ -4897,12 +5161,12 @@ function readPlaywrightJson(source, file) {
4897
5161
  function looksLikePlaywrightJson(source) {
4898
5162
  return /^\s*\{/.test(source) && source.includes('"suites"');
4899
5163
  }
4900
- var import_node_fs5, SOURCE3, TOKEN3, NOT_FAILED, JsonReportError;
5164
+ var import_node_fs6, SOURCE3, TOKEN3, NOT_FAILED, JsonReportError;
4901
5165
  var init_playwright_json = __esm({
4902
5166
  "src/importers/playwright-json.ts"() {
4903
5167
  "use strict";
4904
5168
  init_cjs_shims();
4905
- import_node_fs5 = require("fs");
5169
+ import_node_fs6 = require("fs");
4906
5170
  init_failure_detail();
4907
5171
  init_result_key();
4908
5172
  SOURCE3 = "playwright";
@@ -5930,11 +6194,11 @@ module.exports = __toCommonJS(cli_exports);
5930
6194
  init_cjs_shims();
5931
6195
  init_api_url();
5932
6196
  var import_commander = require("commander");
5933
- var import_node_fs6 = require("fs");
6197
+ var import_node_fs7 = require("fs");
5934
6198
  var import_node_path4 = require("path");
5935
6199
  var import_node_url = require("url");
5936
6200
  var __dirname = (0, import_node_path4.dirname)((0, import_node_url.fileURLToPath)(importMetaUrl));
5937
- var pkg = JSON.parse((0, import_node_fs6.readFileSync)((0, import_node_path4.join)(__dirname, "..", "package.json"), "utf-8"));
6201
+ var pkg = JSON.parse((0, import_node_fs7.readFileSync)((0, import_node_path4.join)(__dirname, "..", "package.json"), "utf-8"));
5938
6202
  function toFormat(s) {
5939
6203
  return s === "json" || s === "markdown" ? s : "console";
5940
6204
  }
@@ -5943,7 +6207,7 @@ function shouldColor(outputPath) {
5943
6207
  }
5944
6208
  function writeReport(text2, outputPath) {
5945
6209
  if (outputPath !== void 0) {
5946
- (0, import_node_fs6.writeFileSync)(outputPath, text2);
6210
+ (0, import_node_fs7.writeFileSync)(outputPath, text2);
5947
6211
  } else {
5948
6212
  process.stdout.write(text2 + "\n");
5949
6213
  }
@@ -6170,7 +6434,7 @@ function createProgram() {
6170
6434
  false
6171
6435
  ).option(
6172
6436
  "--upload-concurrency <n>",
6173
- "Files uploaded at once, 1 to 32 (default 4; PLUNE_UPLOAD_CONCURRENCY sets it too). A 429 is retried PLUNE_UPLOAD_RETRIES times (default 6)",
6437
+ "File uploads in flight at once, 1 to 32 (default 4; PLUNE_UPLOAD_CONCURRENCY sets it too). Files are announced and confirmed up to 100 to a request; a 429 is retried PLUNE_UPLOAD_RETRIES times (default 6)",
6174
6438
  (v) => {
6175
6439
  if (!/^\d+$/.test(v) || Number(v) < 1 || Number(v) > 32) throw new import_commander.InvalidArgumentError("a whole number from 1 to 32.");
6176
6440
  return Number(v);
@@ -6442,7 +6706,7 @@ function isMainModule() {
6442
6706
  const entry = process.argv[1];
6443
6707
  if (entry === void 0) return false;
6444
6708
  try {
6445
- return (0, import_node_fs6.realpathSync)(entry) === (0, import_node_fs6.realpathSync)((0, import_node_url.fileURLToPath)(importMetaUrl));
6709
+ return (0, import_node_fs7.realpathSync)(entry) === (0, import_node_fs7.realpathSync)((0, import_node_url.fileURLToPath)(importMetaUrl));
6446
6710
  } catch {
6447
6711
  return false;
6448
6712
  }
@@ -48,8 +48,8 @@ __export(reporter_core_exports, {
48
48
  module.exports = __toCommonJS(reporter_core_exports);
49
49
 
50
50
  // src/reporter-core/session.ts
51
- var import_node_crypto = require("crypto");
52
- var import_node_fs4 = require("fs");
51
+ var import_node_crypto2 = require("crypto");
52
+ var import_node_fs5 = require("fs");
53
53
  var import_node_path3 = require("path");
54
54
  var import_node_util2 = require("util");
55
55
 
@@ -519,6 +519,27 @@ function webLink(href) {
519
519
  return href !== void 0 && /^https?:\/\//i.test(href) && URL.canParse(href) ? href : void 0;
520
520
  }
521
521
 
522
+ // src/reporter-core/files-batch.ts
523
+ var import_node_crypto = require("crypto");
524
+ var import_node_fs4 = require("fs");
525
+ var FILES_BATCH_MAX = 100;
526
+ var DIRECT_BATCH_BYTES = 6 * 1024 * 1024;
527
+ var BATCH_PIPELINES = 2;
528
+ async function sha256File(path3) {
529
+ const hash = (0, import_node_crypto.createHash)("sha256");
530
+ for await (const part of (0, import_node_fs4.createReadStream)(path3)) hash.update(part);
531
+ return hash.digest("hex");
532
+ }
533
+ var sha256Bytes = (bytes) => (0, import_node_crypto.createHash)("sha256").update(bytes).digest("hex");
534
+ var BUCKET_CODES = /* @__PURE__ */ new Set([
535
+ "bucket_unreachable",
536
+ "not_connected",
537
+ "platform_not_configured",
538
+ "upload_expired",
539
+ "file_mismatch",
540
+ "file_missing"
541
+ ]);
542
+
522
543
  // src/reporter-core/fallback.ts
523
544
  var fs = __toESM(require("fs"), 1);
524
545
  var path2 = __toESM(require("path"), 1);
@@ -631,7 +652,7 @@ function tally(counts, noun) {
631
652
  }
632
653
  function boundKey(key) {
633
654
  if (key.value.length <= KEY_MAX) return key;
634
- const digest = (0, import_node_crypto.createHash)("sha256").update(key.value).digest("hex").slice(0, 16);
655
+ const digest = (0, import_node_crypto2.createHash)("sha256").update(key.value).digest("hex").slice(0, 16);
635
656
  let head = key.value.slice(0, KEY_MAX - digest.length - 1);
636
657
  if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
637
658
  return { ...key, value: `${head}#${digest}` };
@@ -685,7 +706,7 @@ async function startRun(passed, declared) {
685
706
  `));
686
707
  const fallbackPath = cfg.fallbackPath ?? DEFAULT_FALLBACK_PATH;
687
708
  const externalKey = cfg.externalKey ?? null;
688
- const sessionMarker = (0, import_node_crypto.randomUUID)();
709
+ const sessionMarker = (0, import_node_crypto2.randomUUID)();
689
710
  const batchSize = Math.min(cfg.batchSize ?? BATCH_DEFAULT, BATCH_MAX);
690
711
  const offerDiscovered = cfg.offerDiscovered ?? false;
691
712
  const token = cfg.token ?? loadToken() ?? "";
@@ -693,6 +714,9 @@ async function startRun(passed, declared) {
693
714
  const uploadRetries = Math.min(Math.max(Math.floor(cfg.uploadRetries ?? UPLOAD_RETRIES.default), UPLOAD_RETRIES.min), UPLOAD_RETRIES.max);
694
715
  let retried = 0;
695
716
  let uploadMs = 0;
717
+ let requests = 0;
718
+ let existed = 0;
719
+ let batchMissing = false;
696
720
  const client = createClient({
697
721
  apiUrl: resolveApiUrl(cfg.apiUrl),
698
722
  token,
@@ -876,6 +900,7 @@ async function startRun(passed, declared) {
876
900
  else log(`plune: ${KINDS[kind].label} "${name}" not uploaded \u2014 ${SKIPPED[why]}.`);
877
901
  }
878
902
  function ask(job, size) {
903
+ requests += 1;
879
904
  return client.post(`/v1/results/${encodeURIComponent(job.id)}/files/uploads`, {
880
905
  files: [{ name: job.name, contentType: job.type, size }]
881
906
  }, uploadRetries);
@@ -910,13 +935,14 @@ async function startRun(passed, declared) {
910
935
  if (job.body !== void 0) bytes = job.body;
911
936
  else {
912
937
  try {
913
- bytes = await import_node_fs4.promises.readFile(job.path);
938
+ bytes = await import_node_fs5.promises.readFile(job.path);
914
939
  } catch {
915
940
  return skip(kind, job.name, "missing");
916
941
  }
917
942
  }
918
943
  const refusal = kind === "text" ? textRefusal(bytes) : void 0;
919
944
  if (refusal !== void 0) return skip(kind, job.name, refusal);
945
+ requests += 1;
920
946
  const out = await client.upload(
921
947
  `/v1/results/${encodeURIComponent(job.id)}/files?name=${encodeURIComponent(job.name)}`,
922
948
  bytes,
@@ -939,6 +965,7 @@ async function startRun(passed, declared) {
939
965
  if (fresh === void 0) return failedToBucket(job, "bad_answer");
940
966
  link = fresh;
941
967
  }
968
+ requests += 1;
942
969
  const done2 = await client.post(`/v1/files/${encodeURIComponent(link.id)}/complete`, {}, uploadRetries);
943
970
  if (!done2.ok) return failedToBucket(job, codeOf(done2));
944
971
  countsOf(job.kind).uploaded += 1;
@@ -963,7 +990,7 @@ async function startRun(passed, declared) {
963
990
  else if (job.path === void 0) return void skip(kind, job.name, "missing");
964
991
  else {
965
992
  try {
966
- size = (await import_node_fs4.promises.stat(job.path)).size;
993
+ size = (await import_node_fs5.promises.stat(job.path)).size;
967
994
  } catch {
968
995
  return void skip(kind, job.name, "missing");
969
996
  }
@@ -999,6 +1026,245 @@ async function startRun(passed, declared) {
999
1026
  }
1000
1027
  return () => uploadToBucket(job, size, link);
1001
1028
  }
1029
+ async function uploadEach(jobs) {
1030
+ const byResult = /* @__PURE__ */ new Map();
1031
+ for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
1032
+ await Promise.all(
1033
+ [...byResult.values()].map(async (group) => {
1034
+ const work = [];
1035
+ for (const job of group) {
1036
+ const act = await plan(job);
1037
+ if (act !== void 0) work.push(gated(act));
1038
+ }
1039
+ await Promise.all(work);
1040
+ })
1041
+ );
1042
+ }
1043
+ async function measure(job) {
1044
+ let size;
1045
+ if (job.body !== void 0) size = job.body.length;
1046
+ else if (job.path === void 0) return void skip(job.kind, job.name, "missing");
1047
+ else {
1048
+ try {
1049
+ size = (await import_node_fs5.promises.stat(job.path)).size;
1050
+ } catch {
1051
+ return void skip(job.kind, job.name, "missing");
1052
+ }
1053
+ }
1054
+ if (size === 0) return void skip(job.kind, job.name, "empty");
1055
+ return size;
1056
+ }
1057
+ const routeMissing = (out) => !out.ok && out.status === 404 && out.detail === "" && out.code === void 0;
1058
+ function takeRoom(job) {
1059
+ if (job.kind === "file") {
1060
+ skip(job.kind, job.name, "noStorage");
1061
+ return false;
1062
+ }
1063
+ const roomKey = `${job.id}/${job.kind}`;
1064
+ const left = rooms.get(roomKey) ?? KINDS[job.kind].room;
1065
+ if (left === 0) {
1066
+ skip(job.kind, job.name, KINDS[job.kind].limit);
1067
+ return false;
1068
+ }
1069
+ rooms.set(roomKey, left - 1);
1070
+ return true;
1071
+ }
1072
+ function refusedItem(job, size, item) {
1073
+ const code = item.code ?? "refused";
1074
+ if (code === "duplicate") existed += 1;
1075
+ else if (code === "no_storage") skip(job.kind, job.name, refusedHere(job, size) ?? "noStorage");
1076
+ else if (code === "too_many_files") skip(job.kind, job.name, KINDS[job.kind].limit);
1077
+ else if (BUCKET_CODES.has(code)) failedToBucket(job, code);
1078
+ else failed(job, code);
1079
+ }
1080
+ const itemPause = (round) => new Promise((r) => setTimeout(r, Math.min(250 * 2 ** round, 4e3)));
1081
+ async function batchOf(jobs) {
1082
+ if (batchMissing) return uploadEach(jobs);
1083
+ const sized = [];
1084
+ for (const job of jobs) {
1085
+ const size = await measure(job);
1086
+ if (size === void 0) continue;
1087
+ try {
1088
+ sized.push({ job, size, sha256: job.body !== void 0 ? sha256Bytes(job.body) : await sha256File(job.path) });
1089
+ } catch {
1090
+ skip(job.kind, job.name, "missing");
1091
+ }
1092
+ }
1093
+ if (sized.length === 0) return;
1094
+ const announced = await gated(() => {
1095
+ requests += 1;
1096
+ return client.post(
1097
+ "/v1/files/uploads-batch",
1098
+ { items: sized.map((s) => ({ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 })) },
1099
+ uploadRetries
1100
+ );
1101
+ });
1102
+ if (routeMissing(announced)) {
1103
+ batchMissing = true;
1104
+ return uploadEach(sized.map((s) => s.job));
1105
+ }
1106
+ const verdicts = announced.ok ? announced.body?.items : void 0;
1107
+ if (!announced.ok || !Array.isArray(verdicts) || verdicts.length !== sized.length) {
1108
+ for (const s of sized) {
1109
+ if (announced.ok) failed(s.job, "bad_answer");
1110
+ else if (announced.code !== void 0) failedToBucket(s.job, announced.code);
1111
+ else failed(s.job, announced.detail || announced.kind);
1112
+ }
1113
+ return;
1114
+ }
1115
+ const puts = [];
1116
+ const directs = [];
1117
+ sized.forEach((s, i) => {
1118
+ const v = verdicts[i] ?? {};
1119
+ if (v.status === "exists") existed += 1;
1120
+ else if (v.status === "direct") {
1121
+ if (takeRoom(s.job)) directs.push(s);
1122
+ } else if (v.status === "upload") {
1123
+ const link = linkOf(v);
1124
+ if (link === void 0) return failed(s.job, "bad_answer");
1125
+ if (!bucketSeen) {
1126
+ bucketSeen = true;
1127
+ log(BUCKET_FOUND);
1128
+ }
1129
+ puts.push({ s, link });
1130
+ } else if (v.status === "refused") refusedItem(s.job, s.size, v);
1131
+ else failed(s.job, "bad_answer");
1132
+ });
1133
+ const toComplete = [];
1134
+ await Promise.all(
1135
+ puts.map(
1136
+ ({ s, link: first }) => gated(async () => {
1137
+ const source = s.job.body !== void 0 ? { body: s.job.body } : { path: s.job.path, size: s.size };
1138
+ let link = first;
1139
+ for (let renewed = false; ; ) {
1140
+ const put = await client.put(link.url, link.headers, source);
1141
+ if (put.ok) break;
1142
+ if (put.status !== 403 || renewed) return failedToBucket(s.job, put.code);
1143
+ renewed = true;
1144
+ requests += 1;
1145
+ const again = await client.post(
1146
+ "/v1/files/uploads-batch",
1147
+ { items: [{ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 }] },
1148
+ uploadRetries
1149
+ );
1150
+ const fresh = again.ok ? linkOf(again.body?.items?.[0]) : void 0;
1151
+ if (fresh === void 0) return failedToBucket(s.job, again.ok ? "bad_answer" : codeOf(again));
1152
+ link = fresh;
1153
+ }
1154
+ toComplete.push({ job: s.job, id: link.id });
1155
+ })
1156
+ )
1157
+ );
1158
+ await complete(toComplete);
1159
+ await sendDirect(directs);
1160
+ }
1161
+ async function complete(items) {
1162
+ for (let round = 0; items.length > 0; round += 1) {
1163
+ const out = await gated(() => {
1164
+ requests += 1;
1165
+ return client.post("/v1/files/complete-batch", { ids: items.map((i) => i.id) }, uploadRetries);
1166
+ });
1167
+ if (routeMissing(out)) {
1168
+ await Promise.all(
1169
+ items.map(
1170
+ (i) => gated(async () => {
1171
+ requests += 1;
1172
+ const one = await client.post(`/v1/files/${encodeURIComponent(i.id)}/complete`, {}, uploadRetries);
1173
+ if (one.ok) countsOf(i.job.kind).uploaded += 1;
1174
+ else failedToBucket(i.job, codeOf(one));
1175
+ })
1176
+ )
1177
+ );
1178
+ return;
1179
+ }
1180
+ const verdicts = out.ok ? out.body?.items : void 0;
1181
+ if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== items.length) {
1182
+ for (const i of items) failedToBucket(i.job, out.ok ? "bad_answer" : codeOf(out));
1183
+ return;
1184
+ }
1185
+ const again = [];
1186
+ items.forEach((i, at) => {
1187
+ const v = verdicts[at] ?? {};
1188
+ if (v.status === "ready") countsOf(i.job.kind).uploaded += 1;
1189
+ else if (v.retryable === true && round < Math.min(uploadRetries, 3)) again.push(i);
1190
+ else refusedItem(i.job, 0, v.status === "refused" ? v : { code: "bad_answer" });
1191
+ });
1192
+ items = again;
1193
+ if (items.length > 0) {
1194
+ retried += items.length;
1195
+ await itemPause(round);
1196
+ }
1197
+ }
1198
+ }
1199
+ async function sendDirect(items) {
1200
+ const groups = [];
1201
+ let bytes = 0;
1202
+ for (const item of items) {
1203
+ const group = groups[groups.length - 1];
1204
+ if (group === void 0 || group.length >= FILES_BATCH_MAX || bytes + item.size > DIRECT_BATCH_BYTES) {
1205
+ groups.push([item]);
1206
+ bytes = item.size;
1207
+ } else {
1208
+ group.push(item);
1209
+ bytes += item.size;
1210
+ }
1211
+ }
1212
+ for (const group of groups) {
1213
+ const body = [];
1214
+ const sent = [];
1215
+ for (const { job, size } of group) {
1216
+ const { kind } = job;
1217
+ if (size > KINDS[kind].bytes) {
1218
+ skip(kind, job.name, KINDS[kind].size);
1219
+ continue;
1220
+ }
1221
+ let data;
1222
+ try {
1223
+ data = job.body ?? await import_node_fs5.promises.readFile(job.path);
1224
+ } catch {
1225
+ skip(kind, job.name, "missing");
1226
+ continue;
1227
+ }
1228
+ const refusal = kind === "text" ? textRefusal(data) : void 0;
1229
+ if (refusal !== void 0) {
1230
+ skip(kind, job.name, refusal);
1231
+ continue;
1232
+ }
1233
+ body.push({ resultId: job.id, name: job.name, contentType: job.type, data: data.toString("base64") });
1234
+ sent.push({ job, size });
1235
+ }
1236
+ if (body.length === 0) continue;
1237
+ const out = await gated(() => {
1238
+ requests += 1;
1239
+ return client.post("/v1/files/direct-batch", { items: body }, uploadRetries);
1240
+ });
1241
+ if (routeMissing(out)) {
1242
+ batchMissing = true;
1243
+ for (const { job, size } of sent) await gated(() => uploadDirect(job, size));
1244
+ continue;
1245
+ }
1246
+ const verdicts = out.ok ? out.body?.items : void 0;
1247
+ if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== sent.length) {
1248
+ for (const { job } of sent) failed(job, out.ok ? "bad_answer" : out.detail || out.kind);
1249
+ continue;
1250
+ }
1251
+ sent.forEach(({ job }, i) => {
1252
+ const v = verdicts[i] ?? {};
1253
+ if (v.status === "stored") countsOf(job.kind).uploaded += 1;
1254
+ else if (v.status === "exists") existed += 1;
1255
+ else refusedItem(job, 0, v.status === "refused" ? v : { code: "bad_answer" });
1256
+ });
1257
+ }
1258
+ }
1259
+ async function uploadBatched(jobs) {
1260
+ const batches = chunk(jobs, FILES_BATCH_MAX);
1261
+ let next = 0;
1262
+ await Promise.all(
1263
+ Array.from({ length: Math.min(BATCH_PIPELINES, batches.length) }, async () => {
1264
+ for (let at = next++; at < batches.length; at = next++) await batchOf(batches[at]);
1265
+ })
1266
+ );
1267
+ }
1002
1268
  async function upload(items, filesByKey) {
1003
1269
  if (filesByKey.size === 0 || !Array.isArray(items)) return;
1004
1270
  const jobs = [];
@@ -1024,18 +1290,8 @@ async function startRun(passed, declared) {
1024
1290
  }
1025
1291
  }
1026
1292
  const started = Date.now();
1027
- const byResult = /* @__PURE__ */ new Map();
1028
- for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
1029
- await Promise.all(
1030
- [...byResult.values()].map(async (group) => {
1031
- const work = [];
1032
- for (const job of group) {
1033
- const act = await plan(job);
1034
- if (act !== void 0) work.push(gated(act));
1035
- }
1036
- await Promise.all(work);
1037
- })
1038
- );
1293
+ if (batchMissing) await uploadEach(jobs);
1294
+ else await uploadBatched(jobs);
1039
1295
  uploadMs += Date.now() - started;
1040
1296
  }
1041
1297
  async function flush() {
@@ -1131,9 +1387,9 @@ async function startRun(passed, declared) {
1131
1387
  if (shortened.size > 0) parts.push(`${shortened.size} test name(s) shortened to fit a key`);
1132
1388
  const files = [stats.screenshots, stats.textFiles, stats.otherFiles];
1133
1389
  const total = (what) => files.reduce((n, counts) => n + counts[what], 0);
1134
- if (total("uploaded") + total("skipped") + total("failed") > 0) {
1390
+ if (total("uploaded") + total("skipped") + total("failed") + existed > 0) {
1135
1391
  log(
1136
- `plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once)`
1392
+ `plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${existed} already there (skipped), ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once, ${requests} request(s) to Plune)`
1137
1393
  );
1138
1394
  }
1139
1395
  log(`plune: ${parts.join(" \xB7 ")}`);
@@ -1192,9 +1448,9 @@ async function startRun(passed, declared) {
1192
1448
  }
1193
1449
 
1194
1450
  // src/util/hash.ts
1195
- var import_node_crypto2 = require("crypto");
1451
+ var import_node_crypto3 = require("crypto");
1196
1452
  function sha256(input) {
1197
- return (0, import_node_crypto2.createHash)("sha256").update(input, "utf8").digest("hex");
1453
+ return (0, import_node_crypto3.createHash)("sha256").update(input, "utf8").digest("hex");
1198
1454
  }
1199
1455
 
1200
1456
  // src/reporter-core/result-key.ts
@@ -1,5 +1,5 @@
1
1
  // src/reporter-core/session.ts
2
- import { createHash, randomUUID } from "crypto";
2
+ import { createHash as createHash2, randomUUID } from "crypto";
3
3
  import { promises as disk } from "fs";
4
4
  import { basename as basename2, extname } from "path";
5
5
  import { TextDecoder } from "util";
@@ -470,6 +470,27 @@ function webLink(href) {
470
470
  return href !== void 0 && /^https?:\/\//i.test(href) && URL.canParse(href) ? href : void 0;
471
471
  }
472
472
 
473
+ // src/reporter-core/files-batch.ts
474
+ import { createHash } from "crypto";
475
+ import { createReadStream as createReadStream2 } from "fs";
476
+ var FILES_BATCH_MAX = 100;
477
+ var DIRECT_BATCH_BYTES = 6 * 1024 * 1024;
478
+ var BATCH_PIPELINES = 2;
479
+ async function sha256File(path3) {
480
+ const hash = createHash("sha256");
481
+ for await (const part of createReadStream2(path3)) hash.update(part);
482
+ return hash.digest("hex");
483
+ }
484
+ var sha256Bytes = (bytes) => createHash("sha256").update(bytes).digest("hex");
485
+ var BUCKET_CODES = /* @__PURE__ */ new Set([
486
+ "bucket_unreachable",
487
+ "not_connected",
488
+ "platform_not_configured",
489
+ "upload_expired",
490
+ "file_mismatch",
491
+ "file_missing"
492
+ ]);
493
+
473
494
  // src/reporter-core/fallback.ts
474
495
  import * as fs from "fs";
475
496
  import * as path2 from "path";
@@ -582,7 +603,7 @@ function tally(counts, noun) {
582
603
  }
583
604
  function boundKey(key) {
584
605
  if (key.value.length <= KEY_MAX) return key;
585
- const digest = createHash("sha256").update(key.value).digest("hex").slice(0, 16);
606
+ const digest = createHash2("sha256").update(key.value).digest("hex").slice(0, 16);
586
607
  let head = key.value.slice(0, KEY_MAX - digest.length - 1);
587
608
  if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
588
609
  return { ...key, value: `${head}#${digest}` };
@@ -644,6 +665,9 @@ async function startRun(passed, declared) {
644
665
  const uploadRetries = Math.min(Math.max(Math.floor(cfg.uploadRetries ?? UPLOAD_RETRIES.default), UPLOAD_RETRIES.min), UPLOAD_RETRIES.max);
645
666
  let retried = 0;
646
667
  let uploadMs = 0;
668
+ let requests = 0;
669
+ let existed = 0;
670
+ let batchMissing = false;
647
671
  const client = createClient({
648
672
  apiUrl: resolveApiUrl(cfg.apiUrl),
649
673
  token,
@@ -827,6 +851,7 @@ async function startRun(passed, declared) {
827
851
  else log(`plune: ${KINDS[kind].label} "${name}" not uploaded \u2014 ${SKIPPED[why]}.`);
828
852
  }
829
853
  function ask(job, size) {
854
+ requests += 1;
830
855
  return client.post(`/v1/results/${encodeURIComponent(job.id)}/files/uploads`, {
831
856
  files: [{ name: job.name, contentType: job.type, size }]
832
857
  }, uploadRetries);
@@ -868,6 +893,7 @@ async function startRun(passed, declared) {
868
893
  }
869
894
  const refusal = kind === "text" ? textRefusal(bytes) : void 0;
870
895
  if (refusal !== void 0) return skip(kind, job.name, refusal);
896
+ requests += 1;
871
897
  const out = await client.upload(
872
898
  `/v1/results/${encodeURIComponent(job.id)}/files?name=${encodeURIComponent(job.name)}`,
873
899
  bytes,
@@ -890,6 +916,7 @@ async function startRun(passed, declared) {
890
916
  if (fresh === void 0) return failedToBucket(job, "bad_answer");
891
917
  link = fresh;
892
918
  }
919
+ requests += 1;
893
920
  const done2 = await client.post(`/v1/files/${encodeURIComponent(link.id)}/complete`, {}, uploadRetries);
894
921
  if (!done2.ok) return failedToBucket(job, codeOf(done2));
895
922
  countsOf(job.kind).uploaded += 1;
@@ -950,6 +977,245 @@ async function startRun(passed, declared) {
950
977
  }
951
978
  return () => uploadToBucket(job, size, link);
952
979
  }
980
+ async function uploadEach(jobs) {
981
+ const byResult = /* @__PURE__ */ new Map();
982
+ for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
983
+ await Promise.all(
984
+ [...byResult.values()].map(async (group) => {
985
+ const work = [];
986
+ for (const job of group) {
987
+ const act = await plan(job);
988
+ if (act !== void 0) work.push(gated(act));
989
+ }
990
+ await Promise.all(work);
991
+ })
992
+ );
993
+ }
994
+ async function measure(job) {
995
+ let size;
996
+ if (job.body !== void 0) size = job.body.length;
997
+ else if (job.path === void 0) return void skip(job.kind, job.name, "missing");
998
+ else {
999
+ try {
1000
+ size = (await disk.stat(job.path)).size;
1001
+ } catch {
1002
+ return void skip(job.kind, job.name, "missing");
1003
+ }
1004
+ }
1005
+ if (size === 0) return void skip(job.kind, job.name, "empty");
1006
+ return size;
1007
+ }
1008
+ const routeMissing = (out) => !out.ok && out.status === 404 && out.detail === "" && out.code === void 0;
1009
+ function takeRoom(job) {
1010
+ if (job.kind === "file") {
1011
+ skip(job.kind, job.name, "noStorage");
1012
+ return false;
1013
+ }
1014
+ const roomKey = `${job.id}/${job.kind}`;
1015
+ const left = rooms.get(roomKey) ?? KINDS[job.kind].room;
1016
+ if (left === 0) {
1017
+ skip(job.kind, job.name, KINDS[job.kind].limit);
1018
+ return false;
1019
+ }
1020
+ rooms.set(roomKey, left - 1);
1021
+ return true;
1022
+ }
1023
+ function refusedItem(job, size, item) {
1024
+ const code = item.code ?? "refused";
1025
+ if (code === "duplicate") existed += 1;
1026
+ else if (code === "no_storage") skip(job.kind, job.name, refusedHere(job, size) ?? "noStorage");
1027
+ else if (code === "too_many_files") skip(job.kind, job.name, KINDS[job.kind].limit);
1028
+ else if (BUCKET_CODES.has(code)) failedToBucket(job, code);
1029
+ else failed(job, code);
1030
+ }
1031
+ const itemPause = (round) => new Promise((r) => setTimeout(r, Math.min(250 * 2 ** round, 4e3)));
1032
+ async function batchOf(jobs) {
1033
+ if (batchMissing) return uploadEach(jobs);
1034
+ const sized = [];
1035
+ for (const job of jobs) {
1036
+ const size = await measure(job);
1037
+ if (size === void 0) continue;
1038
+ try {
1039
+ sized.push({ job, size, sha256: job.body !== void 0 ? sha256Bytes(job.body) : await sha256File(job.path) });
1040
+ } catch {
1041
+ skip(job.kind, job.name, "missing");
1042
+ }
1043
+ }
1044
+ if (sized.length === 0) return;
1045
+ const announced = await gated(() => {
1046
+ requests += 1;
1047
+ return client.post(
1048
+ "/v1/files/uploads-batch",
1049
+ { items: sized.map((s) => ({ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 })) },
1050
+ uploadRetries
1051
+ );
1052
+ });
1053
+ if (routeMissing(announced)) {
1054
+ batchMissing = true;
1055
+ return uploadEach(sized.map((s) => s.job));
1056
+ }
1057
+ const verdicts = announced.ok ? announced.body?.items : void 0;
1058
+ if (!announced.ok || !Array.isArray(verdicts) || verdicts.length !== sized.length) {
1059
+ for (const s of sized) {
1060
+ if (announced.ok) failed(s.job, "bad_answer");
1061
+ else if (announced.code !== void 0) failedToBucket(s.job, announced.code);
1062
+ else failed(s.job, announced.detail || announced.kind);
1063
+ }
1064
+ return;
1065
+ }
1066
+ const puts = [];
1067
+ const directs = [];
1068
+ sized.forEach((s, i) => {
1069
+ const v = verdicts[i] ?? {};
1070
+ if (v.status === "exists") existed += 1;
1071
+ else if (v.status === "direct") {
1072
+ if (takeRoom(s.job)) directs.push(s);
1073
+ } else if (v.status === "upload") {
1074
+ const link = linkOf(v);
1075
+ if (link === void 0) return failed(s.job, "bad_answer");
1076
+ if (!bucketSeen) {
1077
+ bucketSeen = true;
1078
+ log(BUCKET_FOUND);
1079
+ }
1080
+ puts.push({ s, link });
1081
+ } else if (v.status === "refused") refusedItem(s.job, s.size, v);
1082
+ else failed(s.job, "bad_answer");
1083
+ });
1084
+ const toComplete = [];
1085
+ await Promise.all(
1086
+ puts.map(
1087
+ ({ s, link: first }) => gated(async () => {
1088
+ const source = s.job.body !== void 0 ? { body: s.job.body } : { path: s.job.path, size: s.size };
1089
+ let link = first;
1090
+ for (let renewed = false; ; ) {
1091
+ const put = await client.put(link.url, link.headers, source);
1092
+ if (put.ok) break;
1093
+ if (put.status !== 403 || renewed) return failedToBucket(s.job, put.code);
1094
+ renewed = true;
1095
+ requests += 1;
1096
+ const again = await client.post(
1097
+ "/v1/files/uploads-batch",
1098
+ { items: [{ resultId: s.job.id, name: s.job.name, contentType: s.job.type, size: s.size, sha256: s.sha256 }] },
1099
+ uploadRetries
1100
+ );
1101
+ const fresh = again.ok ? linkOf(again.body?.items?.[0]) : void 0;
1102
+ if (fresh === void 0) return failedToBucket(s.job, again.ok ? "bad_answer" : codeOf(again));
1103
+ link = fresh;
1104
+ }
1105
+ toComplete.push({ job: s.job, id: link.id });
1106
+ })
1107
+ )
1108
+ );
1109
+ await complete(toComplete);
1110
+ await sendDirect(directs);
1111
+ }
1112
+ async function complete(items) {
1113
+ for (let round = 0; items.length > 0; round += 1) {
1114
+ const out = await gated(() => {
1115
+ requests += 1;
1116
+ return client.post("/v1/files/complete-batch", { ids: items.map((i) => i.id) }, uploadRetries);
1117
+ });
1118
+ if (routeMissing(out)) {
1119
+ await Promise.all(
1120
+ items.map(
1121
+ (i) => gated(async () => {
1122
+ requests += 1;
1123
+ const one = await client.post(`/v1/files/${encodeURIComponent(i.id)}/complete`, {}, uploadRetries);
1124
+ if (one.ok) countsOf(i.job.kind).uploaded += 1;
1125
+ else failedToBucket(i.job, codeOf(one));
1126
+ })
1127
+ )
1128
+ );
1129
+ return;
1130
+ }
1131
+ const verdicts = out.ok ? out.body?.items : void 0;
1132
+ if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== items.length) {
1133
+ for (const i of items) failedToBucket(i.job, out.ok ? "bad_answer" : codeOf(out));
1134
+ return;
1135
+ }
1136
+ const again = [];
1137
+ items.forEach((i, at) => {
1138
+ const v = verdicts[at] ?? {};
1139
+ if (v.status === "ready") countsOf(i.job.kind).uploaded += 1;
1140
+ else if (v.retryable === true && round < Math.min(uploadRetries, 3)) again.push(i);
1141
+ else refusedItem(i.job, 0, v.status === "refused" ? v : { code: "bad_answer" });
1142
+ });
1143
+ items = again;
1144
+ if (items.length > 0) {
1145
+ retried += items.length;
1146
+ await itemPause(round);
1147
+ }
1148
+ }
1149
+ }
1150
+ async function sendDirect(items) {
1151
+ const groups = [];
1152
+ let bytes = 0;
1153
+ for (const item of items) {
1154
+ const group = groups[groups.length - 1];
1155
+ if (group === void 0 || group.length >= FILES_BATCH_MAX || bytes + item.size > DIRECT_BATCH_BYTES) {
1156
+ groups.push([item]);
1157
+ bytes = item.size;
1158
+ } else {
1159
+ group.push(item);
1160
+ bytes += item.size;
1161
+ }
1162
+ }
1163
+ for (const group of groups) {
1164
+ const body = [];
1165
+ const sent = [];
1166
+ for (const { job, size } of group) {
1167
+ const { kind } = job;
1168
+ if (size > KINDS[kind].bytes) {
1169
+ skip(kind, job.name, KINDS[kind].size);
1170
+ continue;
1171
+ }
1172
+ let data;
1173
+ try {
1174
+ data = job.body ?? await disk.readFile(job.path);
1175
+ } catch {
1176
+ skip(kind, job.name, "missing");
1177
+ continue;
1178
+ }
1179
+ const refusal = kind === "text" ? textRefusal(data) : void 0;
1180
+ if (refusal !== void 0) {
1181
+ skip(kind, job.name, refusal);
1182
+ continue;
1183
+ }
1184
+ body.push({ resultId: job.id, name: job.name, contentType: job.type, data: data.toString("base64") });
1185
+ sent.push({ job, size });
1186
+ }
1187
+ if (body.length === 0) continue;
1188
+ const out = await gated(() => {
1189
+ requests += 1;
1190
+ return client.post("/v1/files/direct-batch", { items: body }, uploadRetries);
1191
+ });
1192
+ if (routeMissing(out)) {
1193
+ batchMissing = true;
1194
+ for (const { job, size } of sent) await gated(() => uploadDirect(job, size));
1195
+ continue;
1196
+ }
1197
+ const verdicts = out.ok ? out.body?.items : void 0;
1198
+ if (!out.ok || !Array.isArray(verdicts) || verdicts.length !== sent.length) {
1199
+ for (const { job } of sent) failed(job, out.ok ? "bad_answer" : out.detail || out.kind);
1200
+ continue;
1201
+ }
1202
+ sent.forEach(({ job }, i) => {
1203
+ const v = verdicts[i] ?? {};
1204
+ if (v.status === "stored") countsOf(job.kind).uploaded += 1;
1205
+ else if (v.status === "exists") existed += 1;
1206
+ else refusedItem(job, 0, v.status === "refused" ? v : { code: "bad_answer" });
1207
+ });
1208
+ }
1209
+ }
1210
+ async function uploadBatched(jobs) {
1211
+ const batches = chunk(jobs, FILES_BATCH_MAX);
1212
+ let next = 0;
1213
+ await Promise.all(
1214
+ Array.from({ length: Math.min(BATCH_PIPELINES, batches.length) }, async () => {
1215
+ for (let at = next++; at < batches.length; at = next++) await batchOf(batches[at]);
1216
+ })
1217
+ );
1218
+ }
953
1219
  async function upload(items, filesByKey) {
954
1220
  if (filesByKey.size === 0 || !Array.isArray(items)) return;
955
1221
  const jobs = [];
@@ -975,18 +1241,8 @@ async function startRun(passed, declared) {
975
1241
  }
976
1242
  }
977
1243
  const started = Date.now();
978
- const byResult = /* @__PURE__ */ new Map();
979
- for (const job of jobs) byResult.set(job.id, [...byResult.get(job.id) ?? [], job]);
980
- await Promise.all(
981
- [...byResult.values()].map(async (group) => {
982
- const work = [];
983
- for (const job of group) {
984
- const act = await plan(job);
985
- if (act !== void 0) work.push(gated(act));
986
- }
987
- await Promise.all(work);
988
- })
989
- );
1244
+ if (batchMissing) await uploadEach(jobs);
1245
+ else await uploadBatched(jobs);
990
1246
  uploadMs += Date.now() - started;
991
1247
  }
992
1248
  async function flush() {
@@ -1082,9 +1338,9 @@ async function startRun(passed, declared) {
1082
1338
  if (shortened.size > 0) parts.push(`${shortened.size} test name(s) shortened to fit a key`);
1083
1339
  const files = [stats.screenshots, stats.textFiles, stats.otherFiles];
1084
1340
  const total = (what) => files.reduce((n, counts) => n + counts[what], 0);
1085
- if (total("uploaded") + total("skipped") + total("failed") > 0) {
1341
+ if (total("uploaded") + total("skipped") + total("failed") + existed > 0) {
1086
1342
  log(
1087
- `plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once)`
1343
+ `plune: files \u2014 ${total("uploaded")} uploaded, ${total("skipped")} skipped, ${existed} already there (skipped), ${retried} retried, ${total("failed")} dropped, in ${(uploadMs / 1e3).toFixed(1)} s (${uploadsAtOnce} at once, ${requests} request(s) to Plune)`
1088
1344
  );
1089
1345
  }
1090
1346
  log(`plune: ${parts.join(" \xB7 ")}`);
@@ -1143,9 +1399,9 @@ async function startRun(passed, declared) {
1143
1399
  }
1144
1400
 
1145
1401
  // src/util/hash.ts
1146
- import { createHash as createHash2 } from "crypto";
1402
+ import { createHash as createHash3 } from "crypto";
1147
1403
  function sha256(input) {
1148
- return createHash2("sha256").update(input, "utf8").digest("hex");
1404
+ return createHash3("sha256").update(input, "utf8").digest("hex");
1149
1405
  }
1150
1406
 
1151
1407
  // src/reporter-core/result-key.ts
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@plune-ai/cli",
3
- "version": "0.22.0",
3
+ "version": "0.23.0",
4
4
  "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
5
5
  "keywords": [
6
6
  "ai",