@xeonr/pipelines-sdk 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +172 -16
  2. package/dist/client.d.ts +63 -4
  3. package/dist/client.d.ts.map +1 -1
  4. package/dist/client.js +43 -1
  5. package/dist/client.js.map +1 -1
  6. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +173 -12
  7. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
  8. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +45 -28
  9. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
  10. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +136 -17
  11. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -1
  12. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +42 -35
  13. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -1
  14. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +187 -2
  15. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
  16. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +38 -5
  17. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
  18. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +4 -2
  19. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
  20. package/dist/gen/xeonr/pipelines/v1/common_pb.js +4 -2
  21. package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
  22. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +277 -4
  23. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
  24. package/dist/gen/xeonr/pipelines/v1/definition_pb.js +100 -7
  25. package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
  26. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +1100 -0
  27. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -0
  28. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +298 -0
  29. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -0
  30. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
  31. package/dist/gen/xeonr/pipelines/v1/pools_pb.js +1 -1
  32. package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
  33. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +769 -0
  34. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -0
  35. package/dist/gen/xeonr/pipelines/v1/queues_pb.js +190 -0
  36. package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -0
  37. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +60 -0
  38. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
  39. package/dist/gen/xeonr/pipelines/v1/registry_pb.js +21 -16
  40. package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
  41. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +846 -3
  42. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
  43. package/dist/gen/xeonr/pipelines/v1/runs_pb.js +243 -27
  44. package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
  45. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +1110 -0
  46. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -0
  47. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +295 -0
  48. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -0
  49. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
  50. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +1 -1
  51. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
  52. package/dist/gen/xeonr/platform/v1/options_pb.d.ts +123 -1
  53. package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
  54. package/dist/gen/xeonr/platform/v1/options_pb.js +90 -1
  55. package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
  56. package/dist/handler.d.ts +50 -0
  57. package/dist/handler.d.ts.map +1 -1
  58. package/dist/handler.js +27 -0
  59. package/dist/handler.js.map +1 -1
  60. package/dist/index.d.ts +11 -0
  61. package/dist/index.d.ts.map +1 -1
  62. package/dist/index.js +5 -0
  63. package/dist/index.js.map +1 -1
  64. package/dist/queues.d.ts +171 -0
  65. package/dist/queues.d.ts.map +1 -0
  66. package/dist/queues.js +135 -0
  67. package/dist/queues.js.map +1 -0
  68. package/dist/reporter.d.ts +120 -0
  69. package/dist/reporter.d.ts.map +1 -0
  70. package/dist/reporter.js +323 -0
  71. package/dist/reporter.js.map +1 -0
  72. package/dist/schedules.d.ts +201 -0
  73. package/dist/schedules.d.ts.map +1 -0
  74. package/dist/schedules.js +127 -0
  75. package/dist/schedules.js.map +1 -0
  76. package/dist/testing/cron.d.ts +33 -0
  77. package/dist/testing/cron.d.ts.map +1 -0
  78. package/dist/testing/cron.js +177 -0
  79. package/dist/testing/cron.js.map +1 -0
  80. package/dist/testing/index.d.ts +129 -4
  81. package/dist/testing/index.d.ts.map +1 -1
  82. package/dist/testing/index.js +723 -30
  83. package/dist/testing/index.js.map +1 -1
  84. package/dist/testing/queues.d.ts +46 -0
  85. package/dist/testing/queues.d.ts.map +1 -0
  86. package/dist/testing/queues.js +70 -0
  87. package/dist/testing/queues.js.map +1 -0
  88. package/dist/testing/schedules.d.ts +113 -0
  89. package/dist/testing/schedules.d.ts.map +1 -0
  90. package/dist/testing/schedules.js +419 -0
  91. package/dist/testing/schedules.js.map +1 -0
  92. package/dist/worker.d.ts +48 -1
  93. package/dist/worker.d.ts.map +1 -1
  94. package/dist/worker.js +67 -6
  95. package/dist/worker.js.map +1 -1
  96. package/package.json +1 -1
@@ -34,7 +34,7 @@
34
34
  * What it does not: secrets, blobs, tiers, routing between pools, the
35
35
  * outbox, and the platform's own deadline enforcement.
36
36
  */
37
- import { randomUUID } from "node:crypto";
37
+ import { createHash, randomUUID } from "node:crypto";
38
38
  import { create, fromJson, toJson } from "@bufbuild/protobuf";
39
39
  import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
40
40
  import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
@@ -43,7 +43,14 @@ import { authInterceptor } from "../transport.js";
43
43
  import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
44
44
  import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
45
45
  import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
46
- import { AttemptSchema, NodeRunSchema, RunSchema, RunsService } from "../gen/xeonr/pipelines/v1/runs_pb.js";
46
+ import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
47
+ import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
48
+ import { pickQueued } from "./queues.js";
49
+ import { FakeClock, FakeSchedules } from "./schedules.js";
50
+ import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
51
+ export { FakeClock, FakeSchedules } from "./schedules.js";
52
+ export { parseCron, nextOccurrence } from "./cron.js";
53
+ export { pickQueued } from "./queues.js";
47
54
  import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
48
55
  /** The tenant the default account acts for. */
49
56
  export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
@@ -58,7 +65,13 @@ export const ALL_SCOPES = [
58
65
  "pipelines:pool:read",
59
66
  "pipelines:pool:edit",
60
67
  "pipelines:worker:edit",
68
+ "pipelines:queue:read",
69
+ "pipelines:queue:edit",
70
+ "pipelines:schedule:read",
71
+ "pipelines:schedule:edit",
61
72
  ];
73
+ /** The product the default account belongs to. */
74
+ export const FAKE_PRODUCT = "00000000-0000-4000-8000-0000000000a1";
62
75
  const TRANSIENT = [
63
76
  FailureClass.ABANDONED,
64
77
  FailureClass.RUNNER_CRASHED,
@@ -68,8 +81,8 @@ const TRANSIENT = [
68
81
  FailureClass.INTERNAL,
69
82
  ];
70
83
  const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
71
- /** How many interruptions an interruptible node gets without spending a retry. */
72
- const FREE_INTERRUPTIONS = 10;
84
+ /** How many interruptions an interruptible node gets without spending a retry (the api's default). */
85
+ const FREE_INTERRUPTIONS = 3;
73
86
  const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
74
87
  /**
75
88
  * An in-memory pipelines-api for tests. Call {@link close} when done (it
@@ -88,6 +101,16 @@ export class FakePipelines {
88
101
  attemptsById = new Map();
89
102
  idempotency = new Map();
90
103
  injected = new Map();
104
+ /** Queues by id. */
105
+ queues = new Map();
106
+ /**
107
+ * The clock schedules run on: real time until a test calls
108
+ * `clock.set()` or `clock.advance()`, each of which fires what is due.
109
+ */
110
+ clock = new FakeClock();
111
+ /** The schedules, with their recorded occurrences. */
112
+ schedules;
113
+ queueClaims = new Map();
91
114
  timer;
92
115
  changed;
93
116
  constructor(opts = {}) {
@@ -102,8 +125,29 @@ export class FakePipelines {
102
125
  retryBackoffMs: opts.retryBackoffMs ?? 0,
103
126
  logMaxBytes: opts.logMaxBytes ?? 0,
104
127
  tickMs: opts.tickMs ?? 10,
128
+ issueCap: opts.issueCap ?? 1000,
129
+ spanCap: opts.spanCap ?? 10_000,
105
130
  };
106
131
  this.changed = signal();
132
+ this.schedules = new FakeSchedules({
133
+ createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES, product: owner.product }, {
134
+ tenant: owner.tenant,
135
+ handler: spec.handler,
136
+ key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
137
+ payload: spec.payload,
138
+ delayMs: 0,
139
+ resource: spec.resource,
140
+ retry: spec.retry,
141
+ timeoutMs: spec.timeoutMs,
142
+ queue: spec.queue,
143
+ interruptible: spec.interruptible,
144
+ trigger: spec.trigger,
145
+ schedule: spec.schedule,
146
+ }),
147
+ runs: () => this.runs.values(),
148
+ runMessage: (id) => this.runMessage(this.run(id)),
149
+ checkHandler: (_product, handler) => this.requireHandler(handler),
150
+ }, this.clock, FAKE_PRODUCT);
107
151
  this.timer = setInterval(() => this.tick(), this.opts.tickMs);
108
152
  this.timer.unref();
109
153
  }
@@ -371,37 +415,107 @@ export class FakePipelines {
371
415
  this.notify();
372
416
  return { state: a.state };
373
417
  },
418
+ appendIssues: (req, ctx) => {
419
+ const who = this.enter("AppendIssues", ctx, "pipelines:worker:edit");
420
+ const a = this.held(who, req.ref);
421
+ if (a.state !== AttemptState.RUNNING)
422
+ throw stale();
423
+ if (req.issues.length < 1 || req.issues.length > 100)
424
+ throw new ConnectError("issues: 1 to 100", Code.InvalidArgument);
425
+ return { accepted: this.appendIssues(a, req.issues) };
426
+ },
427
+ setProgress: (req, ctx) => {
428
+ const who = this.enter("SetProgress", ctx, "pipelines:worker:edit");
429
+ const a = this.held(who, req.ref);
430
+ if (a.state !== AttemptState.RUNNING)
431
+ throw stale();
432
+ if (!(req.fraction >= 0 && req.fraction <= 1))
433
+ throw new ConnectError("fraction: 0 to 1", Code.InvalidArgument);
434
+ if (req.stageState === StageState.PENDING)
435
+ throw new ConnectError("stage_state: a stage never moves back to PENDING", Code.InvalidArgument);
436
+ if (req.stage && !/^[a-z][a-z0-9_.-]{0,63}$/.test(req.stage))
437
+ throw new ConnectError("stage: invalid key", Code.InvalidArgument);
438
+ if (req.message.length > 255)
439
+ throw new ConnectError("message: at most 255", Code.InvalidArgument);
440
+ this.setProgress(a, req);
441
+ return {};
442
+ },
443
+ reportSpans: (req, ctx) => {
444
+ const who = this.enter("ReportSpans", ctx, "pipelines:worker:edit");
445
+ const a = this.held(who, req.ref);
446
+ if (a.state !== AttemptState.RUNNING)
447
+ throw stale();
448
+ if (req.spans.length < 1 || req.spans.length > 1000)
449
+ throw new ConnectError("spans: 1 to 1000", Code.InvalidArgument);
450
+ for (const sp of req.spans) {
451
+ if (!/^[0-9a-f]{32}$/.test(sp.traceId) || !/^[0-9a-f]{16}$/.test(sp.spanId) || !sp.startedAt || !sp.endedAt || !sp.name) {
452
+ throw new ConnectError("spans: trace_id, span_id, name, started_at and ended_at are required", Code.InvalidArgument);
453
+ }
454
+ }
455
+ const room = Math.max(0, this.opts.spanCap - a.spans.length);
456
+ a.spans.push(...req.spans.slice(0, room));
457
+ return { accepted: Math.min(room, req.spans.length) };
458
+ },
374
459
  });
460
+ router.service(QueuesService, this.queuesService());
461
+ router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
375
462
  router.service(RunsService, {
463
+ listIssues: (req, ctx) => {
464
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
465
+ this.visible(who, req.runId);
466
+ const issues = this.issuesOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId, minSeverity: req.minSeverity });
467
+ return {
468
+ issues: issues.map((i) => create(RunIssueSchema, { ...i, detail: i.detail, createdAt: timestampFromMs(i.createdAt) })),
469
+ nextCursor: "",
470
+ };
471
+ },
472
+ getLogs: (req, ctx) => {
473
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
474
+ const a = this.attemptsById.get(req.attemptId);
475
+ if (!a)
476
+ throw new ConnectError("no such attempt", Code.NotFound);
477
+ this.visible(who, a.runId);
478
+ const all = Buffer.concat(a.chunks);
479
+ const from = Math.min(Number(req.offset), all.length);
480
+ const to = req.limitBytes ? Math.min(all.length, from + req.limitBytes) : all.length;
481
+ const complete = to === all.length && (a.logFinal || a.completedAt !== undefined);
482
+ return { data: new Uint8Array(all.subarray(from, to)), nextOffset: BigInt(to), complete, truncated: a.logTruncated };
483
+ },
484
+ getRunTrace: (req, ctx) => {
485
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
486
+ this.visible(who, req.runId);
487
+ return { spans: this.traceOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId }), truncated: false };
488
+ },
376
489
  createJob: (req, ctx) => {
377
490
  const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
378
491
  this.actsFor(who, req.tenant);
379
- const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
492
+ const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
380
493
  if (existing)
381
494
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
382
- this.requireHandler(req.handler);
383
- const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, {}, req.delay ? durationMs(req.delay) : 0);
384
- if (req.retry && req.retry.max > 10)
385
- throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
386
- if (req.retry?.when.includes(FailureClass.CANCELLED))
387
- throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
388
- this.addNode(run, {
389
- id: "job",
495
+ const run = this.createJobRun(who, {
496
+ tenant: req.tenant,
390
497
  handler: req.handler,
391
- uses: "",
392
- with: req.payload ?? {},
393
- needs: [],
394
- retry: req.retry ? { max: req.retry.max, when: req.retry.when } : undefined,
498
+ key: req.idempotencyKey,
499
+ payload: req.payload ?? {},
500
+ delayMs: req.delay ? durationMs(req.delay) : 0,
501
+ resource: req.resource,
502
+ retry: req.retry,
395
503
  timeoutMs: req.timeout ? durationMs(req.timeout) : 0,
396
- interruptible: false,
504
+ queue: req.queue,
505
+ interruptible: req.interruptible,
397
506
  });
398
- this.advance(run);
399
507
  return { run: this.runMessage(run), deduplicated: false };
400
508
  },
509
+ retryNode: (req, ctx) => {
510
+ const who = this.enter("RetryNode", ctx, "pipelines:run:edit");
511
+ const run = this.visible(who, req.runId);
512
+ this.retryNode(run, req.nodeId);
513
+ return { run: this.runMessage(run) };
514
+ },
401
515
  createRun: (req, ctx) => {
402
516
  const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
403
517
  this.actsFor(who, req.tenant);
404
- const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
518
+ const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
405
519
  if (existing)
406
520
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
407
521
  if (req.source.case !== "inlineDefinition")
@@ -410,6 +524,9 @@ export class FakePipelines {
410
524
  if (def.nodes.length === 0)
411
525
  throw new ConnectError("definition: no nodes", Code.InvalidArgument);
412
526
  const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
527
+ for (const sp of specs)
528
+ if (sp.handler)
529
+ sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
413
530
  checkGraph(specs);
414
531
  const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, req.variables ?? {}, 0);
415
532
  for (const spec of specs)
@@ -537,13 +654,39 @@ export class FakePipelines {
537
654
  throw new ConnectError(`handler ${handler} is not a published worker package`, Code.InvalidArgument);
538
655
  }
539
656
  }
657
+ /**
658
+ * Holds a handler node's `with` to its manifest, as the api does when a run
659
+ * is created: every key must be a declared input, every required input
660
+ * given; defaults fill the rest. Returns the inputs with defaults.
661
+ */
662
+ checkInputs(nodeId, handler, given) {
663
+ const pkg = this.packages.get(handler);
664
+ const version = pkg ? [...pkg.versions.keys()].at(-1) : "";
665
+ const manifest = pkg?.versions.get(version);
666
+ const inputs = manifest?.inputs ?? [];
667
+ for (const k of Object.keys(given)) {
668
+ if (!inputs.some((i) => i.name === k)) {
669
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": input "${k}" is not an input of ${handler}@${version}`, Code.InvalidArgument);
670
+ }
671
+ }
672
+ const out = { ...given };
673
+ for (const i of inputs) {
674
+ if (out[i.name] !== undefined)
675
+ continue;
676
+ if (i.default)
677
+ out[i.name] = toJson(ValueSchema, i.default);
678
+ else if (i.required)
679
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": required input "${i.name}" is not given`, Code.InvalidArgument);
680
+ }
681
+ return out;
682
+ }
540
683
  // ---- the scheduler ----------------------------------------------------
541
684
  newRun(who, tenant, key, resource, variables, delayMs) {
542
685
  const now = Date.now();
543
686
  const run = {
544
687
  id: randomUUID(),
545
688
  tenant,
546
- product: who.product ?? who.tenant,
689
+ product: productOf(who),
547
690
  idempotencyKey: key,
548
691
  createdBy: `urn:fake:sa:${who.tenant}`,
549
692
  resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
@@ -558,6 +701,14 @@ export class FakePipelines {
558
701
  cancelRequestedAt: undefined,
559
702
  cancelReason: "",
560
703
  nodes: new Map(),
704
+ queueId: "",
705
+ queue: "",
706
+ trigger: RunTrigger.API,
707
+ scheduleId: "",
708
+ scheduledFor: undefined,
709
+ scheduleTarget: "",
710
+ issues: [],
711
+ issuesTruncated: false,
561
712
  };
562
713
  this.runs.set(run.id, run);
563
714
  this.idempotency.set(`${run.product}\u0000${key}`, run.id);
@@ -633,6 +784,7 @@ export class FakePipelines {
633
784
  else {
634
785
  run.state = RunState.SUCCEEDED;
635
786
  }
787
+ this.runFinished(run);
636
788
  }
637
789
  }
638
790
  resolve(run, n) {
@@ -683,14 +835,23 @@ export class FakePipelines {
683
835
  completedAt: undefined,
684
836
  handlerFailureKind: HandlerFailureKind.UNSPECIFIED,
685
837
  releases: 0,
838
+ offeredAt: Date.now(),
839
+ progress: undefined,
840
+ spans: [],
686
841
  };
687
842
  n.attempts.push(a);
688
843
  this.attemptsById.set(a.id, a);
689
844
  this.notify();
690
845
  }
846
+ /**
847
+ * Claims for a worker: the attempts of handler nodes of full runs first,
848
+ * in creation order, as they always were; then jobs, through their
849
+ * queues' limits and fairness rule (see ./queues.ts).
850
+ */
691
851
  claim(pool, handlers, workerId, max) {
692
852
  const now = Date.now();
693
853
  const out = [];
854
+ const queued = [];
694
855
  for (const run of this.runs.values()) {
695
856
  if (run.tenant !== pool.owner?.tenant || run.cancelRequestedAt !== undefined || run.notBefore > now)
696
857
  continue;
@@ -698,21 +859,48 @@ export class FakePipelines {
698
859
  if (!handlers.includes(n.handler))
699
860
  continue;
700
861
  for (const a of n.attempts) {
701
- if (out.length >= max)
702
- return out;
703
862
  if (a.state !== AttemptState.OFFERED || a.notBefore > now)
704
863
  continue;
705
- a.state = AttemptState.CLAIMED;
706
- a.epoch += 1n;
707
- a.workerId = workerId;
708
- a.poolId = pool.id;
709
- a.leaseExpiresAt = now + this.opts.leaseMs;
710
- out.push(a);
864
+ if (run.queueId)
865
+ queued.push({ item: a, queueId: run.queueId, tenant: run.tenant, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
866
+ else if (out.length < max)
867
+ out.push(a);
711
868
  }
712
869
  }
713
870
  }
871
+ if (out.length < max && queued.length > 0) {
872
+ const limits = new Map();
873
+ for (const q of this.queues.values())
874
+ limits.set(q.id, queueLimits(q));
875
+ const picks = pickQueued(queued, limits, { running: (q, t) => this.queueRunning(q, t), claimedSince: (q, since) => (this.queueClaims.get(q) ?? []).filter((x) => x >= since).length }, now, max - out.length);
876
+ for (const p of picks) {
877
+ out.push(p.item);
878
+ this.queueClaims.set(p.queueId, [...(this.queueClaims.get(p.queueId) ?? []).filter((x) => x > now - 86_400_000), now]);
879
+ }
880
+ }
881
+ for (const a of out) {
882
+ a.state = AttemptState.CLAIMED;
883
+ a.epoch += 1n;
884
+ a.workerId = workerId;
885
+ a.poolId = pool.id;
886
+ a.leaseExpiresAt = now + this.opts.leaseMs;
887
+ this.initProgress(a);
888
+ }
714
889
  return out;
715
890
  }
891
+ /** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
892
+ queueRunning(queueId, tenant) {
893
+ let n = 0;
894
+ for (const run of this.runs.values()) {
895
+ if (run.queueId !== queueId || (tenant !== undefined && run.tenant !== tenant))
896
+ continue;
897
+ for (const node of run.nodes.values())
898
+ for (const a of node.attempts)
899
+ if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING)
900
+ n++;
901
+ }
902
+ return n;
903
+ }
716
904
  describe(a) {
717
905
  const run = this.run(a.runId);
718
906
  const n = run.nodes.get(a.nodeId);
@@ -770,11 +958,16 @@ export class FakePipelines {
770
958
  const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
771
959
  const freeUsed = n.attempts.filter(free).length;
772
960
  if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
961
+ // Up to the bound, an interruption is offered again at once.
773
962
  this.offer(run, n, a.inputs, 0);
774
963
  return;
775
964
  }
776
965
  const retries = n.attempts.slice(0, -1).filter((x) => !free(x)).length;
777
- if (!permanent && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
966
+ // As the api: a worker that vanished (ABANDONED) may have had outside
967
+ // effects, so its node runs again automatically only when it declares
968
+ // itself interruptible.
969
+ const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
970
+ if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
778
971
  this.offer(run, n, a.inputs, this.opts.retryBackoffMs);
779
972
  return;
780
973
  }
@@ -836,6 +1029,7 @@ export class FakePipelines {
836
1029
  }
837
1030
  }
838
1031
  }
1032
+ this.schedules.evaluate();
839
1033
  this.notify();
840
1034
  }
841
1035
  notify() {
@@ -843,6 +1037,430 @@ export class FakePipelines {
843
1037
  this.changed = signal();
844
1038
  prev.resolve();
845
1039
  }
1040
+ /** QueuesService, as the contract states it. */
1041
+ queuesService() {
1042
+ const readable = (who, q) => {
1043
+ if (!q || q.owner?.product !== productOf(who) || (q.owner.tenant && q.owner.tenant !== who.tenant))
1044
+ throw new ConnectError("no such queue", Code.NotFound);
1045
+ return q;
1046
+ };
1047
+ const byId = (who, id) => readable(who, this.queues.get(id));
1048
+ return {
1049
+ setQueue: (req, ctx) => {
1050
+ const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
1051
+ const q = req.queue;
1052
+ if (!q?.owner)
1053
+ throw new ConnectError("queue.owner: required", Code.InvalidArgument);
1054
+ if (q.owner.product !== productOf(who))
1055
+ throw new ConnectError("queue.owner.product: a product sets only its own queues", Code.PermissionDenied);
1056
+ if (q.owner.tenant && q.owner.tenant !== who.tenant)
1057
+ throw new ConnectError("queue.owner.tenant: the caller does not act for it", Code.PermissionDenied);
1058
+ if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
1059
+ throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
1060
+ if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
1061
+ throw new ConnectError("max_concurrency_per_tenant must not exceed a non-zero max_concurrency", Code.InvalidArgument);
1062
+ }
1063
+ if (req.updateMask)
1064
+ throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
1065
+ const existing = [...this.queues.values()].find((x) => x.owner?.product === q.owner.product && x.owner.tenant === q.owner.tenant && x.name === q.name);
1066
+ const now = Date.now();
1067
+ const next = create(QueueSchema, {
1068
+ ...q,
1069
+ id: existing?.id ?? randomUUID(),
1070
+ paused: existing?.paused ?? false,
1071
+ pausedReason: existing?.pausedReason ?? "",
1072
+ pausedBy: existing?.pausedBy ?? "",
1073
+ ...(existing?.pausedAt ? { pausedAt: existing.pausedAt } : {}),
1074
+ createdAt: existing?.createdAt ?? timestampFromMs(now),
1075
+ updatedAt: timestampFromMs(now),
1076
+ });
1077
+ this.queues.set(next.id, next);
1078
+ this.notify();
1079
+ return { queue: next, created: !existing };
1080
+ },
1081
+ getQueue: (req, ctx) => {
1082
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1083
+ if (req.queue.case === "queueId")
1084
+ return { queue: byId(who, req.queue.value) };
1085
+ if (req.queue.case !== "byName")
1086
+ throw new ConnectError("queue: required", Code.InvalidArgument);
1087
+ const { owner, name } = req.queue.value;
1088
+ if (owner?.product !== productOf(who))
1089
+ throw new ConnectError("no such queue", Code.NotFound);
1090
+ const found = [...this.queues.values()].find((x) => x.owner?.product === owner.product && x.owner.tenant === owner.tenant && x.name === name);
1091
+ if (found)
1092
+ return { queue: readable(who, found) };
1093
+ if (name === "default" && !owner.tenant)
1094
+ return { queue: this.defaultQueue(owner.product) };
1095
+ throw new ConnectError("no such queue", Code.NotFound);
1096
+ },
1097
+ listQueues: (req, ctx) => {
1098
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1099
+ const product = req.product || productOf(who);
1100
+ if (product !== productOf(who))
1101
+ return { queues: [], nextCursor: "" };
1102
+ const queues = [...this.queues.values()]
1103
+ .filter((q) => q.owner?.product === product && (!req.tenant || q.owner.tenant === req.tenant) && (!q.owner.tenant || q.owner.tenant === who.tenant))
1104
+ .sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
1105
+ return { queues, nextCursor: "" };
1106
+ },
1107
+ markQueuePaused: (req, ctx) => {
1108
+ const who = this.enter("MarkQueuePaused", ctx, "pipelines:queue:edit");
1109
+ const q = byId(who, req.queueId);
1110
+ if (!q.paused) {
1111
+ q.paused = true;
1112
+ q.pausedAt = timestampFromMs(Date.now());
1113
+ q.pausedBy = `urn:fake:sa:${who.tenant}`;
1114
+ q.pausedReason = req.reason;
1115
+ }
1116
+ return { queue: q };
1117
+ },
1118
+ markQueueResumed: (req, ctx) => {
1119
+ const who = this.enter("MarkQueueResumed", ctx, "pipelines:queue:edit");
1120
+ const q = byId(who, req.queueId);
1121
+ q.paused = false;
1122
+ q.pausedReason = "";
1123
+ q.pausedBy = "";
1124
+ delete q.pausedAt;
1125
+ this.notify();
1126
+ return { queue: q };
1127
+ },
1128
+ deleteQueue: (req, ctx) => {
1129
+ const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
1130
+ const q = byId(who, req.queueId);
1131
+ const waiting = [...this.runs.values()].filter((r) => r.queueId === q.id && (r.state === RunState.QUEUED || (r.state === RunState.RUNNING && this.attemptsOf(r.id).some((a) => a.state === AttemptState.OFFERED))));
1132
+ if (waiting.length > 0 && !req.cancelWaiting)
1133
+ throw new ConnectError(`queue ${q.name} has ${waiting.length} waiting jobs`, Code.FailedPrecondition);
1134
+ for (const r of waiting)
1135
+ this.cancel(r, "queue deleted");
1136
+ if (q.name === "default" && !q.owner?.tenant) {
1137
+ this.queues.set(q.id, create(QueueSchema, { id: q.id, owner: q.owner, name: "default", createdAt: q.createdAt, updatedAt: timestampFromMs(Date.now()) }));
1138
+ }
1139
+ else {
1140
+ this.queues.delete(q.id);
1141
+ }
1142
+ return { cancelledJobs: waiting.length };
1143
+ },
1144
+ getQueueStats: (req, ctx) => {
1145
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1146
+ const q = byId(who, req.queueId);
1147
+ const now = Date.now();
1148
+ const windowMs = req.window ? durationMs(req.window) : 3_600_000;
1149
+ const runs = [...this.runs.values()].filter((r) => r.queueId === q.id);
1150
+ let waiting = 0;
1151
+ let delayed = 0;
1152
+ let running = 0;
1153
+ let started = 0n;
1154
+ let oldest;
1155
+ const tenants = new Map();
1156
+ for (const r of runs) {
1157
+ const t = tenants.get(r.tenant) ?? { waiting: 0, running: 0 };
1158
+ for (const a of this.attemptsOf(r.id)) {
1159
+ if (a.state === AttemptState.OFFERED) {
1160
+ if (a.notBefore > now || r.notBefore > now)
1161
+ delayed++;
1162
+ else {
1163
+ waiting++;
1164
+ t.waiting++;
1165
+ const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
1166
+ oldest = oldest === undefined ? since : Math.min(oldest, since);
1167
+ }
1168
+ }
1169
+ if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
1170
+ running++;
1171
+ t.running++;
1172
+ }
1173
+ if (a.startedAt && a.startedAt >= now - windowMs)
1174
+ started++;
1175
+ }
1176
+ if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
1177
+ delayed++;
1178
+ tenants.set(r.tenant, t);
1179
+ }
1180
+ const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
1181
+ return {
1182
+ waiting,
1183
+ delayed,
1184
+ running,
1185
+ ...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
1186
+ tenants: [...tenants.entries()]
1187
+ .map(([tenant, v]) => ({ tenant, ...v }))
1188
+ .sort((a, b) => b.waiting - a.waiting || b.running - a.running)
1189
+ .slice(0, req.topTenants || 10),
1190
+ started,
1191
+ failed,
1192
+ paused: q.paused,
1193
+ at: timestampFromMs(now),
1194
+ };
1195
+ },
1196
+ listQueueJobs: (req, ctx) => {
1197
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1198
+ const q = byId(who, req.queueId);
1199
+ const runs = [...this.runs.values()]
1200
+ .filter((r) => r.queueId === q.id && (!req.tenant || r.tenant === req.tenant) && (req.states.length === 0 || req.states.includes(r.state)))
1201
+ .reverse();
1202
+ return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
1203
+ },
1204
+ };
1205
+ }
1206
+ // ---- jobs and queues --------------------------------------------------
1207
+ /** CreateJob's work, shared with schedules: resolve the queue, apply its limits, make the one-node run. */
1208
+ createJobRun(who, spec) {
1209
+ this.requireHandler(spec.handler);
1210
+ if (spec.retry && spec.retry.max > 10)
1211
+ throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
1212
+ if (spec.retry?.when.includes(FailureClass.CANCELLED))
1213
+ throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
1214
+ const q = this.resolveQueue(productOf(who), spec.tenant, spec.queue);
1215
+ if (!q)
1216
+ throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among this product's queues`, Code.NotFound);
1217
+ const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
1218
+ const payload = this.checkInputs("job", spec.handler, spec.payload);
1219
+ const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
1220
+ run.queueId = q.id;
1221
+ run.queue = q.name;
1222
+ run.trigger = spec.trigger ?? RunTrigger.API;
1223
+ if (spec.schedule) {
1224
+ run.scheduleId = spec.schedule.id;
1225
+ run.scheduledFor = spec.schedule.scheduledFor;
1226
+ run.scheduleTarget = spec.schedule.target;
1227
+ }
1228
+ this.addNode(run, {
1229
+ id: "job",
1230
+ handler: spec.handler,
1231
+ uses: "",
1232
+ with: payload,
1233
+ needs: [],
1234
+ retry,
1235
+ timeoutMs,
1236
+ interruptible: spec.interruptible,
1237
+ });
1238
+ this.advance(run);
1239
+ return run;
1240
+ }
1241
+ /** A job's queue: the tenant's own of that name, else the product-wide one; empty is `default`. */
1242
+ resolveQueue(product, tenant, name) {
1243
+ const want = name || "default";
1244
+ const all = [...this.queues.values()].filter((q) => q.owner?.product === product && q.name === want);
1245
+ const found = all.find((q) => q.owner?.tenant === tenant) ?? all.find((q) => !q.owner?.tenant);
1246
+ if (found)
1247
+ return found;
1248
+ return want === "default" ? this.defaultQueue(product) : undefined;
1249
+ }
1250
+ /** The product's `default` queue, made on first use with no limits. */
1251
+ defaultQueue(product) {
1252
+ const existing = [...this.queues.values()].find((q) => q.owner?.product === product && !q.owner.tenant && q.name === "default");
1253
+ if (existing)
1254
+ return existing;
1255
+ const now = Date.now();
1256
+ const q = create(QueueSchema, { id: randomUUID(), owner: { product, tenant: "" }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
1257
+ this.queues.set(q.id, q);
1258
+ return q;
1259
+ }
1260
+ /** RetryNode: reopens a finished run and offers the node again; on a job, a requeue. */
1261
+ retryNode(run, nodeId) {
1262
+ const n = run.nodes.get(nodeId);
1263
+ if (!n)
1264
+ throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
1265
+ if (!terminal(run.state))
1266
+ throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
1267
+ if (!n.handler || n.attempts.length === 0 || !terminalNode(n.state))
1268
+ throw new ConnectError("the node never settled", Code.FailedPrecondition);
1269
+ run.state = RunState.RUNNING;
1270
+ run.finishedAt = undefined;
1271
+ run.failureClass = FailureClass.UNSPECIFIED;
1272
+ run.failedNodeId = "";
1273
+ run.cancelRequestedAt = undefined;
1274
+ // The node and everything downstream of it run again.
1275
+ const reset = new Set([nodeId]);
1276
+ for (let grew = true; grew;) {
1277
+ grew = false;
1278
+ for (const x of run.nodes.values()) {
1279
+ if (!reset.has(x.id) && x.needs.some((d) => reset.has(d))) {
1280
+ reset.add(x.id);
1281
+ grew = true;
1282
+ }
1283
+ }
1284
+ }
1285
+ for (const id of reset) {
1286
+ const x = run.nodes.get(id);
1287
+ x.state = id === nodeId ? NodeState.READY : NodeState.PENDING;
1288
+ x.failureClass = FailureClass.UNSPECIFIED;
1289
+ x.kv = {};
1290
+ }
1291
+ const inputs = this.resolve(run, n);
1292
+ this.offer(run, n, typeof inputs === "string" ? {} : inputs, 0);
1293
+ }
1294
+ // ---- issues, progress and spans ---------------------------------------
1295
+ /** Every declared stage PENDING from the claim, as the platform does. */
1296
+ initProgress(a) {
1297
+ if (a.progress)
1298
+ return;
1299
+ const n = this.run(a.runId).nodes.get(a.nodeId);
1300
+ const pkg = this.packages.get(n.handler);
1301
+ const manifest = pkg ? [...pkg.versions.values()].at(-1) : undefined;
1302
+ const stages = manifest?.stages ?? [];
1303
+ if (stages.length === 0)
1304
+ return;
1305
+ a.progress = {
1306
+ fraction: 0,
1307
+ message: "",
1308
+ processed: 0n,
1309
+ total: 0n,
1310
+ failed: 0n,
1311
+ updatedAt: Date.now(),
1312
+ reports: 0,
1313
+ stages: stages.map((st) => ({ key: st.key, name: st.name, weight: st.weight || 1, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n })),
1314
+ };
1315
+ }
1316
+ setProgress(a, req) {
1317
+ const now = Date.now();
1318
+ const p = (a.progress ??= { fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n, stages: [], updatedAt: now, reports: 0 });
1319
+ p.reports++;
1320
+ p.updatedAt = now;
1321
+ p.message = req.message;
1322
+ p.processed = req.processed;
1323
+ p.total = req.total;
1324
+ p.failed = req.failed;
1325
+ if (!req.stage) {
1326
+ p.fraction = Math.max(p.fraction, req.fraction);
1327
+ return;
1328
+ }
1329
+ let st = p.stages.find((x) => x.key === req.stage);
1330
+ if (!st) {
1331
+ st = { key: req.stage, name: req.stageName || req.stage, weight: 0, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n };
1332
+ p.stages.push(st);
1333
+ }
1334
+ // Reporting a stage finishes the one before it.
1335
+ for (const other of p.stages) {
1336
+ if (other !== st && other.state === StageState.RUNNING) {
1337
+ other.state = StageState.SUCCEEDED;
1338
+ other.fraction = 1;
1339
+ other.finishedAt = now;
1340
+ }
1341
+ }
1342
+ const state = req.stageState === StageState.UNSPECIFIED ? StageState.RUNNING : req.stageState;
1343
+ if (st.state === StageState.PENDING || st.state === StageState.RUNNING) {
1344
+ st.state = state;
1345
+ st.startedAt ??= now;
1346
+ if (state !== StageState.RUNNING)
1347
+ st.finishedAt = now;
1348
+ }
1349
+ st.fraction = state === StageState.SUCCEEDED ? 1 : req.fraction;
1350
+ st.message = req.message;
1351
+ st.processed = req.processed;
1352
+ st.total = req.total;
1353
+ st.failed = req.failed;
1354
+ const total = p.stages.reduce((s, x) => s + x.weight, 0);
1355
+ if (total > 0) {
1356
+ const done = p.stages.reduce((s, x) => s + (x.state === StageState.SUCCEEDED || x.state === StageState.FAILED || x.state === StageState.SKIPPED ? x.weight : x.state === StageState.RUNNING ? x.weight * x.fraction : 0), 0);
1357
+ p.fraction = Math.max(p.fraction, Math.min(1, done / total));
1358
+ }
1359
+ }
1360
+ appendIssues(a, issues) {
1361
+ const run = this.run(a.runId);
1362
+ let accepted = 0;
1363
+ for (const i of issues) {
1364
+ if (!i.message)
1365
+ throw new ConnectError("issues.message: required", Code.InvalidArgument);
1366
+ if (i.severity === 0)
1367
+ throw new ConnectError("issues.severity: required", Code.InvalidArgument);
1368
+ }
1369
+ for (const i of issues) {
1370
+ if (run.issues.length >= this.opts.issueCap) {
1371
+ run.issuesTruncated = true;
1372
+ continue;
1373
+ }
1374
+ run.issues.push({
1375
+ id: randomUUID(),
1376
+ runId: run.id,
1377
+ nodeId: a.nodeId,
1378
+ attemptId: a.id,
1379
+ stage: i.stage,
1380
+ severity: i.severity,
1381
+ subject: i.subject || i.subjectId,
1382
+ subjectKind: i.subjectKind,
1383
+ subjectId: i.subjectId,
1384
+ message: i.message,
1385
+ detail: i.detail,
1386
+ createdAt: Date.now(),
1387
+ });
1388
+ accepted++;
1389
+ }
1390
+ return accepted;
1391
+ }
1392
+ /** Issues of a run, filtered as ListIssues filters them. */
1393
+ issuesOf(runId, filter = {}) {
1394
+ return this.run(runId).issues.filter((i) => (!filter.nodeId || i.nodeId === filter.nodeId) && (!filter.attemptId || i.attemptId === filter.attemptId) && (!filter.minSeverity || i.severity >= filter.minSeverity));
1395
+ }
1396
+ /** The run's trace: the platform's spans for the run, its nodes and attempts, and the step spans reported. */
1397
+ traceOf(runId, filter = {}) {
1398
+ const run = this.run(runId);
1399
+ const traceId = run.id.replaceAll("-", "");
1400
+ const sid = (s) => s.replaceAll("-", "").slice(0, 16);
1401
+ const out = [
1402
+ create(TraceSpanSchema, {
1403
+ traceId,
1404
+ spanId: sid(run.id),
1405
+ name: `run ${run.id}`,
1406
+ source: TraceSpanSource.PLATFORM,
1407
+ startedAt: timestampFromMs(run.createdAt),
1408
+ ...(run.finishedAt ? { endedAt: timestampFromMs(run.finishedAt) } : {}),
1409
+ }),
1410
+ ];
1411
+ for (const n of run.nodes.values()) {
1412
+ if (filter.nodeId && n.id !== filter.nodeId)
1413
+ continue;
1414
+ const nodeSpan = sid(randomUUIDFrom(run.id + n.id));
1415
+ const first = n.attempts[0];
1416
+ out.push(create(TraceSpanSchema, {
1417
+ traceId,
1418
+ spanId: nodeSpan,
1419
+ parentSpanId: sid(run.id),
1420
+ name: `node ${n.id}`,
1421
+ source: TraceSpanSource.PLATFORM,
1422
+ nodeId: n.id,
1423
+ startedAt: timestampFromMs(first?.offeredAt ?? run.createdAt),
1424
+ }));
1425
+ for (const a of n.attempts) {
1426
+ if (filter.attemptId && a.id !== filter.attemptId)
1427
+ continue;
1428
+ out.push(create(TraceSpanSchema, {
1429
+ traceId,
1430
+ spanId: sid(a.id),
1431
+ parentSpanId: nodeSpan,
1432
+ name: `attempt ${a.number}`,
1433
+ source: TraceSpanSource.PLATFORM,
1434
+ nodeId: n.id,
1435
+ attemptId: a.id,
1436
+ startedAt: timestampFromMs(a.offeredAt),
1437
+ ...(a.completedAt ? { endedAt: timestampFromMs(a.completedAt) } : {}),
1438
+ }));
1439
+ for (const sp of a.spans) {
1440
+ out.push(create(TraceSpanSchema, {
1441
+ traceId: sp.traceId,
1442
+ spanId: sp.spanId,
1443
+ parentSpanId: sp.parentSpanId || sid(a.id),
1444
+ name: sp.name,
1445
+ source: TraceSpanSource.STEP,
1446
+ nodeId: n.id,
1447
+ attemptId: a.id,
1448
+ ...(sp.startedAt ? { startedAt: sp.startedAt } : {}),
1449
+ ...(sp.endedAt ? { endedAt: sp.endedAt } : {}),
1450
+ status: sp.status,
1451
+ statusMessage: sp.statusMessage,
1452
+ ...(sp.attributes ? { attributes: sp.attributes } : {}),
1453
+ }));
1454
+ }
1455
+ }
1456
+ }
1457
+ return out.sort((x, y) => Number(x.startedAt?.seconds ?? 0n) - Number(y.startedAt?.seconds ?? 0n));
1458
+ }
1459
+ /** Runs whenever a run becomes terminal. */
1460
+ runFinished(run) {
1461
+ // The schedules are made after the router; a run cannot finish before.
1462
+ this.schedules?.runFinished(run);
1463
+ }
846
1464
  // ---- messages ---------------------------------------------------------
847
1465
  runMessage(r) {
848
1466
  return create(RunSchema, {
@@ -862,6 +1480,16 @@ export class FakePipelines {
862
1480
  ...(r.finishedAt ? { finishedAt: timestampFromMs(r.finishedAt) } : {}),
863
1481
  ...(r.cancelRequestedAt ? { cancelRequestedAt: timestampFromMs(r.cancelRequestedAt) } : {}),
864
1482
  cancelReason: r.cancelReason,
1483
+ queueId: r.queueId,
1484
+ queue: r.queue,
1485
+ trigger: r.trigger,
1486
+ scheduleId: r.scheduleId,
1487
+ ...(r.scheduledFor !== undefined ? { scheduledFor: timestampFromMs(r.scheduledFor) } : {}),
1488
+ scheduleTarget: r.scheduleTarget,
1489
+ issueInfoCount: r.issues.filter((i) => i.severity === IssueSeverity.INFO).length,
1490
+ issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
1491
+ issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
1492
+ issuesTruncated: r.issuesTruncated,
865
1493
  });
866
1494
  }
867
1495
  nodeMessage(n) {
@@ -886,6 +1514,31 @@ export class FakePipelines {
886
1514
  message: a.message,
887
1515
  logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
888
1516
  logTruncated: a.logTruncated,
1517
+ ...(a.progress
1518
+ ? {
1519
+ progress: {
1520
+ fraction: a.progress.fraction,
1521
+ message: a.progress.message,
1522
+ processed: a.progress.processed,
1523
+ total: a.progress.total,
1524
+ failed: a.progress.failed,
1525
+ updatedAt: timestampFromMs(a.progress.updatedAt),
1526
+ stages: a.progress.stages.map((st) => ({
1527
+ key: st.key,
1528
+ name: st.name,
1529
+ weight: st.weight,
1530
+ state: st.state,
1531
+ fraction: st.fraction,
1532
+ message: st.message,
1533
+ processed: st.processed,
1534
+ total: st.total,
1535
+ failed: st.failed,
1536
+ ...(st.startedAt ? { startedAt: timestampFromMs(st.startedAt) } : {}),
1537
+ ...(st.finishedAt ? { finishedAt: timestampFromMs(st.finishedAt) } : {}),
1538
+ })),
1539
+ },
1540
+ }
1541
+ : {}),
889
1542
  })),
890
1543
  });
891
1544
  }
@@ -1012,4 +1665,44 @@ function aborted(s) {
1012
1665
  s.addEventListener("abort", () => r(), { once: true });
1013
1666
  });
1014
1667
  }
1668
+ /** The product an account belongs to. */
1669
+ function productOf(who) {
1670
+ return who.product ?? FAKE_PRODUCT;
1671
+ }
1672
+ /** A queue's limits as the selection reads them. */
1673
+ function queueLimits(q) {
1674
+ return {
1675
+ id: q.id,
1676
+ priority: q.priority || 1,
1677
+ paused: q.paused,
1678
+ maxConcurrency: q.maxConcurrency,
1679
+ maxConcurrencyPerTenant: q.maxConcurrencyPerTenant,
1680
+ ...(q.rateLimit?.window ? { rateLimit: { limit: q.rateLimit.limit, windowMs: durationMs(q.rateLimit.window) } } : {}),
1681
+ };
1682
+ }
1683
+ /**
1684
+ * A job's retry and timeout under its queue: the queue's defaults when the
1685
+ * job gives none, and its ceilings, past which the job is refused.
1686
+ */
1687
+ function jobLimits(q, retry, timeoutMs) {
1688
+ const ceiling = q.defaultRetry;
1689
+ if (retry && ceiling) {
1690
+ if (retry.max > ceiling.max)
1691
+ throw new ConnectError(`retry.max ${retry.max} is above queue ${q.name}'s ${ceiling.max}`, Code.InvalidArgument);
1692
+ const allowed = ceiling.when.length > 0 ? ceiling.when : TRANSIENT;
1693
+ const outside = retry.when.filter((c) => !allowed.includes(c));
1694
+ if (outside.length > 0)
1695
+ throw new ConnectError(`retry.when lists ${outside.map((c) => FailureClass[c]).join(", ")}, which queue ${q.name} does not retry`, Code.InvalidArgument);
1696
+ }
1697
+ const maxTimeout = q.defaultTimeout ? durationMs(q.defaultTimeout) : 0;
1698
+ if (timeoutMs > 0 && maxTimeout > 0 && timeoutMs > maxTimeout) {
1699
+ throw new ConnectError(`timeout ${timeoutMs} ms is above queue ${q.name}'s ${maxTimeout} ms`, Code.InvalidArgument);
1700
+ }
1701
+ const r = retry ?? ceiling;
1702
+ return { retry: r ? { max: r.max, when: r.when } : undefined, timeoutMs: timeoutMs || maxTimeout };
1703
+ }
1704
+ /** A stable UUID-shaped id from a string, for the fake's platform spans. */
1705
+ function randomUUIDFrom(s) {
1706
+ return createHash("sha256").update(s).digest("hex").slice(0, 32);
1707
+ }
1015
1708
  //# sourceMappingURL=index.js.map