@xeonr/pipelines-sdk 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/README.md +231 -22
  2. package/dist/auth.d.ts +1 -1
  3. package/dist/client.d.ts +73 -10
  4. package/dist/client.d.ts.map +1 -1
  5. package/dist/client.js +47 -4
  6. package/dist/client.js.map +1 -1
  7. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +179 -16
  8. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
  9. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +45 -28
  10. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
  11. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +136 -17
  12. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -1
  13. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +42 -35
  14. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -1
  15. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +224 -8
  16. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
  17. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +43 -9
  18. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
  19. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +64 -8
  20. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
  21. package/dist/gen/xeonr/pipelines/v1/common_pb.js +62 -5
  22. package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
  23. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +312 -9
  24. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
  25. package/dist/gen/xeonr/pipelines/v1/definition_pb.js +100 -7
  26. package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
  27. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +1072 -0
  28. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -0
  29. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +299 -0
  30. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -0
  31. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +103 -8
  32. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
  33. package/dist/gen/xeonr/pipelines/v1/pools_pb.js +32 -24
  34. package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
  35. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +711 -0
  36. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -0
  37. package/dist/gen/xeonr/pipelines/v1/queues_pb.js +185 -0
  38. package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -0
  39. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +92 -23
  40. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
  41. package/dist/gen/xeonr/pipelines/v1/registry_pb.js +36 -33
  42. package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
  43. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +1191 -33
  44. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
  45. package/dist/gen/xeonr/pipelines/v1/runs_pb.js +419 -35
  46. package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
  47. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts +194 -0
  48. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts.map +1 -0
  49. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js +100 -0
  50. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js.map +1 -0
  51. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +1186 -0
  52. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -0
  53. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +335 -0
  54. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -0
  55. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +32 -5
  56. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
  57. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +12 -3
  58. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
  59. package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts +270 -27
  60. package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts.map +1 -1
  61. package/dist/gen/xeonr/pipelines/v1/sinks_pb.js +163 -24
  62. package/dist/gen/xeonr/pipelines/v1/sinks_pb.js.map +1 -1
  63. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts +391 -0
  64. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts.map +1 -0
  65. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js +100 -0
  66. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js.map +1 -0
  67. package/dist/gen/xeonr/platform/v1/options_pb.d.ts +137 -1
  68. package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
  69. package/dist/gen/xeonr/platform/v1/options_pb.js +104 -1
  70. package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
  71. package/dist/handler.d.ts +51 -1
  72. package/dist/handler.d.ts.map +1 -1
  73. package/dist/handler.js +27 -0
  74. package/dist/handler.js.map +1 -1
  75. package/dist/index.d.ts +13 -2
  76. package/dist/index.d.ts.map +1 -1
  77. package/dist/index.js +7 -2
  78. package/dist/index.js.map +1 -1
  79. package/dist/queues.d.ts +158 -0
  80. package/dist/queues.d.ts.map +1 -0
  81. package/dist/queues.js +117 -0
  82. package/dist/queues.js.map +1 -0
  83. package/dist/reporter.d.ts +120 -0
  84. package/dist/reporter.d.ts.map +1 -0
  85. package/dist/reporter.js +323 -0
  86. package/dist/reporter.js.map +1 -0
  87. package/dist/schedules.d.ts +193 -0
  88. package/dist/schedules.d.ts.map +1 -0
  89. package/dist/schedules.js +126 -0
  90. package/dist/schedules.js.map +1 -0
  91. package/dist/testing/cron.d.ts +33 -0
  92. package/dist/testing/cron.d.ts.map +1 -0
  93. package/dist/testing/cron.js +177 -0
  94. package/dist/testing/cron.js.map +1 -0
  95. package/dist/testing/index.d.ts +229 -7
  96. package/dist/testing/index.d.ts.map +1 -1
  97. package/dist/testing/index.js +1304 -103
  98. package/dist/testing/index.js.map +1 -1
  99. package/dist/testing/pipelines.d.ts +99 -0
  100. package/dist/testing/pipelines.d.ts.map +1 -0
  101. package/dist/testing/pipelines.js +445 -0
  102. package/dist/testing/pipelines.js.map +1 -0
  103. package/dist/testing/queues.d.ts +49 -0
  104. package/dist/testing/queues.d.ts.map +1 -0
  105. package/dist/testing/queues.js +72 -0
  106. package/dist/testing/queues.js.map +1 -0
  107. package/dist/testing/schedules.d.ts +125 -0
  108. package/dist/testing/schedules.d.ts.map +1 -0
  109. package/dist/testing/schedules.js +422 -0
  110. package/dist/testing/schedules.js.map +1 -0
  111. package/dist/worker.d.ts +48 -7
  112. package/dist/worker.d.ts.map +1 -1
  113. package/dist/worker.js +59 -9
  114. package/dist/worker.js.map +1 -1
  115. package/package.json +1 -1
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * @xeonr/pipelines-sdk/testing: an in-memory fake of pipelines-api's worker
3
- * and jobs surface, so a product can unit-test its handlers, and the code
3
+ * and jobs surface, so a service can unit-test its handlers, and the code
4
4
  * that enqueues them, with no server.
5
5
  *
6
6
  * It is a real Connect router behind an in-memory transport, so the SDK's
@@ -31,19 +31,45 @@
31
31
  * `handler_failure_kind`, retries by class, free interruptions on an
32
32
  * `interruptible` node, cancel, and error injection.
33
33
  *
34
+ * Stored pipelines (`PipelinesService`, see ./pipelines.ts) and the runs of
35
+ * their versions (`CreateRun` with `pipeline` or `snapshot_id`, and a
36
+ * schedule's pipeline target), with the policy's node defaults and
37
+ * `max_concurrent_runs`. Sub-pipeline nodes: pinned to a version when the
38
+ * snapshot is made (cycles and nesting over 3 refused), one child run per
39
+ * attempt about the parent's resource, the child's outputs mapped back, a
40
+ * failed child failing the attempt as `DOWNSTREAM_FAILED`, a cancel passed
41
+ * down. `for_each` nodes on handler and sub-pipeline nodes: the list from a
42
+ * literal, a variable or an upstream kv output, `${{ item }}`,
43
+ * `${{ item.<path> }}` and `${{ item_index }}`, `max_items`,
44
+ * `max_parallel`, each element retried on its own, the three `on_failure`
45
+ * policies, outputs aggregated in element order, and RetryNode running only
46
+ * the failed elements again.
47
+ *
34
48
  * What it does not: secrets, blobs, tiers, routing between pools, the
35
- * outbox, and the platform's own deadline enforcement.
49
+ * outbox, exclusions, `for_each` on an external (image) node, and the
50
+ * platform's own deadline enforcement.
36
51
  */
37
- import { randomUUID } from "node:crypto";
38
- import { create, fromJson, toJson } from "@bufbuild/protobuf";
39
- import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
52
+ import { createHash, randomUUID } from "node:crypto";
53
+ import { clone, create, fromJson, toJson } from "@bufbuild/protobuf";
54
+ import { DurationSchema, durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
40
55
  import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
41
56
  import { tokenSource } from "../auth.js";
42
57
  import { authInterceptor } from "../transport.js";
43
58
  import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
59
+ import { DefinitionSchema, ForEachFailure, RetrySchema } from "../gen/xeonr/pipelines/v1/definition_pb.js";
60
+ import { PipelineState, PipelinesService } from "../gen/xeonr/pipelines/v1/pipelines_pb.js";
44
61
  import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
45
62
  import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
46
- import { AttemptSchema, NodeRunSchema, RunSchema, RunsService } from "../gen/xeonr/pipelines/v1/runs_pb.js";
63
+ import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunOrigin, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
64
+ import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
65
+ import { FakeStoredPipelines, takesRunsAbout } from "./pipelines.js";
66
+ import { pickQueued } from "./queues.js";
67
+ import { FakeClock, FakeSchedules } from "./schedules.js";
68
+ import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
69
+ export { FakeClock, FakeSchedules } from "./schedules.js";
70
+ export { parseCron, nextOccurrence } from "./cron.js";
71
+ export { pickQueued } from "./queues.js";
72
+ export { FakeStoredPipelines, resolveVariables } from "./pipelines.js";
47
73
  import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
48
74
  /** The tenant the default account acts for. */
49
75
  export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
@@ -58,6 +84,12 @@ export const ALL_SCOPES = [
58
84
  "pipelines:pool:read",
59
85
  "pipelines:pool:edit",
60
86
  "pipelines:worker:edit",
87
+ "pipelines:queue:read",
88
+ "pipelines:queue:edit",
89
+ "pipelines:schedule:read",
90
+ "pipelines:schedule:edit",
91
+ "pipelines:pipeline:read",
92
+ "pipelines:pipeline:edit",
61
93
  ];
62
94
  const TRANSIENT = [
63
95
  FailureClass.ABANDONED,
@@ -68,9 +100,11 @@ const TRANSIENT = [
68
100
  FailureClass.INTERNAL,
69
101
  ];
70
102
  const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
71
- /** How many interruptions an interruptible node gets without spending a retry. */
72
- const FREE_INTERRUPTIONS = 10;
103
+ /** How many interruptions an interruptible node gets without spending a retry (the api's default). */
104
+ const FREE_INTERRUPTIONS = 3;
73
105
  const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
106
+ /** How deeply runs nest through sub-pipelines, counting the outermost (the api's MaxDepth). */
107
+ const MAX_DEPTH = 3;
74
108
  /**
75
109
  * An in-memory pipelines-api for tests. Call {@link close} when done (it
76
110
  * stops the timer that looks at leases and delays).
@@ -88,6 +122,18 @@ export class FakePipelines {
88
122
  attemptsById = new Map();
89
123
  idempotency = new Map();
90
124
  injected = new Map();
125
+ /** Queues by id. */
126
+ queues = new Map();
127
+ /**
128
+ * The clock schedules run on: real time until a test calls
129
+ * `clock.set()` or `clock.advance()`, each of which fires what is due.
130
+ */
131
+ clock = new FakeClock();
132
+ /** The schedules, with their recorded occurrences. */
133
+ schedules;
134
+ /** Stored pipelines, their versions and the snapshots runs execute. */
135
+ pipelines;
136
+ queueClaims = new Map();
91
137
  timer;
92
138
  changed;
93
139
  constructor(opts = {}) {
@@ -102,8 +148,54 @@ export class FakePipelines {
102
148
  retryBackoffMs: opts.retryBackoffMs ?? 0,
103
149
  logMaxBytes: opts.logMaxBytes ?? 0,
104
150
  tickMs: opts.tickMs ?? 10,
151
+ issueCap: opts.issueCap ?? 1000,
152
+ spanCap: opts.spanCap ?? 10_000,
105
153
  };
106
154
  this.changed = signal();
155
+ this.pipelines = new FakeStoredPipelines({
156
+ enter: (method, ctx, scope) => this.enter(method, ctx, scope),
157
+ freeze: (def, env) => this.freeze(def, env),
158
+ });
159
+ this.schedules = new FakeSchedules({
160
+ createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
161
+ tenant: owner.tenant,
162
+ handler: spec.handler,
163
+ key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
164
+ payload: spec.payload,
165
+ delayMs: 0,
166
+ resource: spec.resource,
167
+ retry: spec.retry,
168
+ timeoutMs: spec.timeoutMs,
169
+ queue: spec.queue,
170
+ interruptible: spec.interruptible,
171
+ trigger: spec.trigger,
172
+ schedule: spec.schedule,
173
+ }),
174
+ startPipeline: (owner, spec) => {
175
+ const st = this.pipelines.get(spec.target.pipelineId);
176
+ if (!st || st.pipeline.tenant !== owner.tenant)
177
+ throw new ConnectError("no such pipeline", Code.NotFound);
178
+ return this.startStored({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
179
+ tenant: owner.tenant,
180
+ key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
181
+ resource: spec.resource ?? (st.pipeline.scope ? { kind: st.pipeline.scope.kind, id: st.pipeline.scope.id } : undefined),
182
+ variables: spec.variables,
183
+ facts: spec.target.attestedFacts,
184
+ stored: st,
185
+ version: spec.target.version,
186
+ trigger: spec.trigger,
187
+ schedule: spec.schedule,
188
+ });
189
+ },
190
+ checkPipeline: (tenant, id) => {
191
+ const st = this.pipelines.get(id);
192
+ if (!st || st.pipeline.tenant !== tenant)
193
+ throw new ConnectError(`schedule.target.pipeline_id: no pipeline ${id} of the tenant`, Code.NotFound);
194
+ },
195
+ runs: () => this.runs.values(),
196
+ runMessage: (id) => this.runMessage(this.run(id)),
197
+ checkHandler: (handler) => this.requireHandler(handler),
198
+ }, this.clock);
107
199
  this.timer = setInterval(() => this.tick(), this.opts.tickMs);
108
200
  this.timer.unref();
109
201
  }
@@ -208,7 +300,7 @@ export class FakePipelines {
208
300
  completeNode(runId, nodeId, result = {}) {
209
301
  const run = this.run(runId);
210
302
  const node = run.nodes.get(nodeId);
211
- if (!node || node.handler)
303
+ if (!node || node.handler || node.pipeline)
212
304
  throw new Error(`FakePipelines.completeNode: ${nodeId} is not an external node of ${runId}`);
213
305
  if (node.state !== NodeState.READY)
214
306
  throw new Error(`FakePipelines.completeNode: ${nodeId} is not ready`);
@@ -371,50 +463,145 @@ export class FakePipelines {
371
463
  this.notify();
372
464
  return { state: a.state };
373
465
  },
466
+ appendIssues: (req, ctx) => {
467
+ const who = this.enter("AppendIssues", ctx, "pipelines:worker:edit");
468
+ const a = this.held(who, req.ref);
469
+ if (a.state !== AttemptState.RUNNING)
470
+ throw stale();
471
+ if (req.issues.length < 1 || req.issues.length > 100)
472
+ throw new ConnectError("issues: 1 to 100", Code.InvalidArgument);
473
+ return { accepted: this.appendIssues(a, req.issues) };
474
+ },
475
+ setProgress: (req, ctx) => {
476
+ const who = this.enter("SetProgress", ctx, "pipelines:worker:edit");
477
+ const a = this.held(who, req.ref);
478
+ if (a.state !== AttemptState.RUNNING)
479
+ throw stale();
480
+ if (!(req.fraction >= 0 && req.fraction <= 1))
481
+ throw new ConnectError("fraction: 0 to 1", Code.InvalidArgument);
482
+ if (req.stageState === StageState.PENDING)
483
+ throw new ConnectError("stage_state: a stage never moves back to PENDING", Code.InvalidArgument);
484
+ if (req.stage && !/^[a-z][a-z0-9_.-]{0,63}$/.test(req.stage))
485
+ throw new ConnectError("stage: invalid key", Code.InvalidArgument);
486
+ if (req.message.length > 255)
487
+ throw new ConnectError("message: at most 255", Code.InvalidArgument);
488
+ this.setProgress(a, req);
489
+ return {};
490
+ },
491
+ reportSpans: (req, ctx) => {
492
+ const who = this.enter("ReportSpans", ctx, "pipelines:worker:edit");
493
+ const a = this.held(who, req.ref);
494
+ if (a.state !== AttemptState.RUNNING)
495
+ throw stale();
496
+ if (req.spans.length < 1 || req.spans.length > 1000)
497
+ throw new ConnectError("spans: 1 to 1000", Code.InvalidArgument);
498
+ for (const sp of req.spans) {
499
+ if (!/^[0-9a-f]{32}$/.test(sp.traceId) || !/^[0-9a-f]{16}$/.test(sp.spanId) || !sp.startedAt || !sp.endedAt || !sp.name) {
500
+ throw new ConnectError("spans: trace_id, span_id, name, started_at and ended_at are required", Code.InvalidArgument);
501
+ }
502
+ }
503
+ const room = Math.max(0, this.opts.spanCap - a.spans.length);
504
+ a.spans.push(...req.spans.slice(0, room));
505
+ return { accepted: Math.min(room, req.spans.length) };
506
+ },
374
507
  });
508
+ router.service(QueuesService, this.queuesService());
509
+ router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
510
+ router.service(PipelinesService, this.pipelines.service());
375
511
  router.service(RunsService, {
512
+ listIssues: (req, ctx) => {
513
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
514
+ this.visible(who, req.runId);
515
+ const issues = this.issuesOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId, minSeverity: req.minSeverity });
516
+ return {
517
+ issues: issues.map((i) => create(RunIssueSchema, { ...i, detail: i.detail, createdAt: timestampFromMs(i.createdAt) })),
518
+ nextCursor: "",
519
+ };
520
+ },
521
+ getLogs: (req, ctx) => {
522
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
523
+ const a = this.attemptsById.get(req.attemptId);
524
+ if (!a)
525
+ throw new ConnectError("no such attempt", Code.NotFound);
526
+ this.visible(who, a.runId);
527
+ const all = Buffer.concat(a.chunks);
528
+ const from = Math.min(Number(req.offset), all.length);
529
+ const to = req.limitBytes ? Math.min(all.length, from + req.limitBytes) : all.length;
530
+ const complete = to === all.length && (a.logFinal || a.completedAt !== undefined);
531
+ return { data: new Uint8Array(all.subarray(from, to)), nextOffset: BigInt(to), complete, truncated: a.logTruncated };
532
+ },
533
+ getRunTrace: (req, ctx) => {
534
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
535
+ this.visible(who, req.runId);
536
+ return { spans: this.traceOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId }), truncated: false };
537
+ },
376
538
  createJob: (req, ctx) => {
377
539
  const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
378
540
  this.actsFor(who, req.tenant);
379
- const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
541
+ const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
380
542
  if (existing)
381
543
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
382
- this.requireHandler(req.handler);
383
- const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, {}, req.delay ? durationMs(req.delay) : 0);
384
- if (req.retry && req.retry.max > 10)
385
- throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
386
- if (req.retry?.when.includes(FailureClass.CANCELLED))
387
- throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
388
- this.addNode(run, {
389
- id: "job",
544
+ const run = this.createJobRun(who, {
545
+ fairnessKey: req.fairnessKey,
546
+ tenant: req.tenant,
390
547
  handler: req.handler,
391
- uses: "",
392
- with: req.payload ?? {},
393
- needs: [],
394
- retry: req.retry ? { max: req.retry.max, when: req.retry.when } : undefined,
548
+ key: req.idempotencyKey,
549
+ payload: req.payload ?? {},
550
+ delayMs: req.delay ? durationMs(req.delay) : 0,
551
+ resource: req.resource,
552
+ retry: req.retry,
395
553
  timeoutMs: req.timeout ? durationMs(req.timeout) : 0,
396
- interruptible: false,
554
+ queue: req.queue,
555
+ interruptible: req.interruptible,
397
556
  });
398
- this.advance(run);
399
557
  return { run: this.runMessage(run), deduplicated: false };
400
558
  },
559
+ retryNode: (req, ctx) => {
560
+ const who = this.enter("RetryNode", ctx, "pipelines:run:edit");
561
+ const run = this.visible(who, req.runId);
562
+ this.retryNode(run, req.nodeId);
563
+ return { run: this.runMessage(run) };
564
+ },
401
565
  createRun: (req, ctx) => {
402
566
  const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
403
567
  this.actsFor(who, req.tenant);
404
- const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
568
+ const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
405
569
  if (existing)
406
570
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
407
- if (req.source.case !== "inlineDefinition")
408
- throw new ConnectError("only inline definitions", Code.Unimplemented);
409
- const def = req.source.value;
410
- if (def.nodes.length === 0)
411
- throw new ConnectError("definition: no nodes", Code.InvalidArgument);
412
- const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
413
- checkGraph(specs);
414
- const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, req.variables ?? {}, 0);
415
- for (const spec of specs)
416
- this.addNode(run, spec);
417
- this.advance(run);
571
+ const start = { tenant: req.tenant, key: req.idempotencyKey, resource: req.resource, variables: req.variables, facts: req.attestedFacts };
572
+ let run;
573
+ switch (req.source.case) {
574
+ case "inlineDefinition": {
575
+ const about = req.resource ? { kind: req.resource.kind, id: req.resource.id } : undefined;
576
+ const def = this.freeze(req.source.value, { tenant: req.tenant, scope: about, about });
577
+ run = this.startSnapshot(who, { ...start, snapshot: this.pipelines.inlineSnapshot(req.tenant, def) });
578
+ break;
579
+ }
580
+ case "snapshotId": {
581
+ const snap = this.pipelines.snapshot(req.source.value);
582
+ if (!snap || snap.tenant !== req.tenant)
583
+ throw new ConnectError("no such snapshot", Code.NotFound);
584
+ run = this.startSnapshot(who, { ...start, snapshot: snap });
585
+ break;
586
+ }
587
+ case "pipeline": {
588
+ const src = req.source.value;
589
+ const sel = src.pipeline;
590
+ const st = sel.case === "pipelineId"
591
+ ? this.pipelines.get(sel.value)
592
+ : sel.case === "byName" && sel.value.tenant === req.tenant
593
+ ? this.pipelines.find(sel.value.tenant, sel.value.scope, sel.value.name)
594
+ : undefined;
595
+ if (!st || st.pipeline.tenant !== req.tenant)
596
+ throw new ConnectError("no such pipeline", Code.NotFound);
597
+ if (!req.resource)
598
+ throw new ConnectError("resource: required", Code.InvalidArgument);
599
+ run = this.startStored(who, { ...start, stored: st, version: src.version, trigger: RunTrigger.API });
600
+ break;
601
+ }
602
+ default:
603
+ throw new ConnectError("source: required", Code.InvalidArgument);
604
+ }
418
605
  return { run: this.runMessage(run), deduplicated: false };
419
606
  },
420
607
  getRun: (req, ctx) => {
@@ -424,7 +611,11 @@ export class FakePipelines {
424
611
  getRunGraph: (req, ctx) => {
425
612
  const who = this.enter("GetRun", ctx, "pipelines:run:read");
426
613
  const run = this.visible(who, req.runId);
427
- return { run: this.runMessage(run), nodes: [...run.nodes.values()].map((n) => this.nodeMessage(n)) };
614
+ return {
615
+ run: this.runMessage(run),
616
+ nodes: [...run.nodes.values()].map((n) => this.nodeMessage(n)),
617
+ ...(run.definition ? { definition: run.definition } : {}),
618
+ };
428
619
  },
429
620
  listRuns: (req, ctx) => {
430
621
  const who = this.enter("GetRun", ctx, "pipelines:run:read");
@@ -537,14 +728,40 @@ export class FakePipelines {
537
728
  throw new ConnectError(`handler ${handler} is not a published worker package`, Code.InvalidArgument);
538
729
  }
539
730
  }
731
+ /**
732
+ * Holds a handler node's `with` to its manifest, as the api does when a run
733
+ * is created: every key must be a declared input, every required input
734
+ * given; defaults fill the rest. Returns the inputs with defaults.
735
+ */
736
+ checkInputs(nodeId, handler, given) {
737
+ const pkg = this.packages.get(handler);
738
+ const version = pkg ? [...pkg.versions.keys()].at(-1) : "";
739
+ const manifest = pkg?.versions.get(version);
740
+ const inputs = manifest?.inputs ?? [];
741
+ for (const k of Object.keys(given)) {
742
+ if (!inputs.some((i) => i.name === k)) {
743
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": input "${k}" is not an input of ${handler}@${version}`, Code.InvalidArgument);
744
+ }
745
+ }
746
+ const out = { ...given };
747
+ for (const i of inputs) {
748
+ if (out[i.name] !== undefined)
749
+ continue;
750
+ if (i.default)
751
+ out[i.name] = toJson(ValueSchema, i.default);
752
+ else if (i.required)
753
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": required input "${i.name}" is not given`, Code.InvalidArgument);
754
+ }
755
+ return out;
756
+ }
540
757
  // ---- the scheduler ----------------------------------------------------
541
- newRun(who, tenant, key, resource, variables, delayMs) {
758
+ newRun(who, tenant, key, resource, variables, delayMs, extra = {}) {
542
759
  const now = Date.now();
543
760
  const run = {
544
761
  id: randomUUID(),
545
762
  tenant,
546
- product: who.product ?? who.tenant,
547
763
  idempotencyKey: key,
764
+ fairnessKey: "",
548
765
  createdBy: `urn:fake:sa:${who.tenant}`,
549
766
  resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
550
767
  variables,
@@ -558,9 +775,27 @@ export class FakePipelines {
558
775
  cancelRequestedAt: undefined,
559
776
  cancelReason: "",
560
777
  nodes: new Map(),
778
+ queueId: "",
779
+ queue: "",
780
+ trigger: RunTrigger.API,
781
+ scheduleId: "",
782
+ scheduledFor: undefined,
783
+ scheduleTarget: "",
784
+ issues: [],
785
+ issuesTruncated: false,
786
+ attestedFacts: undefined,
787
+ snapshotId: "",
788
+ origin: RunOrigin.INLINE,
789
+ pipelineId: "",
790
+ pipelineVersion: 0,
791
+ parentRunId: "",
792
+ parentNodeId: "",
793
+ definition: undefined,
794
+ maxConcurrentRuns: 0,
795
+ ...extra,
561
796
  };
562
797
  this.runs.set(run.id, run);
563
- this.idempotency.set(`${run.product}\u0000${key}`, run.id);
798
+ this.idempotency.set(`${run.tenant}\u0000${key}`, run.id);
564
799
  return run;
565
800
  }
566
801
  addNode(run, spec) {
@@ -577,7 +812,170 @@ export class FakePipelines {
577
812
  retry: spec.retry ?? { max: this.opts.defaultRetries, when: [] },
578
813
  timeoutMs: spec.timeoutMs || this.opts.defaultTimeoutMs,
579
814
  interruptible: spec.interruptible,
815
+ pipeline: spec.pipeline,
816
+ forEach: spec.forEach,
817
+ elements: undefined,
818
+ });
819
+ }
820
+ // ---- definitions, snapshots and starts ---------------------------------
821
+ /**
822
+ * Checks a definition as CreateRun does and freezes it as its snapshot
823
+ * holds it: the policy's node defaults filled, each sub-pipeline pinned to
824
+ * the version it names (0, the latest now), a child of another tenant, an
825
+ * unpublished or archived one, one not taking runs about `env.about`, a
826
+ * cycle back to `env.self` or nesting over 3 refused.
827
+ */
828
+ freeze(def, env) {
829
+ if (def.nodes.length === 0)
830
+ throw new ConnectError("definition: no nodes", Code.InvalidArgument);
831
+ const out = clone(DefinitionSchema, def);
832
+ for (const n of out.nodes) {
833
+ if (!n.retry && env.defaultRetry)
834
+ n.retry = clone(RetrySchema, env.defaultRetry);
835
+ if (!n.timeout && env.defaultTimeout)
836
+ n.timeout = clone(DurationSchema, env.defaultTimeout);
837
+ if (n.action.case === "pipeline") {
838
+ const ref = n.action.value;
839
+ const st = ref.pipelineId ? this.pipelines.get(ref.pipelineId) : this.pipelines.find(env.tenant, ref.scope ?? env.scope, ref.name);
840
+ const label = ref.pipelineId || ref.name;
841
+ if (!st || st.pipeline.tenant !== env.tenant)
842
+ throw new ConnectError(`the definition is invalid: node "${n.id}": no pipeline ${label} of this tenant`, Code.InvalidArgument);
843
+ const v = this.pipelines.version(st, ref.version);
844
+ if (st.pipeline.latestVersion === 0 || !v)
845
+ throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} has no published version ${ref.version || ""}`.trimEnd(), Code.InvalidArgument);
846
+ if (st.pipeline.state === PipelineState.ARCHIVED)
847
+ throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} is archived`, Code.InvalidArgument);
848
+ if (env.about)
849
+ this.checkChild(n.id, st, env.about);
850
+ if (env.self && (st.pipeline.id === env.self || this.reaches(this.pipelines.snapshot(v.snapshotId).definition, env.self))) {
851
+ throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} reaches this pipeline again, a cycle`, Code.InvalidArgument);
852
+ }
853
+ const child = this.pipelines.snapshot(v.snapshotId).definition;
854
+ for (const o of ref.outputs) {
855
+ if (!child.nodes.some((c) => c.id === o.node))
856
+ throw new ConnectError(`the definition is invalid: node "${n.id}": output ${o.name} names node ${o.node}, which pipeline ${st.pipeline.id} does not have`, Code.InvalidArgument);
857
+ }
858
+ const declared = new Map((v.spec?.variables ?? []).map((x) => [x.name, x]));
859
+ for (const k of Object.keys(n.with)) {
860
+ if (!declared.has(k))
861
+ throw new ConnectError(`the definition is invalid: node "${n.id}": ${JSON.stringify(k)} is not a variable pipeline ${st.pipeline.id} declares`, Code.InvalidArgument);
862
+ }
863
+ for (const x of declared.values()) {
864
+ if (x.required && !x.default && !(x.name in n.with))
865
+ throw new ConnectError(`the definition is invalid: node "${n.id}": variable ${JSON.stringify(x.name)} of pipeline ${st.pipeline.id} is required`, Code.InvalidArgument);
866
+ }
867
+ n.action.value = { ...ref, pipelineId: st.pipeline.id, version: v.version };
868
+ }
869
+ }
870
+ this.compile(out);
871
+ const depth = this.depth(out);
872
+ if (depth > MAX_DEPTH)
873
+ throw new ConnectError(`the definition is invalid: it nests ${depth} deep, and runs nest at most ${MAX_DEPTH} deep counting this one`, Code.InvalidArgument);
874
+ return out;
875
+ }
876
+ /** A frozen definition's nodes, checked: handlers' inputs, the graph, for_each's rules. */
877
+ compile(def) {
878
+ const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
879
+ for (const sp of specs)
880
+ if (sp.handler)
881
+ sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
882
+ checkGraph(specs);
883
+ return specs;
884
+ }
885
+ /** How deep runs of a frozen definition nest through its pinned sub-pipelines, counting its own. */
886
+ depth(def) {
887
+ let most = 0;
888
+ for (const n of def.nodes) {
889
+ if (n.action.case !== "pipeline")
890
+ continue;
891
+ const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
892
+ if (snap)
893
+ most = Math.max(most, this.depth(snap.definition));
894
+ }
895
+ return 1 + most;
896
+ }
897
+ /** Whether a frozen definition reaches pipeline `id` through its pinned sub-pipelines. */
898
+ reaches(def, id) {
899
+ return def.nodes.some((n) => {
900
+ if (n.action.case !== "pipeline")
901
+ return false;
902
+ if (n.action.value.pipelineId === id)
903
+ return true;
904
+ const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
905
+ return snap ? this.reaches(snap.definition, id) : false;
906
+ });
907
+ }
908
+ childSnapshot(pipelineId, version) {
909
+ const st = this.pipelines.get(pipelineId);
910
+ const v = st && this.pipelines.version(st, version);
911
+ return v ? this.pipelines.snapshot(v.snapshotId) : undefined;
912
+ }
913
+ /** A sub-pipeline's child must take runs about the parent's resource, as if started directly. */
914
+ checkChild(nodeId, st, about) {
915
+ if (!takesRunsAbout(st.pipeline, about)) {
916
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": pipeline ${st.pipeline.id} (${st.pipeline.name}) takes runs about its scope or a resource of a kind in its resource_kinds, not ${JSON.stringify(about?.kind ?? "")}`, Code.InvalidArgument);
917
+ }
918
+ }
919
+ /** Every pinned child, at any depth, against the resource a run is about. */
920
+ checkChildren(def, about) {
921
+ for (const n of def.nodes) {
922
+ if (n.action.case !== "pipeline")
923
+ continue;
924
+ const st = this.pipelines.get(n.action.value.pipelineId);
925
+ this.checkChild(n.id, st, about);
926
+ const snap = this.childSnapshot(st.pipeline.id, n.action.value.version);
927
+ if (snap)
928
+ this.checkChildren(snap.definition, about);
929
+ }
930
+ }
931
+ /** Starts a run of a snapshot: an inline definition's, an earlier one by id, or a version's. */
932
+ startSnapshot(who, spec) {
933
+ this.checkChildren(spec.snapshot.definition, spec.resource);
934
+ const specs = this.compile(spec.snapshot.definition);
935
+ const run = this.newRun(who, spec.tenant, spec.key, spec.resource, spec.variables ?? {}, 0, {
936
+ attestedFacts: spec.facts,
937
+ snapshotId: spec.snapshot.id,
938
+ origin: spec.snapshot.origin,
939
+ definition: spec.snapshot.definition,
940
+ ...spec.extra,
941
+ });
942
+ for (const s of specs)
943
+ this.addNode(run, s);
944
+ this.advance(run);
945
+ return run;
946
+ }
947
+ /** Starts a version of a stored pipeline: CreateRun's `pipeline` source, a schedule's target, a sub-pipeline's child. */
948
+ startStored(who, spec) {
949
+ const { version, snapshot, variables } = this.pipelines.resolveStart(spec.stored, spec.version, spec.resource, spec.variables);
950
+ const run = this.startSnapshot(who, {
951
+ ...spec,
952
+ variables,
953
+ snapshot,
954
+ extra: {
955
+ pipelineId: spec.stored.pipeline.id,
956
+ pipelineVersion: version.version,
957
+ maxConcurrentRuns: version.spec?.policy?.maxConcurrentRuns ?? 0,
958
+ trigger: spec.trigger,
959
+ ...(spec.parent ? { parentRunId: spec.parent.runId, parentNodeId: spec.parent.nodeId } : {}),
960
+ },
580
961
  });
962
+ if (spec.schedule) {
963
+ run.scheduleId = spec.schedule.id;
964
+ run.scheduledFor = spec.schedule.scheduledFor;
965
+ run.scheduleTarget = spec.schedule.target;
966
+ }
967
+ return run;
968
+ }
969
+ /** Whether a run of a pipeline must wait for its policy's `max_concurrent_runs`. */
970
+ atCap(run) {
971
+ if (run.maxConcurrentRuns <= 0)
972
+ return false;
973
+ let active = 0;
974
+ for (const r of this.runs.values()) {
975
+ if (r.id !== run.id && r.pipelineId === run.pipelineId && r.state !== RunState.QUEUED && !terminal(r.state))
976
+ active++;
977
+ }
978
+ return active >= run.maxConcurrentRuns;
581
979
  }
582
980
  /** Makes ready what can be, skips what cannot run, and settles the run. */
583
981
  advance(run) {
@@ -585,6 +983,9 @@ export class FakePipelines {
585
983
  return;
586
984
  if (run.notBefore > Date.now() && run.cancelRequestedAt === undefined)
587
985
  return;
986
+ // A stored pipeline's run waits QUEUED under its policy's cap (the tick looks again).
987
+ if (run.state === RunState.QUEUED && run.cancelRequestedAt === undefined && this.atCap(run))
988
+ return;
588
989
  for (let changed = true; changed;) {
589
990
  changed = false;
590
991
  for (const n of run.nodes.values()) {
@@ -604,16 +1005,7 @@ export class FakePipelines {
604
1005
  run.startedAt ??= Date.now();
605
1006
  if (run.state === RunState.QUEUED)
606
1007
  run.state = RunState.RUNNING;
607
- if (n.handler) {
608
- const inputs = this.resolve(run, n);
609
- if (typeof inputs === "string") {
610
- n.state = NodeState.FAILED;
611
- n.failureClass = FailureClass.INVALID_IO;
612
- }
613
- else {
614
- this.offer(run, n, inputs, 0);
615
- }
616
- }
1008
+ this.dispatch(run, n);
617
1009
  changed = true;
618
1010
  }
619
1011
  }
@@ -633,38 +1025,208 @@ export class FakePipelines {
633
1025
  else {
634
1026
  run.state = RunState.SUCCEEDED;
635
1027
  }
1028
+ this.runFinished(run);
636
1029
  }
637
1030
  }
638
- resolve(run, n) {
639
- const out = {};
640
- for (const [k, v] of Object.entries(n.with)) {
1031
+ /**
1032
+ * A node's `with` for one attempt (of one element): whole-value
1033
+ * references, also inside lists and objects, resolved. A string is why
1034
+ * one does not resolve.
1035
+ */
1036
+ resolve(run, n, el) {
1037
+ let failed = "";
1038
+ const walk = (v) => {
1039
+ if (Array.isArray(v))
1040
+ return v.map(walk);
1041
+ if (v !== null && typeof v === "object")
1042
+ return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
641
1043
  const m = typeof v === "string" ? REF.exec(v) : null;
642
- if (!m) {
643
- out[k] = v;
644
- continue;
645
- }
646
- const path = m[1].split(".");
647
- if ((path[0] === "steps" || path[0] === "needs") && path[2] === "outputs" && path[3] === "kv" && path[4]) {
648
- const value = run.nodes.get(path[1])?.kv[path[4]];
649
- if (value === undefined)
650
- return `${m[1]} does not resolve`;
651
- out[k] = value;
1044
+ if (!m)
1045
+ return v;
1046
+ const value = this.lookup(run, m[1], el);
1047
+ if (value === undefined)
1048
+ failed ||= `${m[1]} does not resolve`;
1049
+ return value ?? null;
1050
+ };
1051
+ const out = walk(n.with);
1052
+ return failed || out;
1053
+ }
1054
+ /** One reference's value: an upstream kv output, a variable, or the element. */
1055
+ lookup(run, expr, el) {
1056
+ const path = expr.split(".");
1057
+ if ((path[0] === "steps" || path[0] === "needs") && path[2] === "outputs" && path[3] === "kv" && path[4])
1058
+ return run.nodes.get(path[1])?.kv[path[4]];
1059
+ if (path[0] === "variables" && path[1])
1060
+ return run.variables[path[1]];
1061
+ if (el && path[0] === "item_index" && path.length === 1)
1062
+ return el.index;
1063
+ if (el && path[0] === "item") {
1064
+ let v = el.item;
1065
+ for (const k of path.slice(1))
1066
+ v = v !== null && typeof v === "object" && !Array.isArray(v) ? v[k] : undefined;
1067
+ return v;
1068
+ }
1069
+ return undefined;
1070
+ }
1071
+ /** A node made ready: its attempt (or child run), its elements, or, for an external node, nothing yet. */
1072
+ dispatch(run, n) {
1073
+ if (n.forEach) {
1074
+ const items = this.items(run, n);
1075
+ if (typeof items === "string") {
1076
+ n.state = NodeState.FAILED;
1077
+ n.failureClass = FailureClass.INVALID_IO;
1078
+ return;
652
1079
  }
653
- else if (path[0] === "variables" && path[1] && run.variables[path[1]] !== undefined) {
654
- out[k] = run.variables[path[1]];
1080
+ n.elements = items.map((item, index) => ({ index, item, state: "pending", failureClass: FailureClass.UNSPECIFIED, kv: {} }));
1081
+ n.state = NodeState.RUNNING;
1082
+ this.launchElements(run, n);
1083
+ return;
1084
+ }
1085
+ if (!n.handler && !n.pipeline)
1086
+ return;
1087
+ const inputs = this.resolve(run, n);
1088
+ if (typeof inputs === "string") {
1089
+ n.state = NodeState.FAILED;
1090
+ n.failureClass = FailureClass.INVALID_IO;
1091
+ }
1092
+ else {
1093
+ this.launch(run, n, inputs, undefined, 0);
1094
+ }
1095
+ }
1096
+ /** A for_each node's list: a JSON array, or a reference resolving to one, within `max_items`. */
1097
+ items(run, n) {
1098
+ const fe = n.forEach;
1099
+ const m = typeof fe.items === "string" ? REF.exec(fe.items) : null;
1100
+ const v = m ? this.lookup(run, m[1], undefined) : fe.items;
1101
+ if (!Array.isArray(v))
1102
+ return "for_each.items does not resolve to a list";
1103
+ if (v.length > (fe.maxItems || 1000))
1104
+ return `for_each.items has ${v.length} elements, over max_items`;
1105
+ return v;
1106
+ }
1107
+ /** One attempt of a node (of one element): offered to a worker, or a sub-pipeline's child run started. */
1108
+ launch(run, n, inputs, el, delayMs) {
1109
+ if (n.pipeline)
1110
+ this.startChild(run, n, inputs, el);
1111
+ else
1112
+ this.offer(run, n, inputs, delayMs, el);
1113
+ }
1114
+ /**
1115
+ * Starts elements while `max_parallel` has room, and settles the node
1116
+ * once none is live and none will start: under FAIL_FAST nothing starts
1117
+ * after a failure.
1118
+ */
1119
+ launchElements(run, n) {
1120
+ const fe = n.forEach;
1121
+ const els = n.elements;
1122
+ const policy = fe.onFailure || ForEachFailure.FAIL_FAST;
1123
+ const stopped = () => run.cancelRequestedAt !== undefined || (policy === ForEachFailure.FAIL_FAST && els.some((e) => e.state === "failed"));
1124
+ let live = els.filter((e) => e.state === "running").length;
1125
+ for (const el of els) {
1126
+ if (stopped() || live >= (fe.maxParallel || 10))
1127
+ break;
1128
+ if (el.state !== "pending")
1129
+ continue;
1130
+ const inputs = this.resolve(run, n, el);
1131
+ if (typeof inputs === "string") {
1132
+ el.state = "failed";
1133
+ el.failureClass = FailureClass.INVALID_IO;
1134
+ continue;
655
1135
  }
656
- else {
657
- return `${m[1]} does not resolve`;
1136
+ el.state = "running";
1137
+ live++;
1138
+ this.launch(run, n, inputs, el, 0);
1139
+ }
1140
+ if (live > 0 || (!stopped() && els.some((e) => e.state === "pending")))
1141
+ return;
1142
+ // Settled: outputs as lists in element order, null where an element gave none.
1143
+ const slots = [...new Set(els.flatMap((e) => (e.state === "succeeded" ? Object.keys(e.kv) : [])))];
1144
+ n.kv = Object.fromEntries(slots.map((k) => [k, els.map((e) => (e.state === "succeeded" ? (e.kv[k] ?? null) : null))]));
1145
+ const failed = els.find((e) => e.state === "failed");
1146
+ if (run.cancelRequestedAt !== undefined) {
1147
+ n.state = NodeState.CANCELLED;
1148
+ n.failureClass = FailureClass.CANCELLED;
1149
+ }
1150
+ else if (failed && policy !== ForEachFailure.ALLOW) {
1151
+ n.state = NodeState.FAILED;
1152
+ n.failureClass = failed.failureClass;
1153
+ }
1154
+ else {
1155
+ n.state = NodeState.SUCCEEDED;
1156
+ }
1157
+ }
1158
+ /** A sub-pipeline node's attempt: one child run of the pinned version, about the parent's resource. */
1159
+ startChild(run, n, inputs, el) {
1160
+ const a = this.newAttempt(run, n, inputs, 0, el);
1161
+ a.state = AttemptState.RUNNING;
1162
+ a.leaseExpiresAt = Number.POSITIVE_INFINITY; // held by the platform, not a worker
1163
+ a.startedAt = Date.now();
1164
+ n.attempts.push(a);
1165
+ this.attemptsById.set(a.id, a);
1166
+ if (!n.forEach)
1167
+ n.state = NodeState.RUNNING;
1168
+ try {
1169
+ const child = this.startStored({ tenant: run.tenant, scopes: ALL_SCOPES }, {
1170
+ tenant: run.tenant,
1171
+ key: `parent:${run.id}:${n.id}:${a.id}`,
1172
+ resource: run.resource,
1173
+ variables: inputs,
1174
+ facts: run.attestedFacts,
1175
+ stored: this.pipelines.get(n.pipeline.id),
1176
+ version: n.pipeline.version,
1177
+ trigger: RunTrigger.PARENT,
1178
+ parent: { runId: run.id, nodeId: n.id },
1179
+ });
1180
+ a.childRunId = child.id;
1181
+ }
1182
+ catch (err) {
1183
+ // Its variables, resolved now, refused: the attempt fails, as the child would.
1184
+ a.state = AttemptState.FAILED;
1185
+ a.failureClass = FailureClass.DOWNSTREAM_FAILED;
1186
+ a.message = ConnectError.from(err).rawMessage;
1187
+ a.completedAt = Date.now();
1188
+ queueMicrotask(() => this.settle(a));
1189
+ }
1190
+ }
1191
+ /** A child run ended: its parent's attempt takes its result. */
1192
+ childFinished(child) {
1193
+ const parent = this.runs.get(child.parentRunId);
1194
+ const n = parent?.nodes.get(child.parentNodeId);
1195
+ const a = n?.attempts.find((x) => x.childRunId === child.id);
1196
+ if (!n || !a || a.state !== AttemptState.RUNNING)
1197
+ return;
1198
+ a.completedAt = Date.now();
1199
+ if (child.state === RunState.SUCCEEDED) {
1200
+ a.state = AttemptState.SUCCEEDED;
1201
+ for (const o of n.pipeline.outputs) {
1202
+ const v = child.nodes.get(o.node)?.kv[o.slot];
1203
+ if (v !== undefined)
1204
+ a.kv[o.name] = v;
658
1205
  }
659
1206
  }
660
- return out;
1207
+ else if (child.state === RunState.CANCELLED) {
1208
+ a.state = AttemptState.CANCELLED;
1209
+ a.failureClass = FailureClass.CANCELLED;
1210
+ }
1211
+ else {
1212
+ a.state = AttemptState.FAILED;
1213
+ a.failureClass = FailureClass.DOWNSTREAM_FAILED;
1214
+ a.message = `the child run ${child.id} failed`;
1215
+ }
1216
+ this.settle(a);
1217
+ }
1218
+ offer(run, n, inputs, delayMs, el) {
1219
+ const a = this.newAttempt(run, n, inputs, delayMs, el);
1220
+ n.attempts.push(a);
1221
+ this.attemptsById.set(a.id, a);
1222
+ this.notify();
661
1223
  }
662
- offer(run, n, inputs, delayMs) {
663
- const a = {
1224
+ newAttempt(run, n, inputs, delayMs, el) {
1225
+ return {
664
1226
  id: randomUUID(),
665
1227
  runId: run.id,
666
1228
  nodeId: n.id,
667
- number: n.attempts.length + 1,
1229
+ number: n.attempts.filter((x) => x.itemIndex === el?.index).length + 1,
668
1230
  state: AttemptState.OFFERED,
669
1231
  epoch: 0n,
670
1232
  workerId: "",
@@ -683,14 +1245,23 @@ export class FakePipelines {
683
1245
  completedAt: undefined,
684
1246
  handlerFailureKind: HandlerFailureKind.UNSPECIFIED,
685
1247
  releases: 0,
1248
+ offeredAt: Date.now(),
1249
+ progress: undefined,
1250
+ spans: [],
1251
+ itemIndex: el?.index,
1252
+ item: el?.item,
1253
+ childRunId: "",
686
1254
  };
687
- n.attempts.push(a);
688
- this.attemptsById.set(a.id, a);
689
- this.notify();
690
1255
  }
1256
+ /**
1257
+ * Claims for a worker: the attempts of handler nodes of full runs first,
1258
+ * in creation order, as they always were; then jobs, through their
1259
+ * queues' limits and fairness rule (see ./queues.ts).
1260
+ */
691
1261
  claim(pool, handlers, workerId, max) {
692
1262
  const now = Date.now();
693
1263
  const out = [];
1264
+ const queued = [];
694
1265
  for (const run of this.runs.values()) {
695
1266
  if (run.tenant !== pool.owner?.tenant || run.cancelRequestedAt !== undefined || run.notBefore > now)
696
1267
  continue;
@@ -698,21 +1269,48 @@ export class FakePipelines {
698
1269
  if (!handlers.includes(n.handler))
699
1270
  continue;
700
1271
  for (const a of n.attempts) {
701
- if (out.length >= max)
702
- return out;
703
1272
  if (a.state !== AttemptState.OFFERED || a.notBefore > now)
704
1273
  continue;
705
- a.state = AttemptState.CLAIMED;
706
- a.epoch += 1n;
707
- a.workerId = workerId;
708
- a.poolId = pool.id;
709
- a.leaseExpiresAt = now + this.opts.leaseMs;
710
- out.push(a);
1274
+ if (run.queueId)
1275
+ queued.push({ item: a, queueId: run.queueId, lane: run.fairnessKey, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
1276
+ else if (out.length < max)
1277
+ out.push(a);
711
1278
  }
712
1279
  }
713
1280
  }
1281
+ if (out.length < max && queued.length > 0) {
1282
+ const limits = new Map();
1283
+ for (const q of this.queues.values())
1284
+ limits.set(q.id, queueLimits(q));
1285
+ const picks = pickQueued(queued, limits, { running: (q, t) => this.queueRunning(q, t), claimedSince: (q, since) => (this.queueClaims.get(q) ?? []).filter((x) => x >= since).length }, now, max - out.length);
1286
+ for (const p of picks) {
1287
+ out.push(p.item);
1288
+ this.queueClaims.set(p.queueId, [...(this.queueClaims.get(p.queueId) ?? []).filter((x) => x > now - 86_400_000), now]);
1289
+ }
1290
+ }
1291
+ for (const a of out) {
1292
+ a.state = AttemptState.CLAIMED;
1293
+ a.epoch += 1n;
1294
+ a.workerId = workerId;
1295
+ a.poolId = pool.id;
1296
+ a.leaseExpiresAt = now + this.opts.leaseMs;
1297
+ this.initProgress(a);
1298
+ }
714
1299
  return out;
715
1300
  }
1301
+ /** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
1302
+ queueRunning(queueId, lane) {
1303
+ let n = 0;
1304
+ for (const run of this.runs.values()) {
1305
+ if (run.queueId !== queueId || (lane !== undefined && run.fairnessKey !== lane))
1306
+ continue;
1307
+ for (const node of run.nodes.values())
1308
+ for (const a of node.attempts)
1309
+ if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING)
1310
+ n++;
1311
+ }
1312
+ return n;
1313
+ }
716
1314
  describe(a) {
717
1315
  const run = this.run(a.runId);
718
1316
  const n = run.nodes.get(a.nodeId);
@@ -754,13 +1352,28 @@ export class FakePipelines {
754
1352
  settle(a) {
755
1353
  const run = this.run(a.runId);
756
1354
  const n = run.nodes.get(a.nodeId);
1355
+ // An element's attempt settles its element; the node settles with the last one.
1356
+ const el = a.itemIndex !== undefined ? n.elements?.[a.itemIndex] : undefined;
1357
+ const mine = n.attempts.filter((x) => x.itemIndex === a.itemIndex);
757
1358
  if (a.state === AttemptState.SUCCEEDED) {
758
- n.state = NodeState.SUCCEEDED;
759
- n.kv = a.kv;
1359
+ if (el) {
1360
+ el.state = "succeeded";
1361
+ el.kv = a.kv;
1362
+ }
1363
+ else {
1364
+ n.state = NodeState.SUCCEEDED;
1365
+ n.kv = a.kv;
1366
+ }
760
1367
  }
761
1368
  else if (a.state === AttemptState.CANCELLED || run.cancelRequestedAt !== undefined) {
762
- n.state = NodeState.CANCELLED;
763
- n.failureClass = FailureClass.CANCELLED;
1369
+ if (el) {
1370
+ el.state = "failed";
1371
+ el.failureClass = FailureClass.CANCELLED;
1372
+ }
1373
+ else {
1374
+ n.state = NodeState.CANCELLED;
1375
+ n.failureClass = FailureClass.CANCELLED;
1376
+ }
764
1377
  }
765
1378
  else {
766
1379
  const permanent = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.PERMANENT;
@@ -768,19 +1381,32 @@ export class FakePipelines {
768
1381
  const when = n.retry.when.length > 0 ? n.retry.when : transientHandler ? [...TRANSIENT, FailureClass.HANDLER_ERROR] : TRANSIENT;
769
1382
  // On an interruptible node an interruption is free, up to a bound.
770
1383
  const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
771
- const freeUsed = n.attempts.filter(free).length;
1384
+ const freeUsed = mine.filter(free).length;
772
1385
  if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
773
- this.offer(run, n, a.inputs, 0);
1386
+ // Up to the bound, an interruption is offered again at once.
1387
+ this.launch(run, n, a.inputs, el, 0);
774
1388
  return;
775
1389
  }
776
- const retries = n.attempts.slice(0, -1).filter((x) => !free(x)).length;
777
- if (!permanent && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
778
- this.offer(run, n, a.inputs, this.opts.retryBackoffMs);
1390
+ const retries = mine.slice(0, -1).filter((x) => !free(x)).length;
1391
+ // As the api: a worker that vanished (ABANDONED) may have had outside
1392
+ // effects, so its node runs again automatically only when it declares
1393
+ // itself interruptible.
1394
+ const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
1395
+ if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
1396
+ this.launch(run, n, a.inputs, el, this.opts.retryBackoffMs);
779
1397
  return;
780
1398
  }
781
- n.state = NodeState.FAILED;
782
- n.failureClass = a.failureClass;
1399
+ if (el) {
1400
+ el.state = "failed";
1401
+ el.failureClass = a.failureClass;
1402
+ }
1403
+ else {
1404
+ n.state = NodeState.FAILED;
1405
+ n.failureClass = a.failureClass;
1406
+ }
783
1407
  }
1408
+ if (el)
1409
+ this.launchElements(run, n);
784
1410
  this.advance(run);
785
1411
  this.notify();
786
1412
  }
@@ -790,8 +1416,11 @@ export class FakePipelines {
790
1416
  run.cancelRequestedAt = Date.now();
791
1417
  run.cancelReason = reason;
792
1418
  run.state = RunState.CANCELLING;
1419
+ const children = [];
793
1420
  for (const n of run.nodes.values()) {
794
1421
  for (const a of n.attempts) {
1422
+ if (a.state === AttemptState.RUNNING && a.childRunId)
1423
+ children.push(this.run(a.childRunId));
795
1424
  if (a.state === AttemptState.OFFERED) {
796
1425
  a.state = AttemptState.CANCELLED;
797
1426
  a.failureClass = FailureClass.CANCELLED;
@@ -803,6 +1432,9 @@ export class FakePipelines {
803
1432
  n.state = n.state === NodeState.PENDING ? NodeState.SKIPPED : NodeState.CANCELLED;
804
1433
  }
805
1434
  }
1435
+ // Cancelling a run cancels its sub-pipelines' children; each one's end settles its attempt.
1436
+ for (const c of children)
1437
+ this.cancel(c, reason);
806
1438
  this.advance(run);
807
1439
  this.notify();
808
1440
  }
@@ -836,6 +1468,7 @@ export class FakePipelines {
836
1468
  }
837
1469
  }
838
1470
  }
1471
+ this.schedules.evaluate();
839
1472
  this.notify();
840
1473
  }
841
1474
  notify() {
@@ -843,13 +1476,449 @@ export class FakePipelines {
843
1476
  this.changed = signal();
844
1477
  prev.resolve();
845
1478
  }
1479
+ /** QueuesService, as the contract states it. */
1480
+ queuesService() {
1481
+ const readable = (who, q) => {
1482
+ if (!q || q.owner?.tenant !== who.tenant)
1483
+ throw new ConnectError("no such queue", Code.NotFound);
1484
+ return q;
1485
+ };
1486
+ const byId = (who, id) => readable(who, this.queues.get(id));
1487
+ return {
1488
+ setQueue: (req, ctx) => {
1489
+ const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
1490
+ const q = req.queue;
1491
+ if (!q?.owner?.tenant)
1492
+ throw new ConnectError("queue.owner.tenant: required", Code.InvalidArgument);
1493
+ if (q.owner.tenant !== who.tenant)
1494
+ throw new ConnectError("queue.owner.tenant: the caller is not a service-account member of it", Code.PermissionDenied);
1495
+ if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
1496
+ throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
1497
+ if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
1498
+ throw new ConnectError("max_concurrency_per_tenant must not exceed a non-zero max_concurrency", Code.InvalidArgument);
1499
+ }
1500
+ if (req.updateMask)
1501
+ throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
1502
+ const existing = [...this.queues.values()].find((x) => x.owner?.tenant === q.owner.tenant && x.name === q.name);
1503
+ const now = Date.now();
1504
+ const next = create(QueueSchema, {
1505
+ ...q,
1506
+ id: existing?.id ?? randomUUID(),
1507
+ paused: existing?.paused ?? false,
1508
+ pausedReason: existing?.pausedReason ?? "",
1509
+ pausedBy: existing?.pausedBy ?? "",
1510
+ ...(existing?.pausedAt ? { pausedAt: existing.pausedAt } : {}),
1511
+ createdAt: existing?.createdAt ?? timestampFromMs(now),
1512
+ updatedAt: timestampFromMs(now),
1513
+ });
1514
+ this.queues.set(next.id, next);
1515
+ this.notify();
1516
+ return { queue: next, created: !existing };
1517
+ },
1518
+ getQueue: (req, ctx) => {
1519
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1520
+ if (req.queue.case === "queueId")
1521
+ return { queue: byId(who, req.queue.value) };
1522
+ if (req.queue.case !== "byName")
1523
+ throw new ConnectError("queue: required", Code.InvalidArgument);
1524
+ const { owner, name } = req.queue.value;
1525
+ if (!owner?.tenant || owner.tenant !== who.tenant)
1526
+ throw new ConnectError("no such queue", Code.NotFound);
1527
+ const found = [...this.queues.values()].find((x) => x.owner?.tenant === owner.tenant && x.name === name);
1528
+ if (found)
1529
+ return { queue: readable(who, found) };
1530
+ if (name === "default")
1531
+ return { queue: this.defaultQueue(owner.tenant) };
1532
+ throw new ConnectError("no such queue", Code.NotFound);
1533
+ },
1534
+ listQueues: (req, ctx) => {
1535
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1536
+ if (!req.tenant)
1537
+ throw new ConnectError("tenant: required", Code.InvalidArgument);
1538
+ if (req.tenant !== who.tenant)
1539
+ return { queues: [], nextCursor: "" };
1540
+ const queues = [...this.queues.values()]
1541
+ .filter((q) => q.owner?.tenant === req.tenant)
1542
+ .sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
1543
+ return { queues, nextCursor: "" };
1544
+ },
1545
+ markQueuePaused: (req, ctx) => {
1546
+ const who = this.enter("MarkQueuePaused", ctx, "pipelines:queue:edit");
1547
+ const q = byId(who, req.queueId);
1548
+ if (!q.paused) {
1549
+ q.paused = true;
1550
+ q.pausedAt = timestampFromMs(Date.now());
1551
+ q.pausedBy = `urn:fake:sa:${who.tenant}`;
1552
+ q.pausedReason = req.reason;
1553
+ }
1554
+ return { queue: q };
1555
+ },
1556
+ markQueueResumed: (req, ctx) => {
1557
+ const who = this.enter("MarkQueueResumed", ctx, "pipelines:queue:edit");
1558
+ const q = byId(who, req.queueId);
1559
+ q.paused = false;
1560
+ q.pausedReason = "";
1561
+ q.pausedBy = "";
1562
+ delete q.pausedAt;
1563
+ this.notify();
1564
+ return { queue: q };
1565
+ },
1566
+ deleteQueue: (req, ctx) => {
1567
+ const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
1568
+ const q = byId(who, req.queueId);
1569
+ const waiting = [...this.runs.values()].filter((r) => r.queueId === q.id && (r.state === RunState.QUEUED || (r.state === RunState.RUNNING && this.attemptsOf(r.id).some((a) => a.state === AttemptState.OFFERED))));
1570
+ if (waiting.length > 0 && !req.cancelWaiting)
1571
+ throw new ConnectError(`queue ${q.name} has ${waiting.length} waiting jobs`, Code.FailedPrecondition);
1572
+ for (const r of waiting)
1573
+ this.cancel(r, "queue deleted");
1574
+ if (q.name === "default" && !q.owner?.tenant) {
1575
+ this.queues.set(q.id, create(QueueSchema, { id: q.id, owner: q.owner, name: "default", createdAt: q.createdAt, updatedAt: timestampFromMs(Date.now()) }));
1576
+ }
1577
+ else {
1578
+ this.queues.delete(q.id);
1579
+ }
1580
+ return { cancelledJobs: waiting.length };
1581
+ },
1582
+ getQueueStats: (req, ctx) => {
1583
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1584
+ const q = byId(who, req.queueId);
1585
+ const now = Date.now();
1586
+ const windowMs = req.window ? durationMs(req.window) : 3_600_000;
1587
+ const runs = [...this.runs.values()].filter((r) => r.queueId === q.id);
1588
+ let waiting = 0;
1589
+ let delayed = 0;
1590
+ let running = 0;
1591
+ let started = 0n;
1592
+ let oldest;
1593
+ for (const r of runs) {
1594
+ for (const a of this.attemptsOf(r.id)) {
1595
+ if (a.state === AttemptState.OFFERED) {
1596
+ if (a.notBefore > now || r.notBefore > now)
1597
+ delayed++;
1598
+ else {
1599
+ waiting++;
1600
+ const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
1601
+ oldest = oldest === undefined ? since : Math.min(oldest, since);
1602
+ }
1603
+ }
1604
+ if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
1605
+ running++;
1606
+ }
1607
+ if (a.startedAt && a.startedAt >= now - windowMs)
1608
+ started++;
1609
+ }
1610
+ if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
1611
+ delayed++;
1612
+ }
1613
+ const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
1614
+ return {
1615
+ waiting,
1616
+ delayed,
1617
+ running,
1618
+ ...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
1619
+ started,
1620
+ failed,
1621
+ paused: q.paused,
1622
+ at: timestampFromMs(now),
1623
+ };
1624
+ },
1625
+ listQueueJobs: (req, ctx) => {
1626
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1627
+ const q = byId(who, req.queueId);
1628
+ const runs = [...this.runs.values()]
1629
+ .filter((r) => r.queueId === q.id && (req.states.length === 0 || req.states.includes(r.state)))
1630
+ .reverse();
1631
+ return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
1632
+ },
1633
+ };
1634
+ }
1635
+ // ---- jobs and queues --------------------------------------------------
1636
+ /** CreateJob's work, shared with schedules: resolve the queue, apply its limits, make the one-node run. */
1637
+ createJobRun(who, spec) {
1638
+ this.requireHandler(spec.handler);
1639
+ if (spec.retry && spec.retry.max > 10)
1640
+ throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
1641
+ if (spec.retry?.when.includes(FailureClass.CANCELLED))
1642
+ throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
1643
+ const q = this.resolveQueue(spec.tenant, spec.queue);
1644
+ if (!q)
1645
+ throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among the tenant's queues`, Code.NotFound);
1646
+ const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
1647
+ const payload = this.checkInputs("job", spec.handler, spec.payload);
1648
+ const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
1649
+ run.queueId = q.id;
1650
+ run.queue = q.name;
1651
+ run.fairnessKey = spec.fairnessKey ?? "";
1652
+ run.trigger = spec.trigger ?? RunTrigger.API;
1653
+ if (spec.schedule) {
1654
+ run.scheduleId = spec.schedule.id;
1655
+ run.scheduledFor = spec.schedule.scheduledFor;
1656
+ run.scheduleTarget = spec.schedule.target;
1657
+ }
1658
+ this.addNode(run, {
1659
+ id: "job",
1660
+ handler: spec.handler,
1661
+ uses: "",
1662
+ with: payload,
1663
+ needs: [],
1664
+ retry,
1665
+ timeoutMs,
1666
+ interruptible: spec.interruptible,
1667
+ });
1668
+ this.advance(run);
1669
+ return run;
1670
+ }
1671
+ /** A job's queue: the tenant's of that name; empty is `default`. */
1672
+ resolveQueue(tenant, name) {
1673
+ const want = name || "default";
1674
+ const found = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === want);
1675
+ if (found)
1676
+ return found;
1677
+ return want === "default" ? this.defaultQueue(tenant) : undefined;
1678
+ }
1679
+ /** The tenant's `default` queue, made on first use with no limits. */
1680
+ defaultQueue(tenant) {
1681
+ const existing = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === "default");
1682
+ if (existing)
1683
+ return existing;
1684
+ const now = Date.now();
1685
+ const q = create(QueueSchema, { id: randomUUID(), owner: { tenant }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
1686
+ this.queues.set(q.id, q);
1687
+ return q;
1688
+ }
1689
+ /** RetryNode: reopens a finished run and offers the node again; on a job, a requeue. */
1690
+ retryNode(run, nodeId) {
1691
+ const n = run.nodes.get(nodeId);
1692
+ if (!n)
1693
+ throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
1694
+ if (!terminal(run.state))
1695
+ throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
1696
+ if ((!n.handler && !n.pipeline) || (n.attempts.length === 0 && !n.forEach) || !terminalNode(n.state))
1697
+ throw new ConnectError("the node never settled", Code.FailedPrecondition);
1698
+ run.state = RunState.RUNNING;
1699
+ run.finishedAt = undefined;
1700
+ run.failureClass = FailureClass.UNSPECIFIED;
1701
+ run.failedNodeId = "";
1702
+ run.cancelRequestedAt = undefined;
1703
+ // The node and everything downstream of it run again.
1704
+ const reset = new Set([nodeId]);
1705
+ for (let grew = true; grew;) {
1706
+ grew = false;
1707
+ for (const x of run.nodes.values()) {
1708
+ if (!reset.has(x.id) && x.needs.some((d) => reset.has(d))) {
1709
+ reset.add(x.id);
1710
+ grew = true;
1711
+ }
1712
+ }
1713
+ }
1714
+ for (const id of reset) {
1715
+ const x = run.nodes.get(id);
1716
+ x.state = id === nodeId ? NodeState.READY : NodeState.PENDING;
1717
+ x.failureClass = FailureClass.UNSPECIFIED;
1718
+ x.kv = {};
1719
+ }
1720
+ if (n.forEach && n.elements) {
1721
+ // Only the failed elements run again; the succeeded ones are kept.
1722
+ for (const el of n.elements) {
1723
+ if (el.state === "succeeded")
1724
+ continue;
1725
+ el.state = "pending";
1726
+ el.failureClass = FailureClass.UNSPECIFIED;
1727
+ el.kv = {};
1728
+ }
1729
+ n.state = NodeState.RUNNING;
1730
+ this.launchElements(run, n);
1731
+ this.advance(run);
1732
+ return;
1733
+ }
1734
+ if (n.forEach) {
1735
+ this.dispatch(run, n);
1736
+ this.advance(run);
1737
+ return;
1738
+ }
1739
+ const inputs = this.resolve(run, n);
1740
+ this.launch(run, n, typeof inputs === "string" ? {} : inputs, undefined, 0);
1741
+ }
1742
+ // ---- issues, progress and spans ---------------------------------------
1743
+ /** Every declared stage PENDING from the claim, as the platform does. */
1744
+ initProgress(a) {
1745
+ if (a.progress)
1746
+ return;
1747
+ const n = this.run(a.runId).nodes.get(a.nodeId);
1748
+ const pkg = this.packages.get(n.handler);
1749
+ const manifest = pkg ? [...pkg.versions.values()].at(-1) : undefined;
1750
+ const stages = manifest?.stages ?? [];
1751
+ if (stages.length === 0)
1752
+ return;
1753
+ a.progress = {
1754
+ fraction: 0,
1755
+ message: "",
1756
+ processed: 0n,
1757
+ total: 0n,
1758
+ failed: 0n,
1759
+ updatedAt: Date.now(),
1760
+ reports: 0,
1761
+ stages: stages.map((st) => ({ key: st.key, name: st.name, weight: st.weight || 1, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n })),
1762
+ };
1763
+ }
1764
+ setProgress(a, req) {
1765
+ const now = Date.now();
1766
+ const p = (a.progress ??= { fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n, stages: [], updatedAt: now, reports: 0 });
1767
+ p.reports++;
1768
+ p.updatedAt = now;
1769
+ p.message = req.message;
1770
+ p.processed = req.processed;
1771
+ p.total = req.total;
1772
+ p.failed = req.failed;
1773
+ if (!req.stage) {
1774
+ p.fraction = Math.max(p.fraction, req.fraction);
1775
+ return;
1776
+ }
1777
+ let st = p.stages.find((x) => x.key === req.stage);
1778
+ if (!st) {
1779
+ st = { key: req.stage, name: req.stageName || req.stage, weight: 0, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n };
1780
+ p.stages.push(st);
1781
+ }
1782
+ // Reporting a stage finishes the one before it.
1783
+ for (const other of p.stages) {
1784
+ if (other !== st && other.state === StageState.RUNNING) {
1785
+ other.state = StageState.SUCCEEDED;
1786
+ other.fraction = 1;
1787
+ other.finishedAt = now;
1788
+ }
1789
+ }
1790
+ const state = req.stageState === StageState.UNSPECIFIED ? StageState.RUNNING : req.stageState;
1791
+ if (st.state === StageState.PENDING || st.state === StageState.RUNNING) {
1792
+ st.state = state;
1793
+ st.startedAt ??= now;
1794
+ if (state !== StageState.RUNNING)
1795
+ st.finishedAt = now;
1796
+ }
1797
+ st.fraction = state === StageState.SUCCEEDED ? 1 : req.fraction;
1798
+ st.message = req.message;
1799
+ st.processed = req.processed;
1800
+ st.total = req.total;
1801
+ st.failed = req.failed;
1802
+ const total = p.stages.reduce((s, x) => s + x.weight, 0);
1803
+ if (total > 0) {
1804
+ const done = p.stages.reduce((s, x) => s + (x.state === StageState.SUCCEEDED || x.state === StageState.FAILED || x.state === StageState.SKIPPED ? x.weight : x.state === StageState.RUNNING ? x.weight * x.fraction : 0), 0);
1805
+ p.fraction = Math.max(p.fraction, Math.min(1, done / total));
1806
+ }
1807
+ }
1808
+ appendIssues(a, issues) {
1809
+ const run = this.run(a.runId);
1810
+ let accepted = 0;
1811
+ for (const i of issues) {
1812
+ if (!i.message)
1813
+ throw new ConnectError("issues.message: required", Code.InvalidArgument);
1814
+ if (i.severity === 0)
1815
+ throw new ConnectError("issues.severity: required", Code.InvalidArgument);
1816
+ }
1817
+ for (const i of issues) {
1818
+ if (run.issues.length >= this.opts.issueCap) {
1819
+ run.issuesTruncated = true;
1820
+ continue;
1821
+ }
1822
+ run.issues.push({
1823
+ id: randomUUID(),
1824
+ runId: run.id,
1825
+ nodeId: a.nodeId,
1826
+ attemptId: a.id,
1827
+ stage: i.stage,
1828
+ severity: i.severity,
1829
+ subject: i.subject || i.subjectId,
1830
+ subjectKind: i.subjectKind,
1831
+ subjectId: i.subjectId,
1832
+ message: i.message,
1833
+ detail: i.detail,
1834
+ createdAt: Date.now(),
1835
+ });
1836
+ accepted++;
1837
+ }
1838
+ return accepted;
1839
+ }
1840
+ /** Issues of a run, filtered as ListIssues filters them. */
1841
+ issuesOf(runId, filter = {}) {
1842
+ return this.run(runId).issues.filter((i) => (!filter.nodeId || i.nodeId === filter.nodeId) && (!filter.attemptId || i.attemptId === filter.attemptId) && (!filter.minSeverity || i.severity >= filter.minSeverity));
1843
+ }
1844
+ /** The run's trace: the platform's spans for the run, its nodes and attempts, and the step spans reported. */
1845
+ traceOf(runId, filter = {}) {
1846
+ const run = this.run(runId);
1847
+ const traceId = run.id.replaceAll("-", "");
1848
+ const sid = (s) => s.replaceAll("-", "").slice(0, 16);
1849
+ const out = [
1850
+ create(TraceSpanSchema, {
1851
+ traceId,
1852
+ spanId: sid(run.id),
1853
+ name: `run ${run.id}`,
1854
+ source: TraceSpanSource.PLATFORM,
1855
+ startedAt: timestampFromMs(run.createdAt),
1856
+ ...(run.finishedAt ? { endedAt: timestampFromMs(run.finishedAt) } : {}),
1857
+ }),
1858
+ ];
1859
+ for (const n of run.nodes.values()) {
1860
+ if (filter.nodeId && n.id !== filter.nodeId)
1861
+ continue;
1862
+ const nodeSpan = sid(randomUUIDFrom(run.id + n.id));
1863
+ const first = n.attempts[0];
1864
+ out.push(create(TraceSpanSchema, {
1865
+ traceId,
1866
+ spanId: nodeSpan,
1867
+ parentSpanId: sid(run.id),
1868
+ name: `node ${n.id}`,
1869
+ source: TraceSpanSource.PLATFORM,
1870
+ nodeId: n.id,
1871
+ startedAt: timestampFromMs(first?.offeredAt ?? run.createdAt),
1872
+ }));
1873
+ for (const a of n.attempts) {
1874
+ if (filter.attemptId && a.id !== filter.attemptId)
1875
+ continue;
1876
+ out.push(create(TraceSpanSchema, {
1877
+ traceId,
1878
+ spanId: sid(a.id),
1879
+ parentSpanId: nodeSpan,
1880
+ name: `attempt ${a.number}`,
1881
+ source: TraceSpanSource.PLATFORM,
1882
+ nodeId: n.id,
1883
+ attemptId: a.id,
1884
+ startedAt: timestampFromMs(a.offeredAt),
1885
+ ...(a.completedAt ? { endedAt: timestampFromMs(a.completedAt) } : {}),
1886
+ }));
1887
+ for (const sp of a.spans) {
1888
+ out.push(create(TraceSpanSchema, {
1889
+ traceId: sp.traceId,
1890
+ spanId: sp.spanId,
1891
+ parentSpanId: sp.parentSpanId || sid(a.id),
1892
+ name: sp.name,
1893
+ source: TraceSpanSource.STEP,
1894
+ nodeId: n.id,
1895
+ attemptId: a.id,
1896
+ ...(sp.startedAt ? { startedAt: sp.startedAt } : {}),
1897
+ ...(sp.endedAt ? { endedAt: sp.endedAt } : {}),
1898
+ status: sp.status,
1899
+ statusMessage: sp.statusMessage,
1900
+ ...(sp.attributes ? { attributes: sp.attributes } : {}),
1901
+ }));
1902
+ }
1903
+ }
1904
+ }
1905
+ return out.sort((x, y) => Number(x.startedAt?.seconds ?? 0n) - Number(y.startedAt?.seconds ?? 0n));
1906
+ }
1907
+ /** Runs whenever a run becomes terminal. */
1908
+ runFinished(run) {
1909
+ // Deferred, so a parent never settles inside the child's own advance.
1910
+ if (run.parentRunId)
1911
+ queueMicrotask(() => this.childFinished(run));
1912
+ // The schedules are made after the router; a run cannot finish before.
1913
+ this.schedules?.runFinished(run);
1914
+ }
846
1915
  // ---- messages ---------------------------------------------------------
847
1916
  runMessage(r) {
848
1917
  return create(RunSchema, {
849
1918
  id: r.id,
850
1919
  tenant: r.tenant,
851
- product: r.product,
852
1920
  idempotencyKey: r.idempotencyKey,
1921
+ fairnessKey: r.fairnessKey,
853
1922
  ...(r.resource ? { resource: r.resource } : {}),
854
1923
  variables: r.variables,
855
1924
  state: r.state,
@@ -862,6 +1931,23 @@ export class FakePipelines {
862
1931
  ...(r.finishedAt ? { finishedAt: timestampFromMs(r.finishedAt) } : {}),
863
1932
  ...(r.cancelRequestedAt ? { cancelRequestedAt: timestampFromMs(r.cancelRequestedAt) } : {}),
864
1933
  cancelReason: r.cancelReason,
1934
+ queueId: r.queueId,
1935
+ queue: r.queue,
1936
+ trigger: r.trigger,
1937
+ scheduleId: r.scheduleId,
1938
+ ...(r.scheduledFor !== undefined ? { scheduledFor: timestampFromMs(r.scheduledFor) } : {}),
1939
+ scheduleTarget: r.scheduleTarget,
1940
+ issueInfoCount: r.issues.filter((i) => i.severity === IssueSeverity.INFO).length,
1941
+ issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
1942
+ issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
1943
+ issuesTruncated: r.issuesTruncated,
1944
+ ...(r.attestedFacts ? { attestedFacts: r.attestedFacts } : {}),
1945
+ snapshotId: r.snapshotId,
1946
+ origin: r.origin,
1947
+ pipelineId: r.pipelineId,
1948
+ pipelineVersion: r.pipelineVersion,
1949
+ parentRunId: r.parentRunId,
1950
+ parentNodeId: r.parentNodeId,
865
1951
  });
866
1952
  }
867
1953
  nodeMessage(n) {
@@ -872,6 +1958,18 @@ export class FakePipelines {
872
1958
  failureClass: n.failureClass,
873
1959
  settledAttemptId: terminalNode(n.state) ? (settled?.id ?? "") : "",
874
1960
  kv: Object.fromEntries(Object.entries(n.kv).map(([k, v]) => [k, jsonValue(v)])),
1961
+ downstreamRunId: n.attempts.findLast((a) => a.childRunId)?.childRunId ?? "",
1962
+ ...(n.elements
1963
+ ? {
1964
+ forEach: {
1965
+ total: n.elements.length,
1966
+ succeeded: n.elements.filter((e) => e.state === "succeeded").length,
1967
+ failed: n.elements.filter((e) => e.state === "failed").length,
1968
+ running: n.elements.filter((e) => e.state === "running").length,
1969
+ pending: n.elements.filter((e) => e.state === "pending").length,
1970
+ },
1971
+ }
1972
+ : {}),
875
1973
  attempts: n.attempts.map((a) => create(AttemptSchema, {
876
1974
  id: a.id,
877
1975
  runId: a.runId,
@@ -884,16 +1982,54 @@ export class FakePipelines {
884
1982
  termination: a.termination,
885
1983
  ...(a.exitCode !== undefined ? { exitCode: a.exitCode } : {}),
886
1984
  message: a.message,
1985
+ childRunId: a.childRunId,
1986
+ ...(a.itemIndex !== undefined ? { itemIndex: a.itemIndex, item: jsonValue(a.item ?? null) } : {}),
887
1987
  logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
888
1988
  logTruncated: a.logTruncated,
1989
+ ...(a.progress
1990
+ ? {
1991
+ progress: {
1992
+ fraction: a.progress.fraction,
1993
+ message: a.progress.message,
1994
+ processed: a.progress.processed,
1995
+ total: a.progress.total,
1996
+ failed: a.progress.failed,
1997
+ updatedAt: timestampFromMs(a.progress.updatedAt),
1998
+ stages: a.progress.stages.map((st) => ({
1999
+ key: st.key,
2000
+ name: st.name,
2001
+ weight: st.weight,
2002
+ state: st.state,
2003
+ fraction: st.fraction,
2004
+ message: st.message,
2005
+ processed: st.processed,
2006
+ total: st.total,
2007
+ failed: st.failed,
2008
+ ...(st.startedAt ? { startedAt: timestampFromMs(st.startedAt) } : {}),
2009
+ ...(st.finishedAt ? { finishedAt: timestampFromMs(st.finishedAt) } : {}),
2010
+ })),
2011
+ },
2012
+ }
2013
+ : {}),
889
2014
  })),
890
2015
  });
891
2016
  }
892
2017
  }
2018
+ /** Every whole-value reference in a value, also inside lists and objects. */
2019
+ function refs(v) {
2020
+ if (Array.isArray(v))
2021
+ return v.flatMap(refs);
2022
+ if (v !== null && typeof v === "object")
2023
+ return Object.values(v).flatMap(refs);
2024
+ const m = typeof v === "string" ? REF.exec(v) : null;
2025
+ return m ? [m[1]] : [];
2026
+ }
2027
+ const ITEM_REF = /^item(_index$|$|\.)/;
893
2028
  /**
894
2029
  * Holds a definition to the server's rules the fake models: unique ids,
895
- * known needs, no cycle, and a reference names only a node in the referring
896
- * node's dependency closure (a reference adds no edge).
2030
+ * known needs, no cycle, a reference names only a node in the referring
2031
+ * node's dependency closure (a reference adds no edge), and `item` is read
2032
+ * only in a for_each node's `with`.
897
2033
  */
898
2034
  function checkGraph(specs) {
899
2035
  const byId = new Map();
@@ -917,18 +2053,22 @@ function checkGraph(specs) {
917
2053
  };
918
2054
  for (const s of specs) {
919
2055
  const reachable = closure(s.id);
920
- for (const v of Object.values(s.with)) {
921
- const m = typeof v === "string" ? REF.exec(v) : null;
922
- const node = m?.[1]?.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
923
- if (node && !reachable.has(node)) {
924
- throw new ConnectError(`node ${s.id}: ${m[1]} names a node outside its needs`, Code.InvalidArgument);
925
- }
2056
+ const items = s.forEach ? refs(s.forEach.items) : [];
2057
+ for (const r of [...refs(s.with), ...items]) {
2058
+ const node = r.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
2059
+ if (node && !reachable.has(node))
2060
+ throw new ConnectError(`node ${s.id}: ${r} names a node outside its needs`, Code.InvalidArgument);
926
2061
  }
2062
+ if (!s.forEach && refs(s.with).some((r) => ITEM_REF.test(r)))
2063
+ throw new ConnectError(`node ${s.id}: item is read only in a for_each node`, Code.InvalidArgument);
2064
+ if (items.some((r) => ITEM_REF.test(r)))
2065
+ throw new ConnectError(`node ${s.id}: for_each.items cannot read item`, Code.InvalidArgument);
927
2066
  }
928
2067
  }
929
2068
  function nodeSpec(n, requireHandler) {
930
2069
  let handler = "";
931
2070
  let uses = "";
2071
+ let pipeline;
932
2072
  if (n.action.case === "handler") {
933
2073
  handler = n.action.value;
934
2074
  requireHandler(handler);
@@ -936,8 +2076,31 @@ function nodeSpec(n, requireHandler) {
936
2076
  else if (n.action.case === "uses") {
937
2077
  uses = `${n.action.value.name}@${n.action.value.version}`;
938
2078
  }
2079
+ else if (n.action.case === "pipeline") {
2080
+ // Pinned by freeze() before a node is compiled.
2081
+ const p = n.action.value;
2082
+ pipeline = { id: p.pipelineId, version: p.version, outputs: p.outputs.map((o) => ({ name: o.name, node: o.node, slot: o.slot })) };
2083
+ }
939
2084
  else {
940
- throw new ConnectError(`node ${n.id}: only handler and uses nodes`, Code.Unimplemented);
2085
+ throw new ConnectError(`node ${n.id}: only handler, uses and pipeline nodes`, Code.Unimplemented);
2086
+ }
2087
+ let forEach;
2088
+ if (n.forEach) {
2089
+ if (n.manual)
2090
+ throw new ConnectError(`node ${n.id}: for_each is not for a manual node`, Code.InvalidArgument);
2091
+ if (uses)
2092
+ throw new ConnectError(`node ${n.id}: the fake runs for_each on handler and pipeline nodes only`, Code.Unimplemented);
2093
+ const fe = n.forEach;
2094
+ if (fe.maxItems > 10_000)
2095
+ throw new ConnectError(`node ${n.id}: for_each.max_items is at most 10000`, Code.InvalidArgument);
2096
+ const items = fe.items ? toJson(ValueSchema, fe.items) : null;
2097
+ if (typeof items === "string" ? !REF.test(items) : !Array.isArray(items)) {
2098
+ throw new ConnectError(`node ${n.id}: for_each.items is a list or a whole-value reference to one`, Code.InvalidArgument);
2099
+ }
2100
+ if (Array.isArray(items) && items.length > (fe.maxItems || 1000)) {
2101
+ throw new ConnectError(`node ${n.id}: for_each.items has ${items.length} elements, over max_items ${fe.maxItems || 1000}`, Code.InvalidArgument);
2102
+ }
2103
+ forEach = { items, maxItems: fe.maxItems, maxParallel: fe.maxParallel, onFailure: fe.onFailure };
941
2104
  }
942
2105
  return {
943
2106
  id: n.id,
@@ -948,6 +2111,8 @@ function nodeSpec(n, requireHandler) {
948
2111
  retry: n.retry ? { max: n.retry.max, when: n.retry.when } : undefined,
949
2112
  timeoutMs: n.timeout ? durationMs(n.timeout) : 0,
950
2113
  interruptible: n.interruptible,
2114
+ ...(pipeline ? { pipeline } : {}),
2115
+ ...(forEach ? { forEach } : {}),
951
2116
  };
952
2117
  }
953
2118
  /** The server's dispatch.Outcome. */
@@ -1012,4 +2177,40 @@ function aborted(s) {
1012
2177
  s.addEventListener("abort", () => r(), { once: true });
1013
2178
  });
1014
2179
  }
2180
+ /** A queue's limits as the selection reads them. */
2181
+ function queueLimits(q) {
2182
+ return {
2183
+ id: q.id,
2184
+ priority: q.priority || 1,
2185
+ paused: q.paused,
2186
+ maxConcurrency: q.maxConcurrency,
2187
+ maxConcurrencyPerTenant: q.maxConcurrencyPerTenant,
2188
+ ...(q.rateLimit?.window ? { rateLimit: { limit: q.rateLimit.limit, windowMs: durationMs(q.rateLimit.window) } } : {}),
2189
+ };
2190
+ }
2191
+ /**
2192
+ * A job's retry and timeout under its queue: the queue's defaults when the
2193
+ * job gives none, and its ceilings, past which the job is refused.
2194
+ */
2195
+ function jobLimits(q, retry, timeoutMs) {
2196
+ const ceiling = q.defaultRetry;
2197
+ if (retry && ceiling) {
2198
+ if (retry.max > ceiling.max)
2199
+ throw new ConnectError(`retry.max ${retry.max} is above queue ${q.name}'s ${ceiling.max}`, Code.InvalidArgument);
2200
+ const allowed = ceiling.when.length > 0 ? ceiling.when : TRANSIENT;
2201
+ const outside = retry.when.filter((c) => !allowed.includes(c));
2202
+ if (outside.length > 0)
2203
+ throw new ConnectError(`retry.when lists ${outside.map((c) => FailureClass[c]).join(", ")}, which queue ${q.name} does not retry`, Code.InvalidArgument);
2204
+ }
2205
+ const maxTimeout = q.defaultTimeout ? durationMs(q.defaultTimeout) : 0;
2206
+ if (timeoutMs > 0 && maxTimeout > 0 && timeoutMs > maxTimeout) {
2207
+ throw new ConnectError(`timeout ${timeoutMs} ms is above queue ${q.name}'s ${maxTimeout} ms`, Code.InvalidArgument);
2208
+ }
2209
+ const r = retry ?? ceiling;
2210
+ return { retry: r ? { max: r.max, when: r.when } : undefined, timeoutMs: timeoutMs || maxTimeout };
2211
+ }
2212
+ /** A stable UUID-shaped id from a string, for the fake's platform spans. */
2213
+ function randomUUIDFrom(s) {
2214
+ return createHash("sha256").update(s).digest("hex").slice(0, 32);
2215
+ }
1015
2216
  //# sourceMappingURL=index.js.map