@xeonr/pipelines-sdk 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +86 -33
  2. package/dist/auth.d.ts +1 -1
  3. package/dist/client.d.ts +14 -10
  4. package/dist/client.d.ts.map +1 -1
  5. package/dist/client.js +5 -4
  6. package/dist/client.js.map +1 -1
  7. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +6 -4
  8. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
  9. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
  10. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +37 -6
  11. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
  12. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +6 -5
  13. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
  14. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +60 -6
  15. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
  16. package/dist/gen/xeonr/pipelines/v1/common_pb.js +58 -3
  17. package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
  18. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +37 -7
  19. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
  20. package/dist/gen/xeonr/pipelines/v1/definition_pb.js +1 -1
  21. package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
  22. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +22 -50
  23. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -1
  24. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +10 -9
  25. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -1
  26. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +103 -8
  27. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
  28. package/dist/gen/xeonr/pipelines/v1/pools_pb.js +32 -24
  29. package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
  30. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +40 -98
  31. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -1
  32. package/dist/gen/xeonr/pipelines/v1/queues_pb.js +46 -51
  33. package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -1
  34. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +36 -27
  35. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
  36. package/dist/gen/xeonr/pipelines/v1/registry_pb.js +16 -18
  37. package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
  38. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +372 -57
  39. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
  40. package/dist/gen/xeonr/pipelines/v1/runs_pb.js +205 -37
  41. package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
  42. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts +194 -0
  43. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts.map +1 -0
  44. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js +100 -0
  45. package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js.map +1 -0
  46. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +138 -62
  47. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -1
  48. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +89 -49
  49. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -1
  50. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +32 -5
  51. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
  52. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +12 -3
  53. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
  54. package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts +270 -27
  55. package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts.map +1 -1
  56. package/dist/gen/xeonr/pipelines/v1/sinks_pb.js +163 -24
  57. package/dist/gen/xeonr/pipelines/v1/sinks_pb.js.map +1 -1
  58. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts +391 -0
  59. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts.map +1 -0
  60. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js +100 -0
  61. package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js.map +1 -0
  62. package/dist/gen/xeonr/platform/v1/options_pb.d.ts +14 -0
  63. package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
  64. package/dist/gen/xeonr/platform/v1/options_pb.js +18 -4
  65. package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
  66. package/dist/handler.d.ts +1 -1
  67. package/dist/index.d.ts +3 -3
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +3 -3
  70. package/dist/index.js.map +1 -1
  71. package/dist/queues.d.ts +15 -28
  72. package/dist/queues.d.ts.map +1 -1
  73. package/dist/queues.js +15 -33
  74. package/dist/queues.js.map +1 -1
  75. package/dist/reporter.d.ts +1 -1
  76. package/dist/reporter.js +4 -4
  77. package/dist/reporter.js.map +1 -1
  78. package/dist/schedules.d.ts +9 -17
  79. package/dist/schedules.d.ts.map +1 -1
  80. package/dist/schedules.js +11 -12
  81. package/dist/schedules.js.map +1 -1
  82. package/dist/testing/index.d.ts +111 -14
  83. package/dist/testing/index.d.ts.map +1 -1
  84. package/dist/testing/index.js +638 -130
  85. package/dist/testing/index.js.map +1 -1
  86. package/dist/testing/pipelines.d.ts +99 -0
  87. package/dist/testing/pipelines.d.ts.map +1 -0
  88. package/dist/testing/pipelines.js +445 -0
  89. package/dist/testing/pipelines.js.map +1 -0
  90. package/dist/testing/queues.d.ts +13 -10
  91. package/dist/testing/queues.d.ts.map +1 -1
  92. package/dist/testing/queues.js +15 -13
  93. package/dist/testing/queues.js.map +1 -1
  94. package/dist/testing/schedules.d.ts +20 -8
  95. package/dist/testing/schedules.d.ts.map +1 -1
  96. package/dist/testing/schedules.js +46 -43
  97. package/dist/testing/schedules.js.map +1 -1
  98. package/dist/worker.d.ts +8 -14
  99. package/dist/worker.d.ts.map +1 -1
  100. package/dist/worker.js +8 -19
  101. package/dist/worker.js.map +1 -1
  102. package/package.json +1 -1
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * @xeonr/pipelines-sdk/testing: an in-memory fake of pipelines-api's worker
3
- * and jobs surface, so a product can unit-test its handlers, and the code
3
+ * and jobs surface, so a service can unit-test its handlers, and the code
4
4
  * that enqueues them, with no server.
5
5
  *
6
6
  * It is a real Connect router behind an in-memory transport, so the SDK's
@@ -31,26 +31,45 @@
31
31
  * `handler_failure_kind`, retries by class, free interruptions on an
32
32
  * `interruptible` node, cancel, and error injection.
33
33
  *
34
+ * Stored pipelines (`PipelinesService`, see ./pipelines.ts) and the runs of
35
+ * their versions (`CreateRun` with `pipeline` or `snapshot_id`, and a
36
+ * schedule's pipeline target), with the policy's node defaults and
37
+ * `max_concurrent_runs`. Sub-pipeline nodes: pinned to a version when the
38
+ * snapshot is made (cycles and nesting over 3 refused), one child run per
39
+ * attempt about the parent's resource, the child's outputs mapped back, a
40
+ * failed child failing the attempt as `DOWNSTREAM_FAILED`, a cancel passed
41
+ * down. `for_each` nodes on handler and sub-pipeline nodes: the list from a
42
+ * literal, a variable or an upstream kv output, `${{ item }}`,
43
+ * `${{ item.<path> }}` and `${{ item_index }}`, `max_items`,
44
+ * `max_parallel`, each element retried on its own, the three `on_failure`
45
+ * policies, outputs aggregated in element order, and RetryNode running only
46
+ * the failed elements again.
47
+ *
34
48
  * What it does not: secrets, blobs, tiers, routing between pools, the
35
- * outbox, and the platform's own deadline enforcement.
49
+ * outbox, exclusions, `for_each` on an external (image) node, and the
50
+ * platform's own deadline enforcement.
36
51
  */
37
52
  import { createHash, randomUUID } from "node:crypto";
38
- import { create, fromJson, toJson } from "@bufbuild/protobuf";
39
- import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
53
+ import { clone, create, fromJson, toJson } from "@bufbuild/protobuf";
54
+ import { DurationSchema, durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
40
55
  import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
41
56
  import { tokenSource } from "../auth.js";
42
57
  import { authInterceptor } from "../transport.js";
43
58
  import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
59
+ import { DefinitionSchema, ForEachFailure, RetrySchema } from "../gen/xeonr/pipelines/v1/definition_pb.js";
60
+ import { PipelineState, PipelinesService } from "../gen/xeonr/pipelines/v1/pipelines_pb.js";
44
61
  import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
45
62
  import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
46
- import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
63
+ import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunOrigin, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
47
64
  import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
65
+ import { FakeStoredPipelines, takesRunsAbout } from "./pipelines.js";
48
66
  import { pickQueued } from "./queues.js";
49
67
  import { FakeClock, FakeSchedules } from "./schedules.js";
50
68
  import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
51
69
  export { FakeClock, FakeSchedules } from "./schedules.js";
52
70
  export { parseCron, nextOccurrence } from "./cron.js";
53
71
  export { pickQueued } from "./queues.js";
72
+ export { FakeStoredPipelines, resolveVariables } from "./pipelines.js";
54
73
  import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
55
74
  /** The tenant the default account acts for. */
56
75
  export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
@@ -69,9 +88,9 @@ export const ALL_SCOPES = [
69
88
  "pipelines:queue:edit",
70
89
  "pipelines:schedule:read",
71
90
  "pipelines:schedule:edit",
91
+ "pipelines:pipeline:read",
92
+ "pipelines:pipeline:edit",
72
93
  ];
73
- /** The product the default account belongs to. */
74
- export const FAKE_PRODUCT = "00000000-0000-4000-8000-0000000000a1";
75
94
  const TRANSIENT = [
76
95
  FailureClass.ABANDONED,
77
96
  FailureClass.RUNNER_CRASHED,
@@ -84,6 +103,8 @@ const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
84
103
  /** How many interruptions an interruptible node gets without spending a retry (the api's default). */
85
104
  const FREE_INTERRUPTIONS = 3;
86
105
  const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
106
+ /** How deeply runs nest through sub-pipelines, counting the outermost (the api's MaxDepth). */
107
+ const MAX_DEPTH = 3;
87
108
  /**
88
109
  * An in-memory pipelines-api for tests. Call {@link close} when done (it
89
110
  * stops the timer that looks at leases and delays).
@@ -110,6 +131,8 @@ export class FakePipelines {
110
131
  clock = new FakeClock();
111
132
  /** The schedules, with their recorded occurrences. */
112
133
  schedules;
134
+ /** Stored pipelines, their versions and the snapshots runs execute. */
135
+ pipelines;
113
136
  queueClaims = new Map();
114
137
  timer;
115
138
  changed;
@@ -129,8 +152,12 @@ export class FakePipelines {
129
152
  spanCap: opts.spanCap ?? 10_000,
130
153
  };
131
154
  this.changed = signal();
155
+ this.pipelines = new FakeStoredPipelines({
156
+ enter: (method, ctx, scope) => this.enter(method, ctx, scope),
157
+ freeze: (def, env) => this.freeze(def, env),
158
+ });
132
159
  this.schedules = new FakeSchedules({
133
- createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES, product: owner.product }, {
160
+ createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
134
161
  tenant: owner.tenant,
135
162
  handler: spec.handler,
136
163
  key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
@@ -144,10 +171,31 @@ export class FakePipelines {
144
171
  trigger: spec.trigger,
145
172
  schedule: spec.schedule,
146
173
  }),
174
+ startPipeline: (owner, spec) => {
175
+ const st = this.pipelines.get(spec.target.pipelineId);
176
+ if (!st || st.pipeline.tenant !== owner.tenant)
177
+ throw new ConnectError("no such pipeline", Code.NotFound);
178
+ return this.startStored({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
179
+ tenant: owner.tenant,
180
+ key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
181
+ resource: spec.resource ?? (st.pipeline.scope ? { kind: st.pipeline.scope.kind, id: st.pipeline.scope.id } : undefined),
182
+ variables: spec.variables,
183
+ facts: spec.target.attestedFacts,
184
+ stored: st,
185
+ version: spec.target.version,
186
+ trigger: spec.trigger,
187
+ schedule: spec.schedule,
188
+ });
189
+ },
190
+ checkPipeline: (tenant, id) => {
191
+ const st = this.pipelines.get(id);
192
+ if (!st || st.pipeline.tenant !== tenant)
193
+ throw new ConnectError(`schedule.target.pipeline_id: no pipeline ${id} of the tenant`, Code.NotFound);
194
+ },
147
195
  runs: () => this.runs.values(),
148
196
  runMessage: (id) => this.runMessage(this.run(id)),
149
- checkHandler: (_product, handler) => this.requireHandler(handler),
150
- }, this.clock, FAKE_PRODUCT);
197
+ checkHandler: (handler) => this.requireHandler(handler),
198
+ }, this.clock);
151
199
  this.timer = setInterval(() => this.tick(), this.opts.tickMs);
152
200
  this.timer.unref();
153
201
  }
@@ -252,7 +300,7 @@ export class FakePipelines {
252
300
  completeNode(runId, nodeId, result = {}) {
253
301
  const run = this.run(runId);
254
302
  const node = run.nodes.get(nodeId);
255
- if (!node || node.handler)
303
+ if (!node || node.handler || node.pipeline)
256
304
  throw new Error(`FakePipelines.completeNode: ${nodeId} is not an external node of ${runId}`);
257
305
  if (node.state !== NodeState.READY)
258
306
  throw new Error(`FakePipelines.completeNode: ${nodeId} is not ready`);
@@ -459,6 +507,7 @@ export class FakePipelines {
459
507
  });
460
508
  router.service(QueuesService, this.queuesService());
461
509
  router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
510
+ router.service(PipelinesService, this.pipelines.service());
462
511
  router.service(RunsService, {
463
512
  listIssues: (req, ctx) => {
464
513
  const who = this.enter("GetRun", ctx, "pipelines:run:read");
@@ -489,10 +538,11 @@ export class FakePipelines {
489
538
  createJob: (req, ctx) => {
490
539
  const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
491
540
  this.actsFor(who, req.tenant);
492
- const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
541
+ const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
493
542
  if (existing)
494
543
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
495
544
  const run = this.createJobRun(who, {
545
+ fairnessKey: req.fairnessKey,
496
546
  tenant: req.tenant,
497
547
  handler: req.handler,
498
548
  key: req.idempotencyKey,
@@ -515,23 +565,43 @@ export class FakePipelines {
515
565
  createRun: (req, ctx) => {
516
566
  const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
517
567
  this.actsFor(who, req.tenant);
518
- const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
568
+ const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
519
569
  if (existing)
520
570
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
521
- if (req.source.case !== "inlineDefinition")
522
- throw new ConnectError("only inline definitions", Code.Unimplemented);
523
- const def = req.source.value;
524
- if (def.nodes.length === 0)
525
- throw new ConnectError("definition: no nodes", Code.InvalidArgument);
526
- const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
527
- for (const sp of specs)
528
- if (sp.handler)
529
- sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
530
- checkGraph(specs);
531
- const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, req.variables ?? {}, 0);
532
- for (const spec of specs)
533
- this.addNode(run, spec);
534
- this.advance(run);
571
+ const start = { tenant: req.tenant, key: req.idempotencyKey, resource: req.resource, variables: req.variables, facts: req.attestedFacts };
572
+ let run;
573
+ switch (req.source.case) {
574
+ case "inlineDefinition": {
575
+ const about = req.resource ? { kind: req.resource.kind, id: req.resource.id } : undefined;
576
+ const def = this.freeze(req.source.value, { tenant: req.tenant, scope: about, about });
577
+ run = this.startSnapshot(who, { ...start, snapshot: this.pipelines.inlineSnapshot(req.tenant, def) });
578
+ break;
579
+ }
580
+ case "snapshotId": {
581
+ const snap = this.pipelines.snapshot(req.source.value);
582
+ if (!snap || snap.tenant !== req.tenant)
583
+ throw new ConnectError("no such snapshot", Code.NotFound);
584
+ run = this.startSnapshot(who, { ...start, snapshot: snap });
585
+ break;
586
+ }
587
+ case "pipeline": {
588
+ const src = req.source.value;
589
+ const sel = src.pipeline;
590
+ const st = sel.case === "pipelineId"
591
+ ? this.pipelines.get(sel.value)
592
+ : sel.case === "byName" && sel.value.tenant === req.tenant
593
+ ? this.pipelines.find(sel.value.tenant, sel.value.scope, sel.value.name)
594
+ : undefined;
595
+ if (!st || st.pipeline.tenant !== req.tenant)
596
+ throw new ConnectError("no such pipeline", Code.NotFound);
597
+ if (!req.resource)
598
+ throw new ConnectError("resource: required", Code.InvalidArgument);
599
+ run = this.startStored(who, { ...start, stored: st, version: src.version, trigger: RunTrigger.API });
600
+ break;
601
+ }
602
+ default:
603
+ throw new ConnectError("source: required", Code.InvalidArgument);
604
+ }
535
605
  return { run: this.runMessage(run), deduplicated: false };
536
606
  },
537
607
  getRun: (req, ctx) => {
@@ -541,7 +611,11 @@ export class FakePipelines {
541
611
  getRunGraph: (req, ctx) => {
542
612
  const who = this.enter("GetRun", ctx, "pipelines:run:read");
543
613
  const run = this.visible(who, req.runId);
544
- return { run: this.runMessage(run), nodes: [...run.nodes.values()].map((n) => this.nodeMessage(n)) };
614
+ return {
615
+ run: this.runMessage(run),
616
+ nodes: [...run.nodes.values()].map((n) => this.nodeMessage(n)),
617
+ ...(run.definition ? { definition: run.definition } : {}),
618
+ };
545
619
  },
546
620
  listRuns: (req, ctx) => {
547
621
  const who = this.enter("GetRun", ctx, "pipelines:run:read");
@@ -681,13 +755,13 @@ export class FakePipelines {
681
755
  return out;
682
756
  }
683
757
  // ---- the scheduler ----------------------------------------------------
684
- newRun(who, tenant, key, resource, variables, delayMs) {
758
+ newRun(who, tenant, key, resource, variables, delayMs, extra = {}) {
685
759
  const now = Date.now();
686
760
  const run = {
687
761
  id: randomUUID(),
688
762
  tenant,
689
- product: productOf(who),
690
763
  idempotencyKey: key,
764
+ fairnessKey: "",
691
765
  createdBy: `urn:fake:sa:${who.tenant}`,
692
766
  resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
693
767
  variables,
@@ -709,9 +783,19 @@ export class FakePipelines {
709
783
  scheduleTarget: "",
710
784
  issues: [],
711
785
  issuesTruncated: false,
786
+ attestedFacts: undefined,
787
+ snapshotId: "",
788
+ origin: RunOrigin.INLINE,
789
+ pipelineId: "",
790
+ pipelineVersion: 0,
791
+ parentRunId: "",
792
+ parentNodeId: "",
793
+ definition: undefined,
794
+ maxConcurrentRuns: 0,
795
+ ...extra,
712
796
  };
713
797
  this.runs.set(run.id, run);
714
- this.idempotency.set(`${run.product}\u0000${key}`, run.id);
798
+ this.idempotency.set(`${run.tenant}\u0000${key}`, run.id);
715
799
  return run;
716
800
  }
717
801
  addNode(run, spec) {
@@ -728,7 +812,170 @@ export class FakePipelines {
728
812
  retry: spec.retry ?? { max: this.opts.defaultRetries, when: [] },
729
813
  timeoutMs: spec.timeoutMs || this.opts.defaultTimeoutMs,
730
814
  interruptible: spec.interruptible,
815
+ pipeline: spec.pipeline,
816
+ forEach: spec.forEach,
817
+ elements: undefined,
818
+ });
819
+ }
820
+ // ---- definitions, snapshots and starts ---------------------------------
821
+ /**
822
+ * Checks a definition as CreateRun does and freezes it as its snapshot
823
+ * holds it: the policy's node defaults filled, each sub-pipeline pinned to
824
+ * the version it names (0, the latest now), a child of another tenant, an
825
+ * unpublished or archived one, one not taking runs about `env.about`, a
826
+ * cycle back to `env.self` or nesting over 3 refused.
827
+ */
828
+ freeze(def, env) {
829
+ if (def.nodes.length === 0)
830
+ throw new ConnectError("definition: no nodes", Code.InvalidArgument);
831
+ const out = clone(DefinitionSchema, def);
832
+ for (const n of out.nodes) {
833
+ if (!n.retry && env.defaultRetry)
834
+ n.retry = clone(RetrySchema, env.defaultRetry);
835
+ if (!n.timeout && env.defaultTimeout)
836
+ n.timeout = clone(DurationSchema, env.defaultTimeout);
837
+ if (n.action.case === "pipeline") {
838
+ const ref = n.action.value;
839
+ const st = ref.pipelineId ? this.pipelines.get(ref.pipelineId) : this.pipelines.find(env.tenant, ref.scope ?? env.scope, ref.name);
840
+ const label = ref.pipelineId || ref.name;
841
+ if (!st || st.pipeline.tenant !== env.tenant)
842
+ throw new ConnectError(`the definition is invalid: node "${n.id}": no pipeline ${label} of this tenant`, Code.InvalidArgument);
843
+ const v = this.pipelines.version(st, ref.version);
844
+ if (st.pipeline.latestVersion === 0 || !v)
845
+ throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} has no published version ${ref.version || ""}`.trimEnd(), Code.InvalidArgument);
846
+ if (st.pipeline.state === PipelineState.ARCHIVED)
847
+ throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} is archived`, Code.InvalidArgument);
848
+ if (env.about)
849
+ this.checkChild(n.id, st, env.about);
850
+ if (env.self && (st.pipeline.id === env.self || this.reaches(this.pipelines.snapshot(v.snapshotId).definition, env.self))) {
851
+ throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} reaches this pipeline again, a cycle`, Code.InvalidArgument);
852
+ }
853
+ const child = this.pipelines.snapshot(v.snapshotId).definition;
854
+ for (const o of ref.outputs) {
855
+ if (!child.nodes.some((c) => c.id === o.node))
856
+ throw new ConnectError(`the definition is invalid: node "${n.id}": output ${o.name} names node ${o.node}, which pipeline ${st.pipeline.id} does not have`, Code.InvalidArgument);
857
+ }
858
+ const declared = new Map((v.spec?.variables ?? []).map((x) => [x.name, x]));
859
+ for (const k of Object.keys(n.with)) {
860
+ if (!declared.has(k))
861
+ throw new ConnectError(`the definition is invalid: node "${n.id}": ${JSON.stringify(k)} is not a variable pipeline ${st.pipeline.id} declares`, Code.InvalidArgument);
862
+ }
863
+ for (const x of declared.values()) {
864
+ if (x.required && !x.default && !(x.name in n.with))
865
+ throw new ConnectError(`the definition is invalid: node "${n.id}": variable ${JSON.stringify(x.name)} of pipeline ${st.pipeline.id} is required`, Code.InvalidArgument);
866
+ }
867
+ n.action.value = { ...ref, pipelineId: st.pipeline.id, version: v.version };
868
+ }
869
+ }
870
+ this.compile(out);
871
+ const depth = this.depth(out);
872
+ if (depth > MAX_DEPTH)
873
+ throw new ConnectError(`the definition is invalid: it nests ${depth} deep, and runs nest at most ${MAX_DEPTH} deep counting this one`, Code.InvalidArgument);
874
+ return out;
875
+ }
876
+ /** A frozen definition's nodes, checked: handlers' inputs, the graph, for_each's rules. */
877
+ compile(def) {
878
+ const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
879
+ for (const sp of specs)
880
+ if (sp.handler)
881
+ sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
882
+ checkGraph(specs);
883
+ return specs;
884
+ }
885
+ /** How deep runs of a frozen definition nest through its pinned sub-pipelines, counting its own. */
886
+ depth(def) {
887
+ let most = 0;
888
+ for (const n of def.nodes) {
889
+ if (n.action.case !== "pipeline")
890
+ continue;
891
+ const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
892
+ if (snap)
893
+ most = Math.max(most, this.depth(snap.definition));
894
+ }
895
+ return 1 + most;
896
+ }
897
+ /** Whether a frozen definition reaches pipeline `id` through its pinned sub-pipelines. */
898
+ reaches(def, id) {
899
+ return def.nodes.some((n) => {
900
+ if (n.action.case !== "pipeline")
901
+ return false;
902
+ if (n.action.value.pipelineId === id)
903
+ return true;
904
+ const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
905
+ return snap ? this.reaches(snap.definition, id) : false;
906
+ });
907
+ }
908
+ childSnapshot(pipelineId, version) {
909
+ const st = this.pipelines.get(pipelineId);
910
+ const v = st && this.pipelines.version(st, version);
911
+ return v ? this.pipelines.snapshot(v.snapshotId) : undefined;
912
+ }
913
+ /** A sub-pipeline's child must take runs about the parent's resource, as if started directly. */
914
+ checkChild(nodeId, st, about) {
915
+ if (!takesRunsAbout(st.pipeline, about)) {
916
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": pipeline ${st.pipeline.id} (${st.pipeline.name}) takes runs about its scope or a resource of a kind in its resource_kinds, not ${JSON.stringify(about?.kind ?? "")}`, Code.InvalidArgument);
917
+ }
918
+ }
919
+ /** Every pinned child, at any depth, against the resource a run is about. */
920
+ checkChildren(def, about) {
921
+ for (const n of def.nodes) {
922
+ if (n.action.case !== "pipeline")
923
+ continue;
924
+ const st = this.pipelines.get(n.action.value.pipelineId);
925
+ this.checkChild(n.id, st, about);
926
+ const snap = this.childSnapshot(st.pipeline.id, n.action.value.version);
927
+ if (snap)
928
+ this.checkChildren(snap.definition, about);
929
+ }
930
+ }
931
+ /** Starts a run of a snapshot: an inline definition's, an earlier one by id, or a version's. */
932
+ startSnapshot(who, spec) {
933
+ this.checkChildren(spec.snapshot.definition, spec.resource);
934
+ const specs = this.compile(spec.snapshot.definition);
935
+ const run = this.newRun(who, spec.tenant, spec.key, spec.resource, spec.variables ?? {}, 0, {
936
+ attestedFacts: spec.facts,
937
+ snapshotId: spec.snapshot.id,
938
+ origin: spec.snapshot.origin,
939
+ definition: spec.snapshot.definition,
940
+ ...spec.extra,
731
941
  });
942
+ for (const s of specs)
943
+ this.addNode(run, s);
944
+ this.advance(run);
945
+ return run;
946
+ }
947
+ /** Starts a version of a stored pipeline: CreateRun's `pipeline` source, a schedule's target, a sub-pipeline's child. */
948
+ startStored(who, spec) {
949
+ const { version, snapshot, variables } = this.pipelines.resolveStart(spec.stored, spec.version, spec.resource, spec.variables);
950
+ const run = this.startSnapshot(who, {
951
+ ...spec,
952
+ variables,
953
+ snapshot,
954
+ extra: {
955
+ pipelineId: spec.stored.pipeline.id,
956
+ pipelineVersion: version.version,
957
+ maxConcurrentRuns: version.spec?.policy?.maxConcurrentRuns ?? 0,
958
+ trigger: spec.trigger,
959
+ ...(spec.parent ? { parentRunId: spec.parent.runId, parentNodeId: spec.parent.nodeId } : {}),
960
+ },
961
+ });
962
+ if (spec.schedule) {
963
+ run.scheduleId = spec.schedule.id;
964
+ run.scheduledFor = spec.schedule.scheduledFor;
965
+ run.scheduleTarget = spec.schedule.target;
966
+ }
967
+ return run;
968
+ }
969
+ /** Whether a run of a pipeline must wait for its policy's `max_concurrent_runs`. */
970
+ atCap(run) {
971
+ if (run.maxConcurrentRuns <= 0)
972
+ return false;
973
+ let active = 0;
974
+ for (const r of this.runs.values()) {
975
+ if (r.id !== run.id && r.pipelineId === run.pipelineId && r.state !== RunState.QUEUED && !terminal(r.state))
976
+ active++;
977
+ }
978
+ return active >= run.maxConcurrentRuns;
732
979
  }
733
980
  /** Makes ready what can be, skips what cannot run, and settles the run. */
734
981
  advance(run) {
@@ -736,6 +983,9 @@ export class FakePipelines {
736
983
  return;
737
984
  if (run.notBefore > Date.now() && run.cancelRequestedAt === undefined)
738
985
  return;
986
+ // A stored pipeline's run waits QUEUED under its policy's cap (the tick looks again).
987
+ if (run.state === RunState.QUEUED && run.cancelRequestedAt === undefined && this.atCap(run))
988
+ return;
739
989
  for (let changed = true; changed;) {
740
990
  changed = false;
741
991
  for (const n of run.nodes.values()) {
@@ -755,16 +1005,7 @@ export class FakePipelines {
755
1005
  run.startedAt ??= Date.now();
756
1006
  if (run.state === RunState.QUEUED)
757
1007
  run.state = RunState.RUNNING;
758
- if (n.handler) {
759
- const inputs = this.resolve(run, n);
760
- if (typeof inputs === "string") {
761
- n.state = NodeState.FAILED;
762
- n.failureClass = FailureClass.INVALID_IO;
763
- }
764
- else {
765
- this.offer(run, n, inputs, 0);
766
- }
767
- }
1008
+ this.dispatch(run, n);
768
1009
  changed = true;
769
1010
  }
770
1011
  }
@@ -787,36 +1028,205 @@ export class FakePipelines {
787
1028
  this.runFinished(run);
788
1029
  }
789
1030
  }
790
- resolve(run, n) {
791
- const out = {};
792
- for (const [k, v] of Object.entries(n.with)) {
1031
+ /**
1032
+ * A node's `with` for one attempt (of one element): whole-value
1033
+ * references, also inside lists and objects, resolved. A string is why
1034
+ * one does not resolve.
1035
+ */
1036
+ resolve(run, n, el) {
1037
+ let failed = "";
1038
+ const walk = (v) => {
1039
+ if (Array.isArray(v))
1040
+ return v.map(walk);
1041
+ if (v !== null && typeof v === "object")
1042
+ return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
793
1043
  const m = typeof v === "string" ? REF.exec(v) : null;
794
- if (!m) {
795
- out[k] = v;
796
- continue;
797
- }
798
- const path = m[1].split(".");
799
- if ((path[0] === "steps" || path[0] === "needs") && path[2] === "outputs" && path[3] === "kv" && path[4]) {
800
- const value = run.nodes.get(path[1])?.kv[path[4]];
801
- if (value === undefined)
802
- return `${m[1]} does not resolve`;
803
- out[k] = value;
1044
+ if (!m)
1045
+ return v;
1046
+ const value = this.lookup(run, m[1], el);
1047
+ if (value === undefined)
1048
+ failed ||= `${m[1]} does not resolve`;
1049
+ return value ?? null;
1050
+ };
1051
+ const out = walk(n.with);
1052
+ return failed || out;
1053
+ }
1054
+ /** One reference's value: an upstream kv output, a variable, or the element. */
1055
+ lookup(run, expr, el) {
1056
+ const path = expr.split(".");
1057
+ if ((path[0] === "steps" || path[0] === "needs") && path[2] === "outputs" && path[3] === "kv" && path[4])
1058
+ return run.nodes.get(path[1])?.kv[path[4]];
1059
+ if (path[0] === "variables" && path[1])
1060
+ return run.variables[path[1]];
1061
+ if (el && path[0] === "item_index" && path.length === 1)
1062
+ return el.index;
1063
+ if (el && path[0] === "item") {
1064
+ let v = el.item;
1065
+ for (const k of path.slice(1))
1066
+ v = v !== null && typeof v === "object" && !Array.isArray(v) ? v[k] : undefined;
1067
+ return v;
1068
+ }
1069
+ return undefined;
1070
+ }
1071
+ /** A node made ready: its attempt (or child run), its elements, or, for an external node, nothing yet. */
1072
+ dispatch(run, n) {
1073
+ if (n.forEach) {
1074
+ const items = this.items(run, n);
1075
+ if (typeof items === "string") {
1076
+ n.state = NodeState.FAILED;
1077
+ n.failureClass = FailureClass.INVALID_IO;
1078
+ return;
804
1079
  }
805
- else if (path[0] === "variables" && path[1] && run.variables[path[1]] !== undefined) {
806
- out[k] = run.variables[path[1]];
1080
+ n.elements = items.map((item, index) => ({ index, item, state: "pending", failureClass: FailureClass.UNSPECIFIED, kv: {} }));
1081
+ n.state = NodeState.RUNNING;
1082
+ this.launchElements(run, n);
1083
+ return;
1084
+ }
1085
+ if (!n.handler && !n.pipeline)
1086
+ return;
1087
+ const inputs = this.resolve(run, n);
1088
+ if (typeof inputs === "string") {
1089
+ n.state = NodeState.FAILED;
1090
+ n.failureClass = FailureClass.INVALID_IO;
1091
+ }
1092
+ else {
1093
+ this.launch(run, n, inputs, undefined, 0);
1094
+ }
1095
+ }
1096
+ /** A for_each node's list: a JSON array, or a reference resolving to one, within `max_items`. */
1097
+ items(run, n) {
1098
+ const fe = n.forEach;
1099
+ const m = typeof fe.items === "string" ? REF.exec(fe.items) : null;
1100
+ const v = m ? this.lookup(run, m[1], undefined) : fe.items;
1101
+ if (!Array.isArray(v))
1102
+ return "for_each.items does not resolve to a list";
1103
+ if (v.length > (fe.maxItems || 1000))
1104
+ return `for_each.items has ${v.length} elements, over max_items`;
1105
+ return v;
1106
+ }
1107
+ /** One attempt of a node (of one element): offered to a worker, or a sub-pipeline's child run started. */
1108
+ launch(run, n, inputs, el, delayMs) {
1109
+ if (n.pipeline)
1110
+ this.startChild(run, n, inputs, el);
1111
+ else
1112
+ this.offer(run, n, inputs, delayMs, el);
1113
+ }
1114
+ /**
1115
+ * Starts elements while `max_parallel` has room, and settles the node
1116
+ * once none is live and none will start: under FAIL_FAST nothing starts
1117
+ * after a failure.
1118
+ */
1119
+ launchElements(run, n) {
1120
+ const fe = n.forEach;
1121
+ const els = n.elements;
1122
+ const policy = fe.onFailure || ForEachFailure.FAIL_FAST;
1123
+ const stopped = () => run.cancelRequestedAt !== undefined || (policy === ForEachFailure.FAIL_FAST && els.some((e) => e.state === "failed"));
1124
+ let live = els.filter((e) => e.state === "running").length;
1125
+ for (const el of els) {
1126
+ if (stopped() || live >= (fe.maxParallel || 10))
1127
+ break;
1128
+ if (el.state !== "pending")
1129
+ continue;
1130
+ const inputs = this.resolve(run, n, el);
1131
+ if (typeof inputs === "string") {
1132
+ el.state = "failed";
1133
+ el.failureClass = FailureClass.INVALID_IO;
1134
+ continue;
807
1135
  }
808
- else {
809
- return `${m[1]} does not resolve`;
1136
+ el.state = "running";
1137
+ live++;
1138
+ this.launch(run, n, inputs, el, 0);
1139
+ }
1140
+ if (live > 0 || (!stopped() && els.some((e) => e.state === "pending")))
1141
+ return;
1142
+ // Settled: outputs as lists in element order, null where an element gave none.
1143
+ const slots = [...new Set(els.flatMap((e) => (e.state === "succeeded" ? Object.keys(e.kv) : [])))];
1144
+ n.kv = Object.fromEntries(slots.map((k) => [k, els.map((e) => (e.state === "succeeded" ? (e.kv[k] ?? null) : null))]));
1145
+ const failed = els.find((e) => e.state === "failed");
1146
+ if (run.cancelRequestedAt !== undefined) {
1147
+ n.state = NodeState.CANCELLED;
1148
+ n.failureClass = FailureClass.CANCELLED;
1149
+ }
1150
+ else if (failed && policy !== ForEachFailure.ALLOW) {
1151
+ n.state = NodeState.FAILED;
1152
+ n.failureClass = failed.failureClass;
1153
+ }
1154
+ else {
1155
+ n.state = NodeState.SUCCEEDED;
1156
+ }
1157
+ }
1158
+ /** A sub-pipeline node's attempt: one child run of the pinned version, about the parent's resource. */
1159
+ startChild(run, n, inputs, el) {
1160
+ const a = this.newAttempt(run, n, inputs, 0, el);
1161
+ a.state = AttemptState.RUNNING;
1162
+ a.leaseExpiresAt = Number.POSITIVE_INFINITY; // held by the platform, not a worker
1163
+ a.startedAt = Date.now();
1164
+ n.attempts.push(a);
1165
+ this.attemptsById.set(a.id, a);
1166
+ if (!n.forEach)
1167
+ n.state = NodeState.RUNNING;
1168
+ try {
1169
+ const child = this.startStored({ tenant: run.tenant, scopes: ALL_SCOPES }, {
1170
+ tenant: run.tenant,
1171
+ key: `parent:${run.id}:${n.id}:${a.id}`,
1172
+ resource: run.resource,
1173
+ variables: inputs,
1174
+ facts: run.attestedFacts,
1175
+ stored: this.pipelines.get(n.pipeline.id),
1176
+ version: n.pipeline.version,
1177
+ trigger: RunTrigger.PARENT,
1178
+ parent: { runId: run.id, nodeId: n.id },
1179
+ });
1180
+ a.childRunId = child.id;
1181
+ }
1182
+ catch (err) {
1183
+ // Its variables, resolved now, refused: the attempt fails, as the child would.
1184
+ a.state = AttemptState.FAILED;
1185
+ a.failureClass = FailureClass.DOWNSTREAM_FAILED;
1186
+ a.message = ConnectError.from(err).rawMessage;
1187
+ a.completedAt = Date.now();
1188
+ queueMicrotask(() => this.settle(a));
1189
+ }
1190
+ }
1191
+ /** A child run ended: its parent's attempt takes its result. */
1192
+ childFinished(child) {
1193
+ const parent = this.runs.get(child.parentRunId);
1194
+ const n = parent?.nodes.get(child.parentNodeId);
1195
+ const a = n?.attempts.find((x) => x.childRunId === child.id);
1196
+ if (!n || !a || a.state !== AttemptState.RUNNING)
1197
+ return;
1198
+ a.completedAt = Date.now();
1199
+ if (child.state === RunState.SUCCEEDED) {
1200
+ a.state = AttemptState.SUCCEEDED;
1201
+ for (const o of n.pipeline.outputs) {
1202
+ const v = child.nodes.get(o.node)?.kv[o.slot];
1203
+ if (v !== undefined)
1204
+ a.kv[o.name] = v;
810
1205
  }
811
1206
  }
812
- return out;
1207
+ else if (child.state === RunState.CANCELLED) {
1208
+ a.state = AttemptState.CANCELLED;
1209
+ a.failureClass = FailureClass.CANCELLED;
1210
+ }
1211
+ else {
1212
+ a.state = AttemptState.FAILED;
1213
+ a.failureClass = FailureClass.DOWNSTREAM_FAILED;
1214
+ a.message = `the child run ${child.id} failed`;
1215
+ }
1216
+ this.settle(a);
1217
+ }
1218
+ offer(run, n, inputs, delayMs, el) {
1219
+ const a = this.newAttempt(run, n, inputs, delayMs, el);
1220
+ n.attempts.push(a);
1221
+ this.attemptsById.set(a.id, a);
1222
+ this.notify();
813
1223
  }
814
- offer(run, n, inputs, delayMs) {
815
- const a = {
1224
+ newAttempt(run, n, inputs, delayMs, el) {
1225
+ return {
816
1226
  id: randomUUID(),
817
1227
  runId: run.id,
818
1228
  nodeId: n.id,
819
- number: n.attempts.length + 1,
1229
+ number: n.attempts.filter((x) => x.itemIndex === el?.index).length + 1,
820
1230
  state: AttemptState.OFFERED,
821
1231
  epoch: 0n,
822
1232
  workerId: "",
@@ -838,10 +1248,10 @@ export class FakePipelines {
838
1248
  offeredAt: Date.now(),
839
1249
  progress: undefined,
840
1250
  spans: [],
1251
+ itemIndex: el?.index,
1252
+ item: el?.item,
1253
+ childRunId: "",
841
1254
  };
842
- n.attempts.push(a);
843
- this.attemptsById.set(a.id, a);
844
- this.notify();
845
1255
  }
846
1256
  /**
847
1257
  * Claims for a worker: the attempts of handler nodes of full runs first,
@@ -862,7 +1272,7 @@ export class FakePipelines {
862
1272
  if (a.state !== AttemptState.OFFERED || a.notBefore > now)
863
1273
  continue;
864
1274
  if (run.queueId)
865
- queued.push({ item: a, queueId: run.queueId, tenant: run.tenant, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
1275
+ queued.push({ item: a, queueId: run.queueId, lane: run.fairnessKey, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
866
1276
  else if (out.length < max)
867
1277
  out.push(a);
868
1278
  }
@@ -889,10 +1299,10 @@ export class FakePipelines {
889
1299
  return out;
890
1300
  }
891
1301
  /** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
892
- queueRunning(queueId, tenant) {
1302
+ queueRunning(queueId, lane) {
893
1303
  let n = 0;
894
1304
  for (const run of this.runs.values()) {
895
- if (run.queueId !== queueId || (tenant !== undefined && run.tenant !== tenant))
1305
+ if (run.queueId !== queueId || (lane !== undefined && run.fairnessKey !== lane))
896
1306
  continue;
897
1307
  for (const node of run.nodes.values())
898
1308
  for (const a of node.attempts)
@@ -942,13 +1352,28 @@ export class FakePipelines {
942
1352
  settle(a) {
943
1353
  const run = this.run(a.runId);
944
1354
  const n = run.nodes.get(a.nodeId);
1355
+ // An element's attempt settles its element; the node settles with the last one.
1356
+ const el = a.itemIndex !== undefined ? n.elements?.[a.itemIndex] : undefined;
1357
+ const mine = n.attempts.filter((x) => x.itemIndex === a.itemIndex);
945
1358
  if (a.state === AttemptState.SUCCEEDED) {
946
- n.state = NodeState.SUCCEEDED;
947
- n.kv = a.kv;
1359
+ if (el) {
1360
+ el.state = "succeeded";
1361
+ el.kv = a.kv;
1362
+ }
1363
+ else {
1364
+ n.state = NodeState.SUCCEEDED;
1365
+ n.kv = a.kv;
1366
+ }
948
1367
  }
949
1368
  else if (a.state === AttemptState.CANCELLED || run.cancelRequestedAt !== undefined) {
950
- n.state = NodeState.CANCELLED;
951
- n.failureClass = FailureClass.CANCELLED;
1369
+ if (el) {
1370
+ el.state = "failed";
1371
+ el.failureClass = FailureClass.CANCELLED;
1372
+ }
1373
+ else {
1374
+ n.state = NodeState.CANCELLED;
1375
+ n.failureClass = FailureClass.CANCELLED;
1376
+ }
952
1377
  }
953
1378
  else {
954
1379
  const permanent = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.PERMANENT;
@@ -956,24 +1381,32 @@ export class FakePipelines {
956
1381
  const when = n.retry.when.length > 0 ? n.retry.when : transientHandler ? [...TRANSIENT, FailureClass.HANDLER_ERROR] : TRANSIENT;
957
1382
  // On an interruptible node an interruption is free, up to a bound.
958
1383
  const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
959
- const freeUsed = n.attempts.filter(free).length;
1384
+ const freeUsed = mine.filter(free).length;
960
1385
  if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
961
1386
  // Up to the bound, an interruption is offered again at once.
962
- this.offer(run, n, a.inputs, 0);
1387
+ this.launch(run, n, a.inputs, el, 0);
963
1388
  return;
964
1389
  }
965
- const retries = n.attempts.slice(0, -1).filter((x) => !free(x)).length;
1390
+ const retries = mine.slice(0, -1).filter((x) => !free(x)).length;
966
1391
  // As the api: a worker that vanished (ABANDONED) may have had outside
967
1392
  // effects, so its node runs again automatically only when it declares
968
1393
  // itself interruptible.
969
1394
  const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
970
1395
  if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
971
- this.offer(run, n, a.inputs, this.opts.retryBackoffMs);
1396
+ this.launch(run, n, a.inputs, el, this.opts.retryBackoffMs);
972
1397
  return;
973
1398
  }
974
- n.state = NodeState.FAILED;
975
- n.failureClass = a.failureClass;
1399
+ if (el) {
1400
+ el.state = "failed";
1401
+ el.failureClass = a.failureClass;
1402
+ }
1403
+ else {
1404
+ n.state = NodeState.FAILED;
1405
+ n.failureClass = a.failureClass;
1406
+ }
976
1407
  }
1408
+ if (el)
1409
+ this.launchElements(run, n);
977
1410
  this.advance(run);
978
1411
  this.notify();
979
1412
  }
@@ -983,8 +1416,11 @@ export class FakePipelines {
983
1416
  run.cancelRequestedAt = Date.now();
984
1417
  run.cancelReason = reason;
985
1418
  run.state = RunState.CANCELLING;
1419
+ const children = [];
986
1420
  for (const n of run.nodes.values()) {
987
1421
  for (const a of n.attempts) {
1422
+ if (a.state === AttemptState.RUNNING && a.childRunId)
1423
+ children.push(this.run(a.childRunId));
988
1424
  if (a.state === AttemptState.OFFERED) {
989
1425
  a.state = AttemptState.CANCELLED;
990
1426
  a.failureClass = FailureClass.CANCELLED;
@@ -996,6 +1432,9 @@ export class FakePipelines {
996
1432
  n.state = n.state === NodeState.PENDING ? NodeState.SKIPPED : NodeState.CANCELLED;
997
1433
  }
998
1434
  }
1435
+ // Cancelling a run cancels its sub-pipelines' children; each one's end settles its attempt.
1436
+ for (const c of children)
1437
+ this.cancel(c, reason);
999
1438
  this.advance(run);
1000
1439
  this.notify();
1001
1440
  }
@@ -1040,7 +1479,7 @@ export class FakePipelines {
1040
1479
  /** QueuesService, as the contract states it. */
1041
1480
  queuesService() {
1042
1481
  const readable = (who, q) => {
1043
- if (!q || q.owner?.product !== productOf(who) || (q.owner.tenant && q.owner.tenant !== who.tenant))
1482
+ if (!q || q.owner?.tenant !== who.tenant)
1044
1483
  throw new ConnectError("no such queue", Code.NotFound);
1045
1484
  return q;
1046
1485
  };
@@ -1049,12 +1488,10 @@ export class FakePipelines {
1049
1488
  setQueue: (req, ctx) => {
1050
1489
  const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
1051
1490
  const q = req.queue;
1052
- if (!q?.owner)
1053
- throw new ConnectError("queue.owner: required", Code.InvalidArgument);
1054
- if (q.owner.product !== productOf(who))
1055
- throw new ConnectError("queue.owner.product: a product sets only its own queues", Code.PermissionDenied);
1056
- if (q.owner.tenant && q.owner.tenant !== who.tenant)
1057
- throw new ConnectError("queue.owner.tenant: the caller does not act for it", Code.PermissionDenied);
1491
+ if (!q?.owner?.tenant)
1492
+ throw new ConnectError("queue.owner.tenant: required", Code.InvalidArgument);
1493
+ if (q.owner.tenant !== who.tenant)
1494
+ throw new ConnectError("queue.owner.tenant: the caller is not a service-account member of it", Code.PermissionDenied);
1058
1495
  if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
1059
1496
  throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
1060
1497
  if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
@@ -1062,7 +1499,7 @@ export class FakePipelines {
1062
1499
  }
1063
1500
  if (req.updateMask)
1064
1501
  throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
1065
- const existing = [...this.queues.values()].find((x) => x.owner?.product === q.owner.product && x.owner.tenant === q.owner.tenant && x.name === q.name);
1502
+ const existing = [...this.queues.values()].find((x) => x.owner?.tenant === q.owner.tenant && x.name === q.name);
1066
1503
  const now = Date.now();
1067
1504
  const next = create(QueueSchema, {
1068
1505
  ...q,
@@ -1085,22 +1522,23 @@ export class FakePipelines {
1085
1522
  if (req.queue.case !== "byName")
1086
1523
  throw new ConnectError("queue: required", Code.InvalidArgument);
1087
1524
  const { owner, name } = req.queue.value;
1088
- if (owner?.product !== productOf(who))
1525
+ if (!owner?.tenant || owner.tenant !== who.tenant)
1089
1526
  throw new ConnectError("no such queue", Code.NotFound);
1090
- const found = [...this.queues.values()].find((x) => x.owner?.product === owner.product && x.owner.tenant === owner.tenant && x.name === name);
1527
+ const found = [...this.queues.values()].find((x) => x.owner?.tenant === owner.tenant && x.name === name);
1091
1528
  if (found)
1092
1529
  return { queue: readable(who, found) };
1093
- if (name === "default" && !owner.tenant)
1094
- return { queue: this.defaultQueue(owner.product) };
1530
+ if (name === "default")
1531
+ return { queue: this.defaultQueue(owner.tenant) };
1095
1532
  throw new ConnectError("no such queue", Code.NotFound);
1096
1533
  },
1097
1534
  listQueues: (req, ctx) => {
1098
1535
  const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1099
- const product = req.product || productOf(who);
1100
- if (product !== productOf(who))
1536
+ if (!req.tenant)
1537
+ throw new ConnectError("tenant: required", Code.InvalidArgument);
1538
+ if (req.tenant !== who.tenant)
1101
1539
  return { queues: [], nextCursor: "" };
1102
1540
  const queues = [...this.queues.values()]
1103
- .filter((q) => q.owner?.product === product && (!req.tenant || q.owner.tenant === req.tenant) && (!q.owner.tenant || q.owner.tenant === who.tenant))
1541
+ .filter((q) => q.owner?.tenant === req.tenant)
1104
1542
  .sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
1105
1543
  return { queues, nextCursor: "" };
1106
1544
  },
@@ -1152,30 +1590,25 @@ export class FakePipelines {
1152
1590
  let running = 0;
1153
1591
  let started = 0n;
1154
1592
  let oldest;
1155
- const tenants = new Map();
1156
1593
  for (const r of runs) {
1157
- const t = tenants.get(r.tenant) ?? { waiting: 0, running: 0 };
1158
1594
  for (const a of this.attemptsOf(r.id)) {
1159
1595
  if (a.state === AttemptState.OFFERED) {
1160
1596
  if (a.notBefore > now || r.notBefore > now)
1161
1597
  delayed++;
1162
1598
  else {
1163
1599
  waiting++;
1164
- t.waiting++;
1165
1600
  const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
1166
1601
  oldest = oldest === undefined ? since : Math.min(oldest, since);
1167
1602
  }
1168
1603
  }
1169
1604
  if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
1170
1605
  running++;
1171
- t.running++;
1172
1606
  }
1173
1607
  if (a.startedAt && a.startedAt >= now - windowMs)
1174
1608
  started++;
1175
1609
  }
1176
1610
  if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
1177
1611
  delayed++;
1178
- tenants.set(r.tenant, t);
1179
1612
  }
1180
1613
  const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
1181
1614
  return {
@@ -1183,10 +1616,6 @@ export class FakePipelines {
1183
1616
  delayed,
1184
1617
  running,
1185
1618
  ...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
1186
- tenants: [...tenants.entries()]
1187
- .map(([tenant, v]) => ({ tenant, ...v }))
1188
- .sort((a, b) => b.waiting - a.waiting || b.running - a.running)
1189
- .slice(0, req.topTenants || 10),
1190
1619
  started,
1191
1620
  failed,
1192
1621
  paused: q.paused,
@@ -1197,7 +1626,7 @@ export class FakePipelines {
1197
1626
  const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1198
1627
  const q = byId(who, req.queueId);
1199
1628
  const runs = [...this.runs.values()]
1200
- .filter((r) => r.queueId === q.id && (!req.tenant || r.tenant === req.tenant) && (req.states.length === 0 || req.states.includes(r.state)))
1629
+ .filter((r) => r.queueId === q.id && (req.states.length === 0 || req.states.includes(r.state)))
1201
1630
  .reverse();
1202
1631
  return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
1203
1632
  },
@@ -1211,14 +1640,15 @@ export class FakePipelines {
1211
1640
  throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
1212
1641
  if (spec.retry?.when.includes(FailureClass.CANCELLED))
1213
1642
  throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
1214
- const q = this.resolveQueue(productOf(who), spec.tenant, spec.queue);
1643
+ const q = this.resolveQueue(spec.tenant, spec.queue);
1215
1644
  if (!q)
1216
- throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among this product's queues`, Code.NotFound);
1645
+ throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among the tenant's queues`, Code.NotFound);
1217
1646
  const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
1218
1647
  const payload = this.checkInputs("job", spec.handler, spec.payload);
1219
1648
  const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
1220
1649
  run.queueId = q.id;
1221
1650
  run.queue = q.name;
1651
+ run.fairnessKey = spec.fairnessKey ?? "";
1222
1652
  run.trigger = spec.trigger ?? RunTrigger.API;
1223
1653
  if (spec.schedule) {
1224
1654
  run.scheduleId = spec.schedule.id;
@@ -1238,22 +1668,21 @@ export class FakePipelines {
1238
1668
  this.advance(run);
1239
1669
  return run;
1240
1670
  }
1241
- /** A job's queue: the tenant's own of that name, else the product-wide one; empty is `default`. */
1242
- resolveQueue(product, tenant, name) {
1671
+ /** A job's queue: the tenant's of that name; empty is `default`. */
1672
+ resolveQueue(tenant, name) {
1243
1673
  const want = name || "default";
1244
- const all = [...this.queues.values()].filter((q) => q.owner?.product === product && q.name === want);
1245
- const found = all.find((q) => q.owner?.tenant === tenant) ?? all.find((q) => !q.owner?.tenant);
1674
+ const found = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === want);
1246
1675
  if (found)
1247
1676
  return found;
1248
- return want === "default" ? this.defaultQueue(product) : undefined;
1677
+ return want === "default" ? this.defaultQueue(tenant) : undefined;
1249
1678
  }
1250
- /** The product's `default` queue, made on first use with no limits. */
1251
- defaultQueue(product) {
1252
- const existing = [...this.queues.values()].find((q) => q.owner?.product === product && !q.owner.tenant && q.name === "default");
1679
+ /** The tenant's `default` queue, made on first use with no limits. */
1680
+ defaultQueue(tenant) {
1681
+ const existing = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === "default");
1253
1682
  if (existing)
1254
1683
  return existing;
1255
1684
  const now = Date.now();
1256
- const q = create(QueueSchema, { id: randomUUID(), owner: { product, tenant: "" }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
1685
+ const q = create(QueueSchema, { id: randomUUID(), owner: { tenant }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
1257
1686
  this.queues.set(q.id, q);
1258
1687
  return q;
1259
1688
  }
@@ -1264,7 +1693,7 @@ export class FakePipelines {
1264
1693
  throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
1265
1694
  if (!terminal(run.state))
1266
1695
  throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
1267
- if (!n.handler || n.attempts.length === 0 || !terminalNode(n.state))
1696
+ if ((!n.handler && !n.pipeline) || (n.attempts.length === 0 && !n.forEach) || !terminalNode(n.state))
1268
1697
  throw new ConnectError("the node never settled", Code.FailedPrecondition);
1269
1698
  run.state = RunState.RUNNING;
1270
1699
  run.finishedAt = undefined;
@@ -1288,8 +1717,27 @@ export class FakePipelines {
1288
1717
  x.failureClass = FailureClass.UNSPECIFIED;
1289
1718
  x.kv = {};
1290
1719
  }
1720
+ if (n.forEach && n.elements) {
1721
+ // Only the failed elements run again; the succeeded ones are kept.
1722
+ for (const el of n.elements) {
1723
+ if (el.state === "succeeded")
1724
+ continue;
1725
+ el.state = "pending";
1726
+ el.failureClass = FailureClass.UNSPECIFIED;
1727
+ el.kv = {};
1728
+ }
1729
+ n.state = NodeState.RUNNING;
1730
+ this.launchElements(run, n);
1731
+ this.advance(run);
1732
+ return;
1733
+ }
1734
+ if (n.forEach) {
1735
+ this.dispatch(run, n);
1736
+ this.advance(run);
1737
+ return;
1738
+ }
1291
1739
  const inputs = this.resolve(run, n);
1292
- this.offer(run, n, typeof inputs === "string" ? {} : inputs, 0);
1740
+ this.launch(run, n, typeof inputs === "string" ? {} : inputs, undefined, 0);
1293
1741
  }
1294
1742
  // ---- issues, progress and spans ---------------------------------------
1295
1743
  /** Every declared stage PENDING from the claim, as the platform does. */
@@ -1458,6 +1906,9 @@ export class FakePipelines {
1458
1906
  }
1459
1907
  /** Runs whenever a run becomes terminal. */
1460
1908
  runFinished(run) {
1909
+ // Deferred, so a parent never settles inside the child's own advance.
1910
+ if (run.parentRunId)
1911
+ queueMicrotask(() => this.childFinished(run));
1461
1912
  // The schedules are made after the router; a run cannot finish before.
1462
1913
  this.schedules?.runFinished(run);
1463
1914
  }
@@ -1466,8 +1917,8 @@ export class FakePipelines {
1466
1917
  return create(RunSchema, {
1467
1918
  id: r.id,
1468
1919
  tenant: r.tenant,
1469
- product: r.product,
1470
1920
  idempotencyKey: r.idempotencyKey,
1921
+ fairnessKey: r.fairnessKey,
1471
1922
  ...(r.resource ? { resource: r.resource } : {}),
1472
1923
  variables: r.variables,
1473
1924
  state: r.state,
@@ -1490,6 +1941,13 @@ export class FakePipelines {
1490
1941
  issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
1491
1942
  issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
1492
1943
  issuesTruncated: r.issuesTruncated,
1944
+ ...(r.attestedFacts ? { attestedFacts: r.attestedFacts } : {}),
1945
+ snapshotId: r.snapshotId,
1946
+ origin: r.origin,
1947
+ pipelineId: r.pipelineId,
1948
+ pipelineVersion: r.pipelineVersion,
1949
+ parentRunId: r.parentRunId,
1950
+ parentNodeId: r.parentNodeId,
1493
1951
  });
1494
1952
  }
1495
1953
  nodeMessage(n) {
@@ -1500,6 +1958,18 @@ export class FakePipelines {
1500
1958
  failureClass: n.failureClass,
1501
1959
  settledAttemptId: terminalNode(n.state) ? (settled?.id ?? "") : "",
1502
1960
  kv: Object.fromEntries(Object.entries(n.kv).map(([k, v]) => [k, jsonValue(v)])),
1961
+ downstreamRunId: n.attempts.findLast((a) => a.childRunId)?.childRunId ?? "",
1962
+ ...(n.elements
1963
+ ? {
1964
+ forEach: {
1965
+ total: n.elements.length,
1966
+ succeeded: n.elements.filter((e) => e.state === "succeeded").length,
1967
+ failed: n.elements.filter((e) => e.state === "failed").length,
1968
+ running: n.elements.filter((e) => e.state === "running").length,
1969
+ pending: n.elements.filter((e) => e.state === "pending").length,
1970
+ },
1971
+ }
1972
+ : {}),
1503
1973
  attempts: n.attempts.map((a) => create(AttemptSchema, {
1504
1974
  id: a.id,
1505
1975
  runId: a.runId,
@@ -1512,6 +1982,8 @@ export class FakePipelines {
1512
1982
  termination: a.termination,
1513
1983
  ...(a.exitCode !== undefined ? { exitCode: a.exitCode } : {}),
1514
1984
  message: a.message,
1985
+ childRunId: a.childRunId,
1986
+ ...(a.itemIndex !== undefined ? { itemIndex: a.itemIndex, item: jsonValue(a.item ?? null) } : {}),
1515
1987
  logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
1516
1988
  logTruncated: a.logTruncated,
1517
1989
  ...(a.progress
@@ -1543,10 +2015,21 @@ export class FakePipelines {
1543
2015
  });
1544
2016
  }
1545
2017
  }
2018
+ /** Every whole-value reference in a value, also inside lists and objects. */
2019
+ function refs(v) {
2020
+ if (Array.isArray(v))
2021
+ return v.flatMap(refs);
2022
+ if (v !== null && typeof v === "object")
2023
+ return Object.values(v).flatMap(refs);
2024
+ const m = typeof v === "string" ? REF.exec(v) : null;
2025
+ return m ? [m[1]] : [];
2026
+ }
2027
+ const ITEM_REF = /^item(_index$|$|\.)/;
1546
2028
  /**
1547
2029
  * Holds a definition to the server's rules the fake models: unique ids,
1548
- * known needs, no cycle, and a reference names only a node in the referring
1549
- * node's dependency closure (a reference adds no edge).
2030
+ * known needs, no cycle, a reference names only a node in the referring
2031
+ * node's dependency closure (a reference adds no edge), and `item` is read
2032
+ * only in a for_each node's `with`.
1550
2033
  */
1551
2034
  function checkGraph(specs) {
1552
2035
  const byId = new Map();
@@ -1570,18 +2053,22 @@ function checkGraph(specs) {
1570
2053
  };
1571
2054
  for (const s of specs) {
1572
2055
  const reachable = closure(s.id);
1573
- for (const v of Object.values(s.with)) {
1574
- const m = typeof v === "string" ? REF.exec(v) : null;
1575
- const node = m?.[1]?.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
1576
- if (node && !reachable.has(node)) {
1577
- throw new ConnectError(`node ${s.id}: ${m[1]} names a node outside its needs`, Code.InvalidArgument);
1578
- }
2056
+ const items = s.forEach ? refs(s.forEach.items) : [];
2057
+ for (const r of [...refs(s.with), ...items]) {
2058
+ const node = r.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
2059
+ if (node && !reachable.has(node))
2060
+ throw new ConnectError(`node ${s.id}: ${r} names a node outside its needs`, Code.InvalidArgument);
1579
2061
  }
2062
+ if (!s.forEach && refs(s.with).some((r) => ITEM_REF.test(r)))
2063
+ throw new ConnectError(`node ${s.id}: item is read only in a for_each node`, Code.InvalidArgument);
2064
+ if (items.some((r) => ITEM_REF.test(r)))
2065
+ throw new ConnectError(`node ${s.id}: for_each.items cannot read item`, Code.InvalidArgument);
1580
2066
  }
1581
2067
  }
1582
2068
  function nodeSpec(n, requireHandler) {
1583
2069
  let handler = "";
1584
2070
  let uses = "";
2071
+ let pipeline;
1585
2072
  if (n.action.case === "handler") {
1586
2073
  handler = n.action.value;
1587
2074
  requireHandler(handler);
@@ -1589,8 +2076,31 @@ function nodeSpec(n, requireHandler) {
1589
2076
  else if (n.action.case === "uses") {
1590
2077
  uses = `${n.action.value.name}@${n.action.value.version}`;
1591
2078
  }
2079
+ else if (n.action.case === "pipeline") {
2080
+ // Pinned by freeze() before a node is compiled.
2081
+ const p = n.action.value;
2082
+ pipeline = { id: p.pipelineId, version: p.version, outputs: p.outputs.map((o) => ({ name: o.name, node: o.node, slot: o.slot })) };
2083
+ }
1592
2084
  else {
1593
- throw new ConnectError(`node ${n.id}: only handler and uses nodes`, Code.Unimplemented);
2085
+ throw new ConnectError(`node ${n.id}: only handler, uses and pipeline nodes`, Code.Unimplemented);
2086
+ }
2087
+ let forEach;
2088
+ if (n.forEach) {
2089
+ if (n.manual)
2090
+ throw new ConnectError(`node ${n.id}: for_each is not for a manual node`, Code.InvalidArgument);
2091
+ if (uses)
2092
+ throw new ConnectError(`node ${n.id}: the fake runs for_each on handler and pipeline nodes only`, Code.Unimplemented);
2093
+ const fe = n.forEach;
2094
+ if (fe.maxItems > 10_000)
2095
+ throw new ConnectError(`node ${n.id}: for_each.max_items is at most 10000`, Code.InvalidArgument);
2096
+ const items = fe.items ? toJson(ValueSchema, fe.items) : null;
2097
+ if (typeof items === "string" ? !REF.test(items) : !Array.isArray(items)) {
2098
+ throw new ConnectError(`node ${n.id}: for_each.items is a list or a whole-value reference to one`, Code.InvalidArgument);
2099
+ }
2100
+ if (Array.isArray(items) && items.length > (fe.maxItems || 1000)) {
2101
+ throw new ConnectError(`node ${n.id}: for_each.items has ${items.length} elements, over max_items ${fe.maxItems || 1000}`, Code.InvalidArgument);
2102
+ }
2103
+ forEach = { items, maxItems: fe.maxItems, maxParallel: fe.maxParallel, onFailure: fe.onFailure };
1594
2104
  }
1595
2105
  return {
1596
2106
  id: n.id,
@@ -1601,6 +2111,8 @@ function nodeSpec(n, requireHandler) {
1601
2111
  retry: n.retry ? { max: n.retry.max, when: n.retry.when } : undefined,
1602
2112
  timeoutMs: n.timeout ? durationMs(n.timeout) : 0,
1603
2113
  interruptible: n.interruptible,
2114
+ ...(pipeline ? { pipeline } : {}),
2115
+ ...(forEach ? { forEach } : {}),
1604
2116
  };
1605
2117
  }
1606
2118
  /** The server's dispatch.Outcome. */
@@ -1665,10 +2177,6 @@ function aborted(s) {
1665
2177
  s.addEventListener("abort", () => r(), { once: true });
1666
2178
  });
1667
2179
  }
1668
- /** The product an account belongs to. */
1669
- function productOf(who) {
1670
- return who.product ?? FAKE_PRODUCT;
1671
- }
1672
2180
  /** A queue's limits as the selection reads them. */
1673
2181
  function queueLimits(q) {
1674
2182
  return {