@xeonr/pipelines-sdk 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -33
- package/dist/auth.d.ts +1 -1
- package/dist/client.d.ts +14 -10
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +5 -4
- package/dist/client.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +6 -4
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +37 -6
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +6 -5
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +60 -6
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.js +58 -3
- package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +37 -7
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +22 -50
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +10 -9
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +103 -8
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js +32 -24
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +40 -98
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js +46 -51
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +36 -27
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js +16 -18
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +372 -57
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js +205 -37
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts +194 -0
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js +100 -0
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +138 -62
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +89 -49
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +32 -5
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +12 -3
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts +270 -27
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js +163 -24
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts +391 -0
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js +100 -0
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js.map +1 -0
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts +14 -0
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/platform/v1/options_pb.js +18 -4
- package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
- package/dist/handler.d.ts +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/queues.d.ts +15 -28
- package/dist/queues.d.ts.map +1 -1
- package/dist/queues.js +15 -33
- package/dist/queues.js.map +1 -1
- package/dist/reporter.d.ts +1 -1
- package/dist/reporter.js +4 -4
- package/dist/reporter.js.map +1 -1
- package/dist/schedules.d.ts +9 -17
- package/dist/schedules.d.ts.map +1 -1
- package/dist/schedules.js +11 -12
- package/dist/schedules.js.map +1 -1
- package/dist/testing/index.d.ts +111 -14
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +638 -130
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/pipelines.d.ts +99 -0
- package/dist/testing/pipelines.d.ts.map +1 -0
- package/dist/testing/pipelines.js +445 -0
- package/dist/testing/pipelines.js.map +1 -0
- package/dist/testing/queues.d.ts +13 -10
- package/dist/testing/queues.d.ts.map +1 -1
- package/dist/testing/queues.js +15 -13
- package/dist/testing/queues.js.map +1 -1
- package/dist/testing/schedules.d.ts +20 -8
- package/dist/testing/schedules.d.ts.map +1 -1
- package/dist/testing/schedules.js +46 -43
- package/dist/testing/schedules.js.map +1 -1
- package/dist/worker.d.ts +8 -14
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +8 -19
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
package/dist/testing/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @xeonr/pipelines-sdk/testing: an in-memory fake of pipelines-api's worker
|
|
3
|
-
* and jobs surface, so a
|
|
3
|
+
* and jobs surface, so a service can unit-test its handlers, and the code
|
|
4
4
|
* that enqueues them, with no server.
|
|
5
5
|
*
|
|
6
6
|
* It is a real Connect router behind an in-memory transport, so the SDK's
|
|
@@ -31,26 +31,45 @@
|
|
|
31
31
|
* `handler_failure_kind`, retries by class, free interruptions on an
|
|
32
32
|
* `interruptible` node, cancel, and error injection.
|
|
33
33
|
*
|
|
34
|
+
* Stored pipelines (`PipelinesService`, see ./pipelines.ts) and the runs of
|
|
35
|
+
* their versions (`CreateRun` with `pipeline` or `snapshot_id`, and a
|
|
36
|
+
* schedule's pipeline target), with the policy's node defaults and
|
|
37
|
+
* `max_concurrent_runs`. Sub-pipeline nodes: pinned to a version when the
|
|
38
|
+
* snapshot is made (cycles and nesting over 3 refused), one child run per
|
|
39
|
+
* attempt about the parent's resource, the child's outputs mapped back, a
|
|
40
|
+
* failed child failing the attempt as `DOWNSTREAM_FAILED`, a cancel passed
|
|
41
|
+
* down. `for_each` nodes on handler and sub-pipeline nodes: the list from a
|
|
42
|
+
* literal, a variable or an upstream kv output, `${{ item }}`,
|
|
43
|
+
* `${{ item.<path> }}` and `${{ item_index }}`, `max_items`,
|
|
44
|
+
* `max_parallel`, each element retried on its own, the three `on_failure`
|
|
45
|
+
* policies, outputs aggregated in element order, and RetryNode running only
|
|
46
|
+
* the failed elements again.
|
|
47
|
+
*
|
|
34
48
|
* What it does not: secrets, blobs, tiers, routing between pools, the
|
|
35
|
-
* outbox,
|
|
49
|
+
* outbox, exclusions, `for_each` on an external (image) node, and the
|
|
50
|
+
* platform's own deadline enforcement.
|
|
36
51
|
*/
|
|
37
52
|
import { createHash, randomUUID } from "node:crypto";
|
|
38
|
-
import { create, fromJson, toJson } from "@bufbuild/protobuf";
|
|
39
|
-
import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
53
|
+
import { clone, create, fromJson, toJson } from "@bufbuild/protobuf";
|
|
54
|
+
import { DurationSchema, durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
40
55
|
import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
|
|
41
56
|
import { tokenSource } from "../auth.js";
|
|
42
57
|
import { authInterceptor } from "../transport.js";
|
|
43
58
|
import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
|
|
59
|
+
import { DefinitionSchema, ForEachFailure, RetrySchema } from "../gen/xeonr/pipelines/v1/definition_pb.js";
|
|
60
|
+
import { PipelineState, PipelinesService } from "../gen/xeonr/pipelines/v1/pipelines_pb.js";
|
|
44
61
|
import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
|
|
45
62
|
import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
|
|
46
|
-
import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
63
|
+
import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunOrigin, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
47
64
|
import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
|
|
65
|
+
import { FakeStoredPipelines, takesRunsAbout } from "./pipelines.js";
|
|
48
66
|
import { pickQueued } from "./queues.js";
|
|
49
67
|
import { FakeClock, FakeSchedules } from "./schedules.js";
|
|
50
68
|
import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
|
|
51
69
|
export { FakeClock, FakeSchedules } from "./schedules.js";
|
|
52
70
|
export { parseCron, nextOccurrence } from "./cron.js";
|
|
53
71
|
export { pickQueued } from "./queues.js";
|
|
72
|
+
export { FakeStoredPipelines, resolveVariables } from "./pipelines.js";
|
|
54
73
|
import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
|
|
55
74
|
/** The tenant the default account acts for. */
|
|
56
75
|
export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
|
|
@@ -69,9 +88,9 @@ export const ALL_SCOPES = [
|
|
|
69
88
|
"pipelines:queue:edit",
|
|
70
89
|
"pipelines:schedule:read",
|
|
71
90
|
"pipelines:schedule:edit",
|
|
91
|
+
"pipelines:pipeline:read",
|
|
92
|
+
"pipelines:pipeline:edit",
|
|
72
93
|
];
|
|
73
|
-
/** The product the default account belongs to. */
|
|
74
|
-
export const FAKE_PRODUCT = "00000000-0000-4000-8000-0000000000a1";
|
|
75
94
|
const TRANSIENT = [
|
|
76
95
|
FailureClass.ABANDONED,
|
|
77
96
|
FailureClass.RUNNER_CRASHED,
|
|
@@ -84,6 +103,8 @@ const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
|
84
103
|
/** How many interruptions an interruptible node gets without spending a retry (the api's default). */
|
|
85
104
|
const FREE_INTERRUPTIONS = 3;
|
|
86
105
|
const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
|
|
106
|
+
/** How deeply runs nest through sub-pipelines, counting the outermost (the api's MaxDepth). */
|
|
107
|
+
const MAX_DEPTH = 3;
|
|
87
108
|
/**
|
|
88
109
|
* An in-memory pipelines-api for tests. Call {@link close} when done (it
|
|
89
110
|
* stops the timer that looks at leases and delays).
|
|
@@ -110,6 +131,8 @@ export class FakePipelines {
|
|
|
110
131
|
clock = new FakeClock();
|
|
111
132
|
/** The schedules, with their recorded occurrences. */
|
|
112
133
|
schedules;
|
|
134
|
+
/** Stored pipelines, their versions and the snapshots runs execute. */
|
|
135
|
+
pipelines;
|
|
113
136
|
queueClaims = new Map();
|
|
114
137
|
timer;
|
|
115
138
|
changed;
|
|
@@ -129,8 +152,12 @@ export class FakePipelines {
|
|
|
129
152
|
spanCap: opts.spanCap ?? 10_000,
|
|
130
153
|
};
|
|
131
154
|
this.changed = signal();
|
|
155
|
+
this.pipelines = new FakeStoredPipelines({
|
|
156
|
+
enter: (method, ctx, scope) => this.enter(method, ctx, scope),
|
|
157
|
+
freeze: (def, env) => this.freeze(def, env),
|
|
158
|
+
});
|
|
132
159
|
this.schedules = new FakeSchedules({
|
|
133
|
-
createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES
|
|
160
|
+
createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
|
|
134
161
|
tenant: owner.tenant,
|
|
135
162
|
handler: spec.handler,
|
|
136
163
|
key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
|
|
@@ -144,10 +171,31 @@ export class FakePipelines {
|
|
|
144
171
|
trigger: spec.trigger,
|
|
145
172
|
schedule: spec.schedule,
|
|
146
173
|
}),
|
|
174
|
+
startPipeline: (owner, spec) => {
|
|
175
|
+
const st = this.pipelines.get(spec.target.pipelineId);
|
|
176
|
+
if (!st || st.pipeline.tenant !== owner.tenant)
|
|
177
|
+
throw new ConnectError("no such pipeline", Code.NotFound);
|
|
178
|
+
return this.startStored({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
|
|
179
|
+
tenant: owner.tenant,
|
|
180
|
+
key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
|
|
181
|
+
resource: spec.resource ?? (st.pipeline.scope ? { kind: st.pipeline.scope.kind, id: st.pipeline.scope.id } : undefined),
|
|
182
|
+
variables: spec.variables,
|
|
183
|
+
facts: spec.target.attestedFacts,
|
|
184
|
+
stored: st,
|
|
185
|
+
version: spec.target.version,
|
|
186
|
+
trigger: spec.trigger,
|
|
187
|
+
schedule: spec.schedule,
|
|
188
|
+
});
|
|
189
|
+
},
|
|
190
|
+
checkPipeline: (tenant, id) => {
|
|
191
|
+
const st = this.pipelines.get(id);
|
|
192
|
+
if (!st || st.pipeline.tenant !== tenant)
|
|
193
|
+
throw new ConnectError(`schedule.target.pipeline_id: no pipeline ${id} of the tenant`, Code.NotFound);
|
|
194
|
+
},
|
|
147
195
|
runs: () => this.runs.values(),
|
|
148
196
|
runMessage: (id) => this.runMessage(this.run(id)),
|
|
149
|
-
checkHandler: (
|
|
150
|
-
}, this.clock
|
|
197
|
+
checkHandler: (handler) => this.requireHandler(handler),
|
|
198
|
+
}, this.clock);
|
|
151
199
|
this.timer = setInterval(() => this.tick(), this.opts.tickMs);
|
|
152
200
|
this.timer.unref();
|
|
153
201
|
}
|
|
@@ -252,7 +300,7 @@ export class FakePipelines {
|
|
|
252
300
|
completeNode(runId, nodeId, result = {}) {
|
|
253
301
|
const run = this.run(runId);
|
|
254
302
|
const node = run.nodes.get(nodeId);
|
|
255
|
-
if (!node || node.handler)
|
|
303
|
+
if (!node || node.handler || node.pipeline)
|
|
256
304
|
throw new Error(`FakePipelines.completeNode: ${nodeId} is not an external node of ${runId}`);
|
|
257
305
|
if (node.state !== NodeState.READY)
|
|
258
306
|
throw new Error(`FakePipelines.completeNode: ${nodeId} is not ready`);
|
|
@@ -459,6 +507,7 @@ export class FakePipelines {
|
|
|
459
507
|
});
|
|
460
508
|
router.service(QueuesService, this.queuesService());
|
|
461
509
|
router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
|
|
510
|
+
router.service(PipelinesService, this.pipelines.service());
|
|
462
511
|
router.service(RunsService, {
|
|
463
512
|
listIssues: (req, ctx) => {
|
|
464
513
|
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
@@ -489,10 +538,11 @@ export class FakePipelines {
|
|
|
489
538
|
createJob: (req, ctx) => {
|
|
490
539
|
const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
|
|
491
540
|
this.actsFor(who, req.tenant);
|
|
492
|
-
const existing = this.idempotency.get(`${
|
|
541
|
+
const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
|
|
493
542
|
if (existing)
|
|
494
543
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
495
544
|
const run = this.createJobRun(who, {
|
|
545
|
+
fairnessKey: req.fairnessKey,
|
|
496
546
|
tenant: req.tenant,
|
|
497
547
|
handler: req.handler,
|
|
498
548
|
key: req.idempotencyKey,
|
|
@@ -515,23 +565,43 @@ export class FakePipelines {
|
|
|
515
565
|
createRun: (req, ctx) => {
|
|
516
566
|
const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
|
|
517
567
|
this.actsFor(who, req.tenant);
|
|
518
|
-
const existing = this.idempotency.get(`${
|
|
568
|
+
const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
|
|
519
569
|
if (existing)
|
|
520
570
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
571
|
+
const start = { tenant: req.tenant, key: req.idempotencyKey, resource: req.resource, variables: req.variables, facts: req.attestedFacts };
|
|
572
|
+
let run;
|
|
573
|
+
switch (req.source.case) {
|
|
574
|
+
case "inlineDefinition": {
|
|
575
|
+
const about = req.resource ? { kind: req.resource.kind, id: req.resource.id } : undefined;
|
|
576
|
+
const def = this.freeze(req.source.value, { tenant: req.tenant, scope: about, about });
|
|
577
|
+
run = this.startSnapshot(who, { ...start, snapshot: this.pipelines.inlineSnapshot(req.tenant, def) });
|
|
578
|
+
break;
|
|
579
|
+
}
|
|
580
|
+
case "snapshotId": {
|
|
581
|
+
const snap = this.pipelines.snapshot(req.source.value);
|
|
582
|
+
if (!snap || snap.tenant !== req.tenant)
|
|
583
|
+
throw new ConnectError("no such snapshot", Code.NotFound);
|
|
584
|
+
run = this.startSnapshot(who, { ...start, snapshot: snap });
|
|
585
|
+
break;
|
|
586
|
+
}
|
|
587
|
+
case "pipeline": {
|
|
588
|
+
const src = req.source.value;
|
|
589
|
+
const sel = src.pipeline;
|
|
590
|
+
const st = sel.case === "pipelineId"
|
|
591
|
+
? this.pipelines.get(sel.value)
|
|
592
|
+
: sel.case === "byName" && sel.value.tenant === req.tenant
|
|
593
|
+
? this.pipelines.find(sel.value.tenant, sel.value.scope, sel.value.name)
|
|
594
|
+
: undefined;
|
|
595
|
+
if (!st || st.pipeline.tenant !== req.tenant)
|
|
596
|
+
throw new ConnectError("no such pipeline", Code.NotFound);
|
|
597
|
+
if (!req.resource)
|
|
598
|
+
throw new ConnectError("resource: required", Code.InvalidArgument);
|
|
599
|
+
run = this.startStored(who, { ...start, stored: st, version: src.version, trigger: RunTrigger.API });
|
|
600
|
+
break;
|
|
601
|
+
}
|
|
602
|
+
default:
|
|
603
|
+
throw new ConnectError("source: required", Code.InvalidArgument);
|
|
604
|
+
}
|
|
535
605
|
return { run: this.runMessage(run), deduplicated: false };
|
|
536
606
|
},
|
|
537
607
|
getRun: (req, ctx) => {
|
|
@@ -541,7 +611,11 @@ export class FakePipelines {
|
|
|
541
611
|
getRunGraph: (req, ctx) => {
|
|
542
612
|
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
543
613
|
const run = this.visible(who, req.runId);
|
|
544
|
-
return {
|
|
614
|
+
return {
|
|
615
|
+
run: this.runMessage(run),
|
|
616
|
+
nodes: [...run.nodes.values()].map((n) => this.nodeMessage(n)),
|
|
617
|
+
...(run.definition ? { definition: run.definition } : {}),
|
|
618
|
+
};
|
|
545
619
|
},
|
|
546
620
|
listRuns: (req, ctx) => {
|
|
547
621
|
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
@@ -681,13 +755,13 @@ export class FakePipelines {
|
|
|
681
755
|
return out;
|
|
682
756
|
}
|
|
683
757
|
// ---- the scheduler ----------------------------------------------------
|
|
684
|
-
newRun(who, tenant, key, resource, variables, delayMs) {
|
|
758
|
+
newRun(who, tenant, key, resource, variables, delayMs, extra = {}) {
|
|
685
759
|
const now = Date.now();
|
|
686
760
|
const run = {
|
|
687
761
|
id: randomUUID(),
|
|
688
762
|
tenant,
|
|
689
|
-
product: productOf(who),
|
|
690
763
|
idempotencyKey: key,
|
|
764
|
+
fairnessKey: "",
|
|
691
765
|
createdBy: `urn:fake:sa:${who.tenant}`,
|
|
692
766
|
resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
|
|
693
767
|
variables,
|
|
@@ -709,9 +783,19 @@ export class FakePipelines {
|
|
|
709
783
|
scheduleTarget: "",
|
|
710
784
|
issues: [],
|
|
711
785
|
issuesTruncated: false,
|
|
786
|
+
attestedFacts: undefined,
|
|
787
|
+
snapshotId: "",
|
|
788
|
+
origin: RunOrigin.INLINE,
|
|
789
|
+
pipelineId: "",
|
|
790
|
+
pipelineVersion: 0,
|
|
791
|
+
parentRunId: "",
|
|
792
|
+
parentNodeId: "",
|
|
793
|
+
definition: undefined,
|
|
794
|
+
maxConcurrentRuns: 0,
|
|
795
|
+
...extra,
|
|
712
796
|
};
|
|
713
797
|
this.runs.set(run.id, run);
|
|
714
|
-
this.idempotency.set(`${run.
|
|
798
|
+
this.idempotency.set(`${run.tenant}\u0000${key}`, run.id);
|
|
715
799
|
return run;
|
|
716
800
|
}
|
|
717
801
|
addNode(run, spec) {
|
|
@@ -728,7 +812,170 @@ export class FakePipelines {
|
|
|
728
812
|
retry: spec.retry ?? { max: this.opts.defaultRetries, when: [] },
|
|
729
813
|
timeoutMs: spec.timeoutMs || this.opts.defaultTimeoutMs,
|
|
730
814
|
interruptible: spec.interruptible,
|
|
815
|
+
pipeline: spec.pipeline,
|
|
816
|
+
forEach: spec.forEach,
|
|
817
|
+
elements: undefined,
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
// ---- definitions, snapshots and starts ---------------------------------
|
|
821
|
+
/**
|
|
822
|
+
* Checks a definition as CreateRun does and freezes it as its snapshot
|
|
823
|
+
* holds it: the policy's node defaults filled, each sub-pipeline pinned to
|
|
824
|
+
* the version it names (0, the latest now), a child of another tenant, an
|
|
825
|
+
* unpublished or archived one, one not taking runs about `env.about`, a
|
|
826
|
+
* cycle back to `env.self` or nesting over 3 refused.
|
|
827
|
+
*/
|
|
828
|
+
freeze(def, env) {
|
|
829
|
+
if (def.nodes.length === 0)
|
|
830
|
+
throw new ConnectError("definition: no nodes", Code.InvalidArgument);
|
|
831
|
+
const out = clone(DefinitionSchema, def);
|
|
832
|
+
for (const n of out.nodes) {
|
|
833
|
+
if (!n.retry && env.defaultRetry)
|
|
834
|
+
n.retry = clone(RetrySchema, env.defaultRetry);
|
|
835
|
+
if (!n.timeout && env.defaultTimeout)
|
|
836
|
+
n.timeout = clone(DurationSchema, env.defaultTimeout);
|
|
837
|
+
if (n.action.case === "pipeline") {
|
|
838
|
+
const ref = n.action.value;
|
|
839
|
+
const st = ref.pipelineId ? this.pipelines.get(ref.pipelineId) : this.pipelines.find(env.tenant, ref.scope ?? env.scope, ref.name);
|
|
840
|
+
const label = ref.pipelineId || ref.name;
|
|
841
|
+
if (!st || st.pipeline.tenant !== env.tenant)
|
|
842
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": no pipeline ${label} of this tenant`, Code.InvalidArgument);
|
|
843
|
+
const v = this.pipelines.version(st, ref.version);
|
|
844
|
+
if (st.pipeline.latestVersion === 0 || !v)
|
|
845
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} has no published version ${ref.version || ""}`.trimEnd(), Code.InvalidArgument);
|
|
846
|
+
if (st.pipeline.state === PipelineState.ARCHIVED)
|
|
847
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} is archived`, Code.InvalidArgument);
|
|
848
|
+
if (env.about)
|
|
849
|
+
this.checkChild(n.id, st, env.about);
|
|
850
|
+
if (env.self && (st.pipeline.id === env.self || this.reaches(this.pipelines.snapshot(v.snapshotId).definition, env.self))) {
|
|
851
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} reaches this pipeline again, a cycle`, Code.InvalidArgument);
|
|
852
|
+
}
|
|
853
|
+
const child = this.pipelines.snapshot(v.snapshotId).definition;
|
|
854
|
+
for (const o of ref.outputs) {
|
|
855
|
+
if (!child.nodes.some((c) => c.id === o.node))
|
|
856
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": output ${o.name} names node ${o.node}, which pipeline ${st.pipeline.id} does not have`, Code.InvalidArgument);
|
|
857
|
+
}
|
|
858
|
+
const declared = new Map((v.spec?.variables ?? []).map((x) => [x.name, x]));
|
|
859
|
+
for (const k of Object.keys(n.with)) {
|
|
860
|
+
if (!declared.has(k))
|
|
861
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": ${JSON.stringify(k)} is not a variable pipeline ${st.pipeline.id} declares`, Code.InvalidArgument);
|
|
862
|
+
}
|
|
863
|
+
for (const x of declared.values()) {
|
|
864
|
+
if (x.required && !x.default && !(x.name in n.with))
|
|
865
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": variable ${JSON.stringify(x.name)} of pipeline ${st.pipeline.id} is required`, Code.InvalidArgument);
|
|
866
|
+
}
|
|
867
|
+
n.action.value = { ...ref, pipelineId: st.pipeline.id, version: v.version };
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
this.compile(out);
|
|
871
|
+
const depth = this.depth(out);
|
|
872
|
+
if (depth > MAX_DEPTH)
|
|
873
|
+
throw new ConnectError(`the definition is invalid: it nests ${depth} deep, and runs nest at most ${MAX_DEPTH} deep counting this one`, Code.InvalidArgument);
|
|
874
|
+
return out;
|
|
875
|
+
}
|
|
876
|
+
/** A frozen definition's nodes, checked: handlers' inputs, the graph, for_each's rules. */
|
|
877
|
+
compile(def) {
|
|
878
|
+
const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
|
|
879
|
+
for (const sp of specs)
|
|
880
|
+
if (sp.handler)
|
|
881
|
+
sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
|
|
882
|
+
checkGraph(specs);
|
|
883
|
+
return specs;
|
|
884
|
+
}
|
|
885
|
+
/** How deep runs of a frozen definition nest through its pinned sub-pipelines, counting its own. */
|
|
886
|
+
depth(def) {
|
|
887
|
+
let most = 0;
|
|
888
|
+
for (const n of def.nodes) {
|
|
889
|
+
if (n.action.case !== "pipeline")
|
|
890
|
+
continue;
|
|
891
|
+
const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
|
|
892
|
+
if (snap)
|
|
893
|
+
most = Math.max(most, this.depth(snap.definition));
|
|
894
|
+
}
|
|
895
|
+
return 1 + most;
|
|
896
|
+
}
|
|
897
|
+
/** Whether a frozen definition reaches pipeline `id` through its pinned sub-pipelines. */
|
|
898
|
+
reaches(def, id) {
|
|
899
|
+
return def.nodes.some((n) => {
|
|
900
|
+
if (n.action.case !== "pipeline")
|
|
901
|
+
return false;
|
|
902
|
+
if (n.action.value.pipelineId === id)
|
|
903
|
+
return true;
|
|
904
|
+
const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
|
|
905
|
+
return snap ? this.reaches(snap.definition, id) : false;
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
childSnapshot(pipelineId, version) {
|
|
909
|
+
const st = this.pipelines.get(pipelineId);
|
|
910
|
+
const v = st && this.pipelines.version(st, version);
|
|
911
|
+
return v ? this.pipelines.snapshot(v.snapshotId) : undefined;
|
|
912
|
+
}
|
|
913
|
+
/** A sub-pipeline's child must take runs about the parent's resource, as if started directly. */
|
|
914
|
+
checkChild(nodeId, st, about) {
|
|
915
|
+
if (!takesRunsAbout(st.pipeline, about)) {
|
|
916
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": pipeline ${st.pipeline.id} (${st.pipeline.name}) takes runs about its scope or a resource of a kind in its resource_kinds, not ${JSON.stringify(about?.kind ?? "")}`, Code.InvalidArgument);
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
/** Every pinned child, at any depth, against the resource a run is about. */
|
|
920
|
+
checkChildren(def, about) {
|
|
921
|
+
for (const n of def.nodes) {
|
|
922
|
+
if (n.action.case !== "pipeline")
|
|
923
|
+
continue;
|
|
924
|
+
const st = this.pipelines.get(n.action.value.pipelineId);
|
|
925
|
+
this.checkChild(n.id, st, about);
|
|
926
|
+
const snap = this.childSnapshot(st.pipeline.id, n.action.value.version);
|
|
927
|
+
if (snap)
|
|
928
|
+
this.checkChildren(snap.definition, about);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
/** Starts a run of a snapshot: an inline definition's, an earlier one by id, or a version's. */
|
|
932
|
+
startSnapshot(who, spec) {
|
|
933
|
+
this.checkChildren(spec.snapshot.definition, spec.resource);
|
|
934
|
+
const specs = this.compile(spec.snapshot.definition);
|
|
935
|
+
const run = this.newRun(who, spec.tenant, spec.key, spec.resource, spec.variables ?? {}, 0, {
|
|
936
|
+
attestedFacts: spec.facts,
|
|
937
|
+
snapshotId: spec.snapshot.id,
|
|
938
|
+
origin: spec.snapshot.origin,
|
|
939
|
+
definition: spec.snapshot.definition,
|
|
940
|
+
...spec.extra,
|
|
731
941
|
});
|
|
942
|
+
for (const s of specs)
|
|
943
|
+
this.addNode(run, s);
|
|
944
|
+
this.advance(run);
|
|
945
|
+
return run;
|
|
946
|
+
}
|
|
947
|
+
/** Starts a version of a stored pipeline: CreateRun's `pipeline` source, a schedule's target, a sub-pipeline's child. */
|
|
948
|
+
startStored(who, spec) {
|
|
949
|
+
const { version, snapshot, variables } = this.pipelines.resolveStart(spec.stored, spec.version, spec.resource, spec.variables);
|
|
950
|
+
const run = this.startSnapshot(who, {
|
|
951
|
+
...spec,
|
|
952
|
+
variables,
|
|
953
|
+
snapshot,
|
|
954
|
+
extra: {
|
|
955
|
+
pipelineId: spec.stored.pipeline.id,
|
|
956
|
+
pipelineVersion: version.version,
|
|
957
|
+
maxConcurrentRuns: version.spec?.policy?.maxConcurrentRuns ?? 0,
|
|
958
|
+
trigger: spec.trigger,
|
|
959
|
+
...(spec.parent ? { parentRunId: spec.parent.runId, parentNodeId: spec.parent.nodeId } : {}),
|
|
960
|
+
},
|
|
961
|
+
});
|
|
962
|
+
if (spec.schedule) {
|
|
963
|
+
run.scheduleId = spec.schedule.id;
|
|
964
|
+
run.scheduledFor = spec.schedule.scheduledFor;
|
|
965
|
+
run.scheduleTarget = spec.schedule.target;
|
|
966
|
+
}
|
|
967
|
+
return run;
|
|
968
|
+
}
|
|
969
|
+
/** Whether a run of a pipeline must wait for its policy's `max_concurrent_runs`. */
|
|
970
|
+
atCap(run) {
|
|
971
|
+
if (run.maxConcurrentRuns <= 0)
|
|
972
|
+
return false;
|
|
973
|
+
let active = 0;
|
|
974
|
+
for (const r of this.runs.values()) {
|
|
975
|
+
if (r.id !== run.id && r.pipelineId === run.pipelineId && r.state !== RunState.QUEUED && !terminal(r.state))
|
|
976
|
+
active++;
|
|
977
|
+
}
|
|
978
|
+
return active >= run.maxConcurrentRuns;
|
|
732
979
|
}
|
|
733
980
|
/** Makes ready what can be, skips what cannot run, and settles the run. */
|
|
734
981
|
advance(run) {
|
|
@@ -736,6 +983,9 @@ export class FakePipelines {
|
|
|
736
983
|
return;
|
|
737
984
|
if (run.notBefore > Date.now() && run.cancelRequestedAt === undefined)
|
|
738
985
|
return;
|
|
986
|
+
// A stored pipeline's run waits QUEUED under its policy's cap (the tick looks again).
|
|
987
|
+
if (run.state === RunState.QUEUED && run.cancelRequestedAt === undefined && this.atCap(run))
|
|
988
|
+
return;
|
|
739
989
|
for (let changed = true; changed;) {
|
|
740
990
|
changed = false;
|
|
741
991
|
for (const n of run.nodes.values()) {
|
|
@@ -755,16 +1005,7 @@ export class FakePipelines {
|
|
|
755
1005
|
run.startedAt ??= Date.now();
|
|
756
1006
|
if (run.state === RunState.QUEUED)
|
|
757
1007
|
run.state = RunState.RUNNING;
|
|
758
|
-
|
|
759
|
-
const inputs = this.resolve(run, n);
|
|
760
|
-
if (typeof inputs === "string") {
|
|
761
|
-
n.state = NodeState.FAILED;
|
|
762
|
-
n.failureClass = FailureClass.INVALID_IO;
|
|
763
|
-
}
|
|
764
|
-
else {
|
|
765
|
-
this.offer(run, n, inputs, 0);
|
|
766
|
-
}
|
|
767
|
-
}
|
|
1008
|
+
this.dispatch(run, n);
|
|
768
1009
|
changed = true;
|
|
769
1010
|
}
|
|
770
1011
|
}
|
|
@@ -787,36 +1028,205 @@ export class FakePipelines {
|
|
|
787
1028
|
this.runFinished(run);
|
|
788
1029
|
}
|
|
789
1030
|
}
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
1031
|
+
/**
|
|
1032
|
+
* A node's `with` for one attempt (of one element): whole-value
|
|
1033
|
+
* references, also inside lists and objects, resolved. A string is why
|
|
1034
|
+
* one does not resolve.
|
|
1035
|
+
*/
|
|
1036
|
+
resolve(run, n, el) {
|
|
1037
|
+
let failed = "";
|
|
1038
|
+
const walk = (v) => {
|
|
1039
|
+
if (Array.isArray(v))
|
|
1040
|
+
return v.map(walk);
|
|
1041
|
+
if (v !== null && typeof v === "object")
|
|
1042
|
+
return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
|
|
793
1043
|
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
794
|
-
if (!m)
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
1044
|
+
if (!m)
|
|
1045
|
+
return v;
|
|
1046
|
+
const value = this.lookup(run, m[1], el);
|
|
1047
|
+
if (value === undefined)
|
|
1048
|
+
failed ||= `${m[1]} does not resolve`;
|
|
1049
|
+
return value ?? null;
|
|
1050
|
+
};
|
|
1051
|
+
const out = walk(n.with);
|
|
1052
|
+
return failed || out;
|
|
1053
|
+
}
|
|
1054
|
+
/** One reference's value: an upstream kv output, a variable, or the element. */
|
|
1055
|
+
lookup(run, expr, el) {
|
|
1056
|
+
const path = expr.split(".");
|
|
1057
|
+
if ((path[0] === "steps" || path[0] === "needs") && path[2] === "outputs" && path[3] === "kv" && path[4])
|
|
1058
|
+
return run.nodes.get(path[1])?.kv[path[4]];
|
|
1059
|
+
if (path[0] === "variables" && path[1])
|
|
1060
|
+
return run.variables[path[1]];
|
|
1061
|
+
if (el && path[0] === "item_index" && path.length === 1)
|
|
1062
|
+
return el.index;
|
|
1063
|
+
if (el && path[0] === "item") {
|
|
1064
|
+
let v = el.item;
|
|
1065
|
+
for (const k of path.slice(1))
|
|
1066
|
+
v = v !== null && typeof v === "object" && !Array.isArray(v) ? v[k] : undefined;
|
|
1067
|
+
return v;
|
|
1068
|
+
}
|
|
1069
|
+
return undefined;
|
|
1070
|
+
}
|
|
1071
|
+
/** A node made ready: its attempt (or child run), its elements, or, for an external node, nothing yet. */
|
|
1072
|
+
dispatch(run, n) {
|
|
1073
|
+
if (n.forEach) {
|
|
1074
|
+
const items = this.items(run, n);
|
|
1075
|
+
if (typeof items === "string") {
|
|
1076
|
+
n.state = NodeState.FAILED;
|
|
1077
|
+
n.failureClass = FailureClass.INVALID_IO;
|
|
1078
|
+
return;
|
|
804
1079
|
}
|
|
805
|
-
|
|
806
|
-
|
|
1080
|
+
n.elements = items.map((item, index) => ({ index, item, state: "pending", failureClass: FailureClass.UNSPECIFIED, kv: {} }));
|
|
1081
|
+
n.state = NodeState.RUNNING;
|
|
1082
|
+
this.launchElements(run, n);
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
1085
|
+
if (!n.handler && !n.pipeline)
|
|
1086
|
+
return;
|
|
1087
|
+
const inputs = this.resolve(run, n);
|
|
1088
|
+
if (typeof inputs === "string") {
|
|
1089
|
+
n.state = NodeState.FAILED;
|
|
1090
|
+
n.failureClass = FailureClass.INVALID_IO;
|
|
1091
|
+
}
|
|
1092
|
+
else {
|
|
1093
|
+
this.launch(run, n, inputs, undefined, 0);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
/** A for_each node's list: a JSON array, or a reference resolving to one, within `max_items`. */
|
|
1097
|
+
items(run, n) {
|
|
1098
|
+
const fe = n.forEach;
|
|
1099
|
+
const m = typeof fe.items === "string" ? REF.exec(fe.items) : null;
|
|
1100
|
+
const v = m ? this.lookup(run, m[1], undefined) : fe.items;
|
|
1101
|
+
if (!Array.isArray(v))
|
|
1102
|
+
return "for_each.items does not resolve to a list";
|
|
1103
|
+
if (v.length > (fe.maxItems || 1000))
|
|
1104
|
+
return `for_each.items has ${v.length} elements, over max_items`;
|
|
1105
|
+
return v;
|
|
1106
|
+
}
|
|
1107
|
+
/** One attempt of a node (of one element): offered to a worker, or a sub-pipeline's child run started. */
|
|
1108
|
+
launch(run, n, inputs, el, delayMs) {
|
|
1109
|
+
if (n.pipeline)
|
|
1110
|
+
this.startChild(run, n, inputs, el);
|
|
1111
|
+
else
|
|
1112
|
+
this.offer(run, n, inputs, delayMs, el);
|
|
1113
|
+
}
|
|
1114
|
+
/**
|
|
1115
|
+
* Starts elements while `max_parallel` has room, and settles the node
|
|
1116
|
+
* once none is live and none will start: under FAIL_FAST nothing starts
|
|
1117
|
+
* after a failure.
|
|
1118
|
+
*/
|
|
1119
|
+
launchElements(run, n) {
|
|
1120
|
+
const fe = n.forEach;
|
|
1121
|
+
const els = n.elements;
|
|
1122
|
+
const policy = fe.onFailure || ForEachFailure.FAIL_FAST;
|
|
1123
|
+
const stopped = () => run.cancelRequestedAt !== undefined || (policy === ForEachFailure.FAIL_FAST && els.some((e) => e.state === "failed"));
|
|
1124
|
+
let live = els.filter((e) => e.state === "running").length;
|
|
1125
|
+
for (const el of els) {
|
|
1126
|
+
if (stopped() || live >= (fe.maxParallel || 10))
|
|
1127
|
+
break;
|
|
1128
|
+
if (el.state !== "pending")
|
|
1129
|
+
continue;
|
|
1130
|
+
const inputs = this.resolve(run, n, el);
|
|
1131
|
+
if (typeof inputs === "string") {
|
|
1132
|
+
el.state = "failed";
|
|
1133
|
+
el.failureClass = FailureClass.INVALID_IO;
|
|
1134
|
+
continue;
|
|
807
1135
|
}
|
|
808
|
-
|
|
809
|
-
|
|
1136
|
+
el.state = "running";
|
|
1137
|
+
live++;
|
|
1138
|
+
this.launch(run, n, inputs, el, 0);
|
|
1139
|
+
}
|
|
1140
|
+
if (live > 0 || (!stopped() && els.some((e) => e.state === "pending")))
|
|
1141
|
+
return;
|
|
1142
|
+
// Settled: outputs as lists in element order, null where an element gave none.
|
|
1143
|
+
const slots = [...new Set(els.flatMap((e) => (e.state === "succeeded" ? Object.keys(e.kv) : [])))];
|
|
1144
|
+
n.kv = Object.fromEntries(slots.map((k) => [k, els.map((e) => (e.state === "succeeded" ? (e.kv[k] ?? null) : null))]));
|
|
1145
|
+
const failed = els.find((e) => e.state === "failed");
|
|
1146
|
+
if (run.cancelRequestedAt !== undefined) {
|
|
1147
|
+
n.state = NodeState.CANCELLED;
|
|
1148
|
+
n.failureClass = FailureClass.CANCELLED;
|
|
1149
|
+
}
|
|
1150
|
+
else if (failed && policy !== ForEachFailure.ALLOW) {
|
|
1151
|
+
n.state = NodeState.FAILED;
|
|
1152
|
+
n.failureClass = failed.failureClass;
|
|
1153
|
+
}
|
|
1154
|
+
else {
|
|
1155
|
+
n.state = NodeState.SUCCEEDED;
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
/** A sub-pipeline node's attempt: one child run of the pinned version, about the parent's resource. */
|
|
1159
|
+
startChild(run, n, inputs, el) {
|
|
1160
|
+
const a = this.newAttempt(run, n, inputs, 0, el);
|
|
1161
|
+
a.state = AttemptState.RUNNING;
|
|
1162
|
+
a.leaseExpiresAt = Number.POSITIVE_INFINITY; // held by the platform, not a worker
|
|
1163
|
+
a.startedAt = Date.now();
|
|
1164
|
+
n.attempts.push(a);
|
|
1165
|
+
this.attemptsById.set(a.id, a);
|
|
1166
|
+
if (!n.forEach)
|
|
1167
|
+
n.state = NodeState.RUNNING;
|
|
1168
|
+
try {
|
|
1169
|
+
const child = this.startStored({ tenant: run.tenant, scopes: ALL_SCOPES }, {
|
|
1170
|
+
tenant: run.tenant,
|
|
1171
|
+
key: `parent:${run.id}:${n.id}:${a.id}`,
|
|
1172
|
+
resource: run.resource,
|
|
1173
|
+
variables: inputs,
|
|
1174
|
+
facts: run.attestedFacts,
|
|
1175
|
+
stored: this.pipelines.get(n.pipeline.id),
|
|
1176
|
+
version: n.pipeline.version,
|
|
1177
|
+
trigger: RunTrigger.PARENT,
|
|
1178
|
+
parent: { runId: run.id, nodeId: n.id },
|
|
1179
|
+
});
|
|
1180
|
+
a.childRunId = child.id;
|
|
1181
|
+
}
|
|
1182
|
+
catch (err) {
|
|
1183
|
+
// Its variables, resolved now, refused: the attempt fails, as the child would.
|
|
1184
|
+
a.state = AttemptState.FAILED;
|
|
1185
|
+
a.failureClass = FailureClass.DOWNSTREAM_FAILED;
|
|
1186
|
+
a.message = ConnectError.from(err).rawMessage;
|
|
1187
|
+
a.completedAt = Date.now();
|
|
1188
|
+
queueMicrotask(() => this.settle(a));
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
/** A child run ended: its parent's attempt takes its result. */
|
|
1192
|
+
childFinished(child) {
|
|
1193
|
+
const parent = this.runs.get(child.parentRunId);
|
|
1194
|
+
const n = parent?.nodes.get(child.parentNodeId);
|
|
1195
|
+
const a = n?.attempts.find((x) => x.childRunId === child.id);
|
|
1196
|
+
if (!n || !a || a.state !== AttemptState.RUNNING)
|
|
1197
|
+
return;
|
|
1198
|
+
a.completedAt = Date.now();
|
|
1199
|
+
if (child.state === RunState.SUCCEEDED) {
|
|
1200
|
+
a.state = AttemptState.SUCCEEDED;
|
|
1201
|
+
for (const o of n.pipeline.outputs) {
|
|
1202
|
+
const v = child.nodes.get(o.node)?.kv[o.slot];
|
|
1203
|
+
if (v !== undefined)
|
|
1204
|
+
a.kv[o.name] = v;
|
|
810
1205
|
}
|
|
811
1206
|
}
|
|
812
|
-
|
|
1207
|
+
else if (child.state === RunState.CANCELLED) {
|
|
1208
|
+
a.state = AttemptState.CANCELLED;
|
|
1209
|
+
a.failureClass = FailureClass.CANCELLED;
|
|
1210
|
+
}
|
|
1211
|
+
else {
|
|
1212
|
+
a.state = AttemptState.FAILED;
|
|
1213
|
+
a.failureClass = FailureClass.DOWNSTREAM_FAILED;
|
|
1214
|
+
a.message = `the child run ${child.id} failed`;
|
|
1215
|
+
}
|
|
1216
|
+
this.settle(a);
|
|
1217
|
+
}
|
|
1218
|
+
offer(run, n, inputs, delayMs, el) {
|
|
1219
|
+
const a = this.newAttempt(run, n, inputs, delayMs, el);
|
|
1220
|
+
n.attempts.push(a);
|
|
1221
|
+
this.attemptsById.set(a.id, a);
|
|
1222
|
+
this.notify();
|
|
813
1223
|
}
|
|
814
|
-
|
|
815
|
-
|
|
1224
|
+
newAttempt(run, n, inputs, delayMs, el) {
|
|
1225
|
+
return {
|
|
816
1226
|
id: randomUUID(),
|
|
817
1227
|
runId: run.id,
|
|
818
1228
|
nodeId: n.id,
|
|
819
|
-
number: n.attempts.length + 1,
|
|
1229
|
+
number: n.attempts.filter((x) => x.itemIndex === el?.index).length + 1,
|
|
820
1230
|
state: AttemptState.OFFERED,
|
|
821
1231
|
epoch: 0n,
|
|
822
1232
|
workerId: "",
|
|
@@ -838,10 +1248,10 @@ export class FakePipelines {
|
|
|
838
1248
|
offeredAt: Date.now(),
|
|
839
1249
|
progress: undefined,
|
|
840
1250
|
spans: [],
|
|
1251
|
+
itemIndex: el?.index,
|
|
1252
|
+
item: el?.item,
|
|
1253
|
+
childRunId: "",
|
|
841
1254
|
};
|
|
842
|
-
n.attempts.push(a);
|
|
843
|
-
this.attemptsById.set(a.id, a);
|
|
844
|
-
this.notify();
|
|
845
1255
|
}
|
|
846
1256
|
/**
|
|
847
1257
|
* Claims for a worker: the attempts of handler nodes of full runs first,
|
|
@@ -862,7 +1272,7 @@ export class FakePipelines {
|
|
|
862
1272
|
if (a.state !== AttemptState.OFFERED || a.notBefore > now)
|
|
863
1273
|
continue;
|
|
864
1274
|
if (run.queueId)
|
|
865
|
-
queued.push({ item: a, queueId: run.queueId,
|
|
1275
|
+
queued.push({ item: a, queueId: run.queueId, lane: run.fairnessKey, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
|
|
866
1276
|
else if (out.length < max)
|
|
867
1277
|
out.push(a);
|
|
868
1278
|
}
|
|
@@ -889,10 +1299,10 @@ export class FakePipelines {
|
|
|
889
1299
|
return out;
|
|
890
1300
|
}
|
|
891
1301
|
/** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
|
|
892
|
-
queueRunning(queueId,
|
|
1302
|
+
queueRunning(queueId, lane) {
|
|
893
1303
|
let n = 0;
|
|
894
1304
|
for (const run of this.runs.values()) {
|
|
895
|
-
if (run.queueId !== queueId || (
|
|
1305
|
+
if (run.queueId !== queueId || (lane !== undefined && run.fairnessKey !== lane))
|
|
896
1306
|
continue;
|
|
897
1307
|
for (const node of run.nodes.values())
|
|
898
1308
|
for (const a of node.attempts)
|
|
@@ -942,13 +1352,28 @@ export class FakePipelines {
|
|
|
942
1352
|
settle(a) {
|
|
943
1353
|
const run = this.run(a.runId);
|
|
944
1354
|
const n = run.nodes.get(a.nodeId);
|
|
1355
|
+
// An element's attempt settles its element; the node settles with the last one.
|
|
1356
|
+
const el = a.itemIndex !== undefined ? n.elements?.[a.itemIndex] : undefined;
|
|
1357
|
+
const mine = n.attempts.filter((x) => x.itemIndex === a.itemIndex);
|
|
945
1358
|
if (a.state === AttemptState.SUCCEEDED) {
|
|
946
|
-
|
|
947
|
-
|
|
1359
|
+
if (el) {
|
|
1360
|
+
el.state = "succeeded";
|
|
1361
|
+
el.kv = a.kv;
|
|
1362
|
+
}
|
|
1363
|
+
else {
|
|
1364
|
+
n.state = NodeState.SUCCEEDED;
|
|
1365
|
+
n.kv = a.kv;
|
|
1366
|
+
}
|
|
948
1367
|
}
|
|
949
1368
|
else if (a.state === AttemptState.CANCELLED || run.cancelRequestedAt !== undefined) {
|
|
950
|
-
|
|
951
|
-
|
|
1369
|
+
if (el) {
|
|
1370
|
+
el.state = "failed";
|
|
1371
|
+
el.failureClass = FailureClass.CANCELLED;
|
|
1372
|
+
}
|
|
1373
|
+
else {
|
|
1374
|
+
n.state = NodeState.CANCELLED;
|
|
1375
|
+
n.failureClass = FailureClass.CANCELLED;
|
|
1376
|
+
}
|
|
952
1377
|
}
|
|
953
1378
|
else {
|
|
954
1379
|
const permanent = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.PERMANENT;
|
|
@@ -956,24 +1381,32 @@ export class FakePipelines {
|
|
|
956
1381
|
const when = n.retry.when.length > 0 ? n.retry.when : transientHandler ? [...TRANSIENT, FailureClass.HANDLER_ERROR] : TRANSIENT;
|
|
957
1382
|
// On an interruptible node an interruption is free, up to a bound.
|
|
958
1383
|
const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
|
|
959
|
-
const freeUsed =
|
|
1384
|
+
const freeUsed = mine.filter(free).length;
|
|
960
1385
|
if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
|
|
961
1386
|
// Up to the bound, an interruption is offered again at once.
|
|
962
|
-
this.
|
|
1387
|
+
this.launch(run, n, a.inputs, el, 0);
|
|
963
1388
|
return;
|
|
964
1389
|
}
|
|
965
|
-
const retries =
|
|
1390
|
+
const retries = mine.slice(0, -1).filter((x) => !free(x)).length;
|
|
966
1391
|
// As the api: a worker that vanished (ABANDONED) may have had outside
|
|
967
1392
|
// effects, so its node runs again automatically only when it declares
|
|
968
1393
|
// itself interruptible.
|
|
969
1394
|
const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
|
|
970
1395
|
if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
|
|
971
|
-
this.
|
|
1396
|
+
this.launch(run, n, a.inputs, el, this.opts.retryBackoffMs);
|
|
972
1397
|
return;
|
|
973
1398
|
}
|
|
974
|
-
|
|
975
|
-
|
|
1399
|
+
if (el) {
|
|
1400
|
+
el.state = "failed";
|
|
1401
|
+
el.failureClass = a.failureClass;
|
|
1402
|
+
}
|
|
1403
|
+
else {
|
|
1404
|
+
n.state = NodeState.FAILED;
|
|
1405
|
+
n.failureClass = a.failureClass;
|
|
1406
|
+
}
|
|
976
1407
|
}
|
|
1408
|
+
if (el)
|
|
1409
|
+
this.launchElements(run, n);
|
|
977
1410
|
this.advance(run);
|
|
978
1411
|
this.notify();
|
|
979
1412
|
}
|
|
@@ -983,8 +1416,11 @@ export class FakePipelines {
|
|
|
983
1416
|
run.cancelRequestedAt = Date.now();
|
|
984
1417
|
run.cancelReason = reason;
|
|
985
1418
|
run.state = RunState.CANCELLING;
|
|
1419
|
+
const children = [];
|
|
986
1420
|
for (const n of run.nodes.values()) {
|
|
987
1421
|
for (const a of n.attempts) {
|
|
1422
|
+
if (a.state === AttemptState.RUNNING && a.childRunId)
|
|
1423
|
+
children.push(this.run(a.childRunId));
|
|
988
1424
|
if (a.state === AttemptState.OFFERED) {
|
|
989
1425
|
a.state = AttemptState.CANCELLED;
|
|
990
1426
|
a.failureClass = FailureClass.CANCELLED;
|
|
@@ -996,6 +1432,9 @@ export class FakePipelines {
|
|
|
996
1432
|
n.state = n.state === NodeState.PENDING ? NodeState.SKIPPED : NodeState.CANCELLED;
|
|
997
1433
|
}
|
|
998
1434
|
}
|
|
1435
|
+
// Cancelling a run cancels its sub-pipelines' children; each one's end settles its attempt.
|
|
1436
|
+
for (const c of children)
|
|
1437
|
+
this.cancel(c, reason);
|
|
999
1438
|
this.advance(run);
|
|
1000
1439
|
this.notify();
|
|
1001
1440
|
}
|
|
@@ -1040,7 +1479,7 @@ export class FakePipelines {
|
|
|
1040
1479
|
/** QueuesService, as the contract states it. */
|
|
1041
1480
|
queuesService() {
|
|
1042
1481
|
const readable = (who, q) => {
|
|
1043
|
-
if (!q || q.owner?.
|
|
1482
|
+
if (!q || q.owner?.tenant !== who.tenant)
|
|
1044
1483
|
throw new ConnectError("no such queue", Code.NotFound);
|
|
1045
1484
|
return q;
|
|
1046
1485
|
};
|
|
@@ -1049,12 +1488,10 @@ export class FakePipelines {
|
|
|
1049
1488
|
setQueue: (req, ctx) => {
|
|
1050
1489
|
const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
|
|
1051
1490
|
const q = req.queue;
|
|
1052
|
-
if (!q?.owner)
|
|
1053
|
-
throw new ConnectError("queue.owner: required", Code.InvalidArgument);
|
|
1054
|
-
if (q.owner.
|
|
1055
|
-
throw new ConnectError("queue.owner.
|
|
1056
|
-
if (q.owner.tenant && q.owner.tenant !== who.tenant)
|
|
1057
|
-
throw new ConnectError("queue.owner.tenant: the caller does not act for it", Code.PermissionDenied);
|
|
1491
|
+
if (!q?.owner?.tenant)
|
|
1492
|
+
throw new ConnectError("queue.owner.tenant: required", Code.InvalidArgument);
|
|
1493
|
+
if (q.owner.tenant !== who.tenant)
|
|
1494
|
+
throw new ConnectError("queue.owner.tenant: the caller is not a service-account member of it", Code.PermissionDenied);
|
|
1058
1495
|
if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
|
|
1059
1496
|
throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
|
|
1060
1497
|
if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
|
|
@@ -1062,7 +1499,7 @@ export class FakePipelines {
|
|
|
1062
1499
|
}
|
|
1063
1500
|
if (req.updateMask)
|
|
1064
1501
|
throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
|
|
1065
|
-
const existing = [...this.queues.values()].find((x) => x.owner?.
|
|
1502
|
+
const existing = [...this.queues.values()].find((x) => x.owner?.tenant === q.owner.tenant && x.name === q.name);
|
|
1066
1503
|
const now = Date.now();
|
|
1067
1504
|
const next = create(QueueSchema, {
|
|
1068
1505
|
...q,
|
|
@@ -1085,22 +1522,23 @@ export class FakePipelines {
|
|
|
1085
1522
|
if (req.queue.case !== "byName")
|
|
1086
1523
|
throw new ConnectError("queue: required", Code.InvalidArgument);
|
|
1087
1524
|
const { owner, name } = req.queue.value;
|
|
1088
|
-
if (owner?.
|
|
1525
|
+
if (!owner?.tenant || owner.tenant !== who.tenant)
|
|
1089
1526
|
throw new ConnectError("no such queue", Code.NotFound);
|
|
1090
|
-
const found = [...this.queues.values()].find((x) => x.owner?.
|
|
1527
|
+
const found = [...this.queues.values()].find((x) => x.owner?.tenant === owner.tenant && x.name === name);
|
|
1091
1528
|
if (found)
|
|
1092
1529
|
return { queue: readable(who, found) };
|
|
1093
|
-
if (name === "default"
|
|
1094
|
-
return { queue: this.defaultQueue(owner.
|
|
1530
|
+
if (name === "default")
|
|
1531
|
+
return { queue: this.defaultQueue(owner.tenant) };
|
|
1095
1532
|
throw new ConnectError("no such queue", Code.NotFound);
|
|
1096
1533
|
},
|
|
1097
1534
|
listQueues: (req, ctx) => {
|
|
1098
1535
|
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1099
|
-
|
|
1100
|
-
|
|
1536
|
+
if (!req.tenant)
|
|
1537
|
+
throw new ConnectError("tenant: required", Code.InvalidArgument);
|
|
1538
|
+
if (req.tenant !== who.tenant)
|
|
1101
1539
|
return { queues: [], nextCursor: "" };
|
|
1102
1540
|
const queues = [...this.queues.values()]
|
|
1103
|
-
.filter((q) => q.owner?.
|
|
1541
|
+
.filter((q) => q.owner?.tenant === req.tenant)
|
|
1104
1542
|
.sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
|
|
1105
1543
|
return { queues, nextCursor: "" };
|
|
1106
1544
|
},
|
|
@@ -1152,30 +1590,25 @@ export class FakePipelines {
|
|
|
1152
1590
|
let running = 0;
|
|
1153
1591
|
let started = 0n;
|
|
1154
1592
|
let oldest;
|
|
1155
|
-
const tenants = new Map();
|
|
1156
1593
|
for (const r of runs) {
|
|
1157
|
-
const t = tenants.get(r.tenant) ?? { waiting: 0, running: 0 };
|
|
1158
1594
|
for (const a of this.attemptsOf(r.id)) {
|
|
1159
1595
|
if (a.state === AttemptState.OFFERED) {
|
|
1160
1596
|
if (a.notBefore > now || r.notBefore > now)
|
|
1161
1597
|
delayed++;
|
|
1162
1598
|
else {
|
|
1163
1599
|
waiting++;
|
|
1164
|
-
t.waiting++;
|
|
1165
1600
|
const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
|
|
1166
1601
|
oldest = oldest === undefined ? since : Math.min(oldest, since);
|
|
1167
1602
|
}
|
|
1168
1603
|
}
|
|
1169
1604
|
if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
|
|
1170
1605
|
running++;
|
|
1171
|
-
t.running++;
|
|
1172
1606
|
}
|
|
1173
1607
|
if (a.startedAt && a.startedAt >= now - windowMs)
|
|
1174
1608
|
started++;
|
|
1175
1609
|
}
|
|
1176
1610
|
if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
|
|
1177
1611
|
delayed++;
|
|
1178
|
-
tenants.set(r.tenant, t);
|
|
1179
1612
|
}
|
|
1180
1613
|
const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
|
|
1181
1614
|
return {
|
|
@@ -1183,10 +1616,6 @@ export class FakePipelines {
|
|
|
1183
1616
|
delayed,
|
|
1184
1617
|
running,
|
|
1185
1618
|
...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
|
|
1186
|
-
tenants: [...tenants.entries()]
|
|
1187
|
-
.map(([tenant, v]) => ({ tenant, ...v }))
|
|
1188
|
-
.sort((a, b) => b.waiting - a.waiting || b.running - a.running)
|
|
1189
|
-
.slice(0, req.topTenants || 10),
|
|
1190
1619
|
started,
|
|
1191
1620
|
failed,
|
|
1192
1621
|
paused: q.paused,
|
|
@@ -1197,7 +1626,7 @@ export class FakePipelines {
|
|
|
1197
1626
|
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1198
1627
|
const q = byId(who, req.queueId);
|
|
1199
1628
|
const runs = [...this.runs.values()]
|
|
1200
|
-
.filter((r) => r.queueId === q.id && (
|
|
1629
|
+
.filter((r) => r.queueId === q.id && (req.states.length === 0 || req.states.includes(r.state)))
|
|
1201
1630
|
.reverse();
|
|
1202
1631
|
return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
|
|
1203
1632
|
},
|
|
@@ -1211,14 +1640,15 @@ export class FakePipelines {
|
|
|
1211
1640
|
throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
|
|
1212
1641
|
if (spec.retry?.when.includes(FailureClass.CANCELLED))
|
|
1213
1642
|
throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
|
|
1214
|
-
const q = this.resolveQueue(
|
|
1643
|
+
const q = this.resolveQueue(spec.tenant, spec.queue);
|
|
1215
1644
|
if (!q)
|
|
1216
|
-
throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among
|
|
1645
|
+
throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among the tenant's queues`, Code.NotFound);
|
|
1217
1646
|
const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
|
|
1218
1647
|
const payload = this.checkInputs("job", spec.handler, spec.payload);
|
|
1219
1648
|
const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
|
|
1220
1649
|
run.queueId = q.id;
|
|
1221
1650
|
run.queue = q.name;
|
|
1651
|
+
run.fairnessKey = spec.fairnessKey ?? "";
|
|
1222
1652
|
run.trigger = spec.trigger ?? RunTrigger.API;
|
|
1223
1653
|
if (spec.schedule) {
|
|
1224
1654
|
run.scheduleId = spec.schedule.id;
|
|
@@ -1238,22 +1668,21 @@ export class FakePipelines {
|
|
|
1238
1668
|
this.advance(run);
|
|
1239
1669
|
return run;
|
|
1240
1670
|
}
|
|
1241
|
-
/** A job's queue: the tenant's
|
|
1242
|
-
resolveQueue(
|
|
1671
|
+
/** A job's queue: the tenant's of that name; empty is `default`. */
|
|
1672
|
+
resolveQueue(tenant, name) {
|
|
1243
1673
|
const want = name || "default";
|
|
1244
|
-
const
|
|
1245
|
-
const found = all.find((q) => q.owner?.tenant === tenant) ?? all.find((q) => !q.owner?.tenant);
|
|
1674
|
+
const found = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === want);
|
|
1246
1675
|
if (found)
|
|
1247
1676
|
return found;
|
|
1248
|
-
return want === "default" ? this.defaultQueue(
|
|
1677
|
+
return want === "default" ? this.defaultQueue(tenant) : undefined;
|
|
1249
1678
|
}
|
|
1250
|
-
/** The
|
|
1251
|
-
defaultQueue(
|
|
1252
|
-
const existing = [...this.queues.values()].find((q) => q.owner?.
|
|
1679
|
+
/** The tenant's `default` queue, made on first use with no limits. */
|
|
1680
|
+
defaultQueue(tenant) {
|
|
1681
|
+
const existing = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === "default");
|
|
1253
1682
|
if (existing)
|
|
1254
1683
|
return existing;
|
|
1255
1684
|
const now = Date.now();
|
|
1256
|
-
const q = create(QueueSchema, { id: randomUUID(), owner: {
|
|
1685
|
+
const q = create(QueueSchema, { id: randomUUID(), owner: { tenant }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
|
|
1257
1686
|
this.queues.set(q.id, q);
|
|
1258
1687
|
return q;
|
|
1259
1688
|
}
|
|
@@ -1264,7 +1693,7 @@ export class FakePipelines {
|
|
|
1264
1693
|
throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
|
|
1265
1694
|
if (!terminal(run.state))
|
|
1266
1695
|
throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
|
|
1267
|
-
if (!n.handler || n.attempts.length === 0 || !terminalNode(n.state))
|
|
1696
|
+
if ((!n.handler && !n.pipeline) || (n.attempts.length === 0 && !n.forEach) || !terminalNode(n.state))
|
|
1268
1697
|
throw new ConnectError("the node never settled", Code.FailedPrecondition);
|
|
1269
1698
|
run.state = RunState.RUNNING;
|
|
1270
1699
|
run.finishedAt = undefined;
|
|
@@ -1288,8 +1717,27 @@ export class FakePipelines {
|
|
|
1288
1717
|
x.failureClass = FailureClass.UNSPECIFIED;
|
|
1289
1718
|
x.kv = {};
|
|
1290
1719
|
}
|
|
1720
|
+
if (n.forEach && n.elements) {
|
|
1721
|
+
// Only the failed elements run again; the succeeded ones are kept.
|
|
1722
|
+
for (const el of n.elements) {
|
|
1723
|
+
if (el.state === "succeeded")
|
|
1724
|
+
continue;
|
|
1725
|
+
el.state = "pending";
|
|
1726
|
+
el.failureClass = FailureClass.UNSPECIFIED;
|
|
1727
|
+
el.kv = {};
|
|
1728
|
+
}
|
|
1729
|
+
n.state = NodeState.RUNNING;
|
|
1730
|
+
this.launchElements(run, n);
|
|
1731
|
+
this.advance(run);
|
|
1732
|
+
return;
|
|
1733
|
+
}
|
|
1734
|
+
if (n.forEach) {
|
|
1735
|
+
this.dispatch(run, n);
|
|
1736
|
+
this.advance(run);
|
|
1737
|
+
return;
|
|
1738
|
+
}
|
|
1291
1739
|
const inputs = this.resolve(run, n);
|
|
1292
|
-
this.
|
|
1740
|
+
this.launch(run, n, typeof inputs === "string" ? {} : inputs, undefined, 0);
|
|
1293
1741
|
}
|
|
1294
1742
|
// ---- issues, progress and spans ---------------------------------------
|
|
1295
1743
|
/** Every declared stage PENDING from the claim, as the platform does. */
|
|
@@ -1458,6 +1906,9 @@ export class FakePipelines {
|
|
|
1458
1906
|
}
|
|
1459
1907
|
/** Runs whenever a run becomes terminal. */
|
|
1460
1908
|
runFinished(run) {
|
|
1909
|
+
// Deferred, so a parent never settles inside the child's own advance.
|
|
1910
|
+
if (run.parentRunId)
|
|
1911
|
+
queueMicrotask(() => this.childFinished(run));
|
|
1461
1912
|
// The schedules are made after the router; a run cannot finish before.
|
|
1462
1913
|
this.schedules?.runFinished(run);
|
|
1463
1914
|
}
|
|
@@ -1466,8 +1917,8 @@ export class FakePipelines {
|
|
|
1466
1917
|
return create(RunSchema, {
|
|
1467
1918
|
id: r.id,
|
|
1468
1919
|
tenant: r.tenant,
|
|
1469
|
-
product: r.product,
|
|
1470
1920
|
idempotencyKey: r.idempotencyKey,
|
|
1921
|
+
fairnessKey: r.fairnessKey,
|
|
1471
1922
|
...(r.resource ? { resource: r.resource } : {}),
|
|
1472
1923
|
variables: r.variables,
|
|
1473
1924
|
state: r.state,
|
|
@@ -1490,6 +1941,13 @@ export class FakePipelines {
|
|
|
1490
1941
|
issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
|
|
1491
1942
|
issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
|
|
1492
1943
|
issuesTruncated: r.issuesTruncated,
|
|
1944
|
+
...(r.attestedFacts ? { attestedFacts: r.attestedFacts } : {}),
|
|
1945
|
+
snapshotId: r.snapshotId,
|
|
1946
|
+
origin: r.origin,
|
|
1947
|
+
pipelineId: r.pipelineId,
|
|
1948
|
+
pipelineVersion: r.pipelineVersion,
|
|
1949
|
+
parentRunId: r.parentRunId,
|
|
1950
|
+
parentNodeId: r.parentNodeId,
|
|
1493
1951
|
});
|
|
1494
1952
|
}
|
|
1495
1953
|
nodeMessage(n) {
|
|
@@ -1500,6 +1958,18 @@ export class FakePipelines {
|
|
|
1500
1958
|
failureClass: n.failureClass,
|
|
1501
1959
|
settledAttemptId: terminalNode(n.state) ? (settled?.id ?? "") : "",
|
|
1502
1960
|
kv: Object.fromEntries(Object.entries(n.kv).map(([k, v]) => [k, jsonValue(v)])),
|
|
1961
|
+
downstreamRunId: n.attempts.findLast((a) => a.childRunId)?.childRunId ?? "",
|
|
1962
|
+
...(n.elements
|
|
1963
|
+
? {
|
|
1964
|
+
forEach: {
|
|
1965
|
+
total: n.elements.length,
|
|
1966
|
+
succeeded: n.elements.filter((e) => e.state === "succeeded").length,
|
|
1967
|
+
failed: n.elements.filter((e) => e.state === "failed").length,
|
|
1968
|
+
running: n.elements.filter((e) => e.state === "running").length,
|
|
1969
|
+
pending: n.elements.filter((e) => e.state === "pending").length,
|
|
1970
|
+
},
|
|
1971
|
+
}
|
|
1972
|
+
: {}),
|
|
1503
1973
|
attempts: n.attempts.map((a) => create(AttemptSchema, {
|
|
1504
1974
|
id: a.id,
|
|
1505
1975
|
runId: a.runId,
|
|
@@ -1512,6 +1982,8 @@ export class FakePipelines {
|
|
|
1512
1982
|
termination: a.termination,
|
|
1513
1983
|
...(a.exitCode !== undefined ? { exitCode: a.exitCode } : {}),
|
|
1514
1984
|
message: a.message,
|
|
1985
|
+
childRunId: a.childRunId,
|
|
1986
|
+
...(a.itemIndex !== undefined ? { itemIndex: a.itemIndex, item: jsonValue(a.item ?? null) } : {}),
|
|
1515
1987
|
logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
|
|
1516
1988
|
logTruncated: a.logTruncated,
|
|
1517
1989
|
...(a.progress
|
|
@@ -1543,10 +2015,21 @@ export class FakePipelines {
|
|
|
1543
2015
|
});
|
|
1544
2016
|
}
|
|
1545
2017
|
}
|
|
2018
|
+
/** Every whole-value reference in a value, also inside lists and objects. */
|
|
2019
|
+
function refs(v) {
|
|
2020
|
+
if (Array.isArray(v))
|
|
2021
|
+
return v.flatMap(refs);
|
|
2022
|
+
if (v !== null && typeof v === "object")
|
|
2023
|
+
return Object.values(v).flatMap(refs);
|
|
2024
|
+
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
2025
|
+
return m ? [m[1]] : [];
|
|
2026
|
+
}
|
|
2027
|
+
const ITEM_REF = /^item(_index$|$|\.)/;
|
|
1546
2028
|
/**
|
|
1547
2029
|
* Holds a definition to the server's rules the fake models: unique ids,
|
|
1548
|
-
* known needs, no cycle,
|
|
1549
|
-
* node's dependency closure (a reference adds no edge)
|
|
2030
|
+
* known needs, no cycle, a reference names only a node in the referring
|
|
2031
|
+
* node's dependency closure (a reference adds no edge), and `item` is read
|
|
2032
|
+
* only in a for_each node's `with`.
|
|
1550
2033
|
*/
|
|
1551
2034
|
function checkGraph(specs) {
|
|
1552
2035
|
const byId = new Map();
|
|
@@ -1570,18 +2053,22 @@ function checkGraph(specs) {
|
|
|
1570
2053
|
};
|
|
1571
2054
|
for (const s of specs) {
|
|
1572
2055
|
const reachable = closure(s.id);
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
const node =
|
|
1576
|
-
if (node && !reachable.has(node))
|
|
1577
|
-
throw new ConnectError(`node ${s.id}: ${
|
|
1578
|
-
}
|
|
2056
|
+
const items = s.forEach ? refs(s.forEach.items) : [];
|
|
2057
|
+
for (const r of [...refs(s.with), ...items]) {
|
|
2058
|
+
const node = r.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
|
|
2059
|
+
if (node && !reachable.has(node))
|
|
2060
|
+
throw new ConnectError(`node ${s.id}: ${r} names a node outside its needs`, Code.InvalidArgument);
|
|
1579
2061
|
}
|
|
2062
|
+
if (!s.forEach && refs(s.with).some((r) => ITEM_REF.test(r)))
|
|
2063
|
+
throw new ConnectError(`node ${s.id}: item is read only in a for_each node`, Code.InvalidArgument);
|
|
2064
|
+
if (items.some((r) => ITEM_REF.test(r)))
|
|
2065
|
+
throw new ConnectError(`node ${s.id}: for_each.items cannot read item`, Code.InvalidArgument);
|
|
1580
2066
|
}
|
|
1581
2067
|
}
|
|
1582
2068
|
function nodeSpec(n, requireHandler) {
|
|
1583
2069
|
let handler = "";
|
|
1584
2070
|
let uses = "";
|
|
2071
|
+
let pipeline;
|
|
1585
2072
|
if (n.action.case === "handler") {
|
|
1586
2073
|
handler = n.action.value;
|
|
1587
2074
|
requireHandler(handler);
|
|
@@ -1589,8 +2076,31 @@ function nodeSpec(n, requireHandler) {
|
|
|
1589
2076
|
else if (n.action.case === "uses") {
|
|
1590
2077
|
uses = `${n.action.value.name}@${n.action.value.version}`;
|
|
1591
2078
|
}
|
|
2079
|
+
else if (n.action.case === "pipeline") {
|
|
2080
|
+
// Pinned by freeze() before a node is compiled.
|
|
2081
|
+
const p = n.action.value;
|
|
2082
|
+
pipeline = { id: p.pipelineId, version: p.version, outputs: p.outputs.map((o) => ({ name: o.name, node: o.node, slot: o.slot })) };
|
|
2083
|
+
}
|
|
1592
2084
|
else {
|
|
1593
|
-
throw new ConnectError(`node ${n.id}: only handler and
|
|
2085
|
+
throw new ConnectError(`node ${n.id}: only handler, uses and pipeline nodes`, Code.Unimplemented);
|
|
2086
|
+
}
|
|
2087
|
+
let forEach;
|
|
2088
|
+
if (n.forEach) {
|
|
2089
|
+
if (n.manual)
|
|
2090
|
+
throw new ConnectError(`node ${n.id}: for_each is not for a manual node`, Code.InvalidArgument);
|
|
2091
|
+
if (uses)
|
|
2092
|
+
throw new ConnectError(`node ${n.id}: the fake runs for_each on handler and pipeline nodes only`, Code.Unimplemented);
|
|
2093
|
+
const fe = n.forEach;
|
|
2094
|
+
if (fe.maxItems > 10_000)
|
|
2095
|
+
throw new ConnectError(`node ${n.id}: for_each.max_items is at most 10000`, Code.InvalidArgument);
|
|
2096
|
+
const items = fe.items ? toJson(ValueSchema, fe.items) : null;
|
|
2097
|
+
if (typeof items === "string" ? !REF.test(items) : !Array.isArray(items)) {
|
|
2098
|
+
throw new ConnectError(`node ${n.id}: for_each.items is a list or a whole-value reference to one`, Code.InvalidArgument);
|
|
2099
|
+
}
|
|
2100
|
+
if (Array.isArray(items) && items.length > (fe.maxItems || 1000)) {
|
|
2101
|
+
throw new ConnectError(`node ${n.id}: for_each.items has ${items.length} elements, over max_items ${fe.maxItems || 1000}`, Code.InvalidArgument);
|
|
2102
|
+
}
|
|
2103
|
+
forEach = { items, maxItems: fe.maxItems, maxParallel: fe.maxParallel, onFailure: fe.onFailure };
|
|
1594
2104
|
}
|
|
1595
2105
|
return {
|
|
1596
2106
|
id: n.id,
|
|
@@ -1601,6 +2111,8 @@ function nodeSpec(n, requireHandler) {
|
|
|
1601
2111
|
retry: n.retry ? { max: n.retry.max, when: n.retry.when } : undefined,
|
|
1602
2112
|
timeoutMs: n.timeout ? durationMs(n.timeout) : 0,
|
|
1603
2113
|
interruptible: n.interruptible,
|
|
2114
|
+
...(pipeline ? { pipeline } : {}),
|
|
2115
|
+
...(forEach ? { forEach } : {}),
|
|
1604
2116
|
};
|
|
1605
2117
|
}
|
|
1606
2118
|
/** The server's dispatch.Outcome. */
|
|
@@ -1665,10 +2177,6 @@ function aborted(s) {
|
|
|
1665
2177
|
s.addEventListener("abort", () => r(), { once: true });
|
|
1666
2178
|
});
|
|
1667
2179
|
}
|
|
1668
|
-
/** The product an account belongs to. */
|
|
1669
|
-
function productOf(who) {
|
|
1670
|
-
return who.product ?? FAKE_PRODUCT;
|
|
1671
|
-
}
|
|
1672
2180
|
/** A queue's limits as the selection reads them. */
|
|
1673
2181
|
function queueLimits(q) {
|
|
1674
2182
|
return {
|