@xeonr/pipelines-sdk 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +231 -22
- package/dist/auth.d.ts +1 -1
- package/dist/client.d.ts +73 -10
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +47 -4
- package/dist/client.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +179 -16
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +45 -28
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +136 -17
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +42 -35
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +224 -8
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +43 -9
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +64 -8
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.js +62 -5
- package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +312 -9
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js +100 -7
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +1072 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +299 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +103 -8
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js +32 -24
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +711 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js +185 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +92 -23
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js +36 -33
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +1191 -33
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js +419 -35
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts +194 -0
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js +100 -0
- package/dist/gen/xeonr/pipelines/v1/schedule_targets_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +1186 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +335 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +32 -5
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +12 -3
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts +270 -27
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js +163 -24
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts +391 -0
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js +100 -0
- package/dist/gen/xeonr/pipelines/v1/tenant_policies_pb.js.map +1 -0
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts +137 -1
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/platform/v1/options_pb.js +104 -1
- package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
- package/dist/handler.d.ts +51 -1
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +27 -0
- package/dist/handler.js.map +1 -1
- package/dist/index.d.ts +13 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/queues.d.ts +158 -0
- package/dist/queues.d.ts.map +1 -0
- package/dist/queues.js +117 -0
- package/dist/queues.js.map +1 -0
- package/dist/reporter.d.ts +120 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporter.js +323 -0
- package/dist/reporter.js.map +1 -0
- package/dist/schedules.d.ts +193 -0
- package/dist/schedules.d.ts.map +1 -0
- package/dist/schedules.js +126 -0
- package/dist/schedules.js.map +1 -0
- package/dist/testing/cron.d.ts +33 -0
- package/dist/testing/cron.d.ts.map +1 -0
- package/dist/testing/cron.js +177 -0
- package/dist/testing/cron.js.map +1 -0
- package/dist/testing/index.d.ts +229 -7
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +1304 -103
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/pipelines.d.ts +99 -0
- package/dist/testing/pipelines.d.ts.map +1 -0
- package/dist/testing/pipelines.js +445 -0
- package/dist/testing/pipelines.js.map +1 -0
- package/dist/testing/queues.d.ts +49 -0
- package/dist/testing/queues.d.ts.map +1 -0
- package/dist/testing/queues.js +72 -0
- package/dist/testing/queues.js.map +1 -0
- package/dist/testing/schedules.d.ts +125 -0
- package/dist/testing/schedules.d.ts.map +1 -0
- package/dist/testing/schedules.js +422 -0
- package/dist/testing/schedules.js.map +1 -0
- package/dist/worker.d.ts +48 -7
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +59 -9
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
package/dist/testing/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @xeonr/pipelines-sdk/testing: an in-memory fake of pipelines-api's worker
|
|
3
|
-
* and jobs surface, so a
|
|
3
|
+
* and jobs surface, so a service can unit-test its handlers, and the code
|
|
4
4
|
* that enqueues them, with no server.
|
|
5
5
|
*
|
|
6
6
|
* It is a real Connect router behind an in-memory transport, so the SDK's
|
|
@@ -31,19 +31,45 @@
|
|
|
31
31
|
* `handler_failure_kind`, retries by class, free interruptions on an
|
|
32
32
|
* `interruptible` node, cancel, and error injection.
|
|
33
33
|
*
|
|
34
|
+
* Stored pipelines (`PipelinesService`, see ./pipelines.ts) and the runs of
|
|
35
|
+
* their versions (`CreateRun` with `pipeline` or `snapshot_id`, and a
|
|
36
|
+
* schedule's pipeline target), with the policy's node defaults and
|
|
37
|
+
* `max_concurrent_runs`. Sub-pipeline nodes: pinned to a version when the
|
|
38
|
+
* snapshot is made (cycles and nesting over 3 refused), one child run per
|
|
39
|
+
* attempt about the parent's resource, the child's outputs mapped back, a
|
|
40
|
+
* failed child failing the attempt as `DOWNSTREAM_FAILED`, a cancel passed
|
|
41
|
+
* down. `for_each` nodes on handler and sub-pipeline nodes: the list from a
|
|
42
|
+
* literal, a variable or an upstream kv output, `${{ item }}`,
|
|
43
|
+
* `${{ item.<path> }}` and `${{ item_index }}`, `max_items`,
|
|
44
|
+
* `max_parallel`, each element retried on its own, the three `on_failure`
|
|
45
|
+
* policies, outputs aggregated in element order, and RetryNode running only
|
|
46
|
+
* the failed elements again.
|
|
47
|
+
*
|
|
34
48
|
* What it does not: secrets, blobs, tiers, routing between pools, the
|
|
35
|
-
* outbox,
|
|
49
|
+
* outbox, exclusions, `for_each` on an external (image) node, and the
|
|
50
|
+
* platform's own deadline enforcement.
|
|
36
51
|
*/
|
|
37
|
-
import { randomUUID } from "node:crypto";
|
|
38
|
-
import { create, fromJson, toJson } from "@bufbuild/protobuf";
|
|
39
|
-
import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
52
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
53
|
+
import { clone, create, fromJson, toJson } from "@bufbuild/protobuf";
|
|
54
|
+
import { DurationSchema, durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
40
55
|
import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
|
|
41
56
|
import { tokenSource } from "../auth.js";
|
|
42
57
|
import { authInterceptor } from "../transport.js";
|
|
43
58
|
import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
|
|
59
|
+
import { DefinitionSchema, ForEachFailure, RetrySchema } from "../gen/xeonr/pipelines/v1/definition_pb.js";
|
|
60
|
+
import { PipelineState, PipelinesService } from "../gen/xeonr/pipelines/v1/pipelines_pb.js";
|
|
44
61
|
import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
|
|
45
62
|
import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
|
|
46
|
-
import { AttemptSchema, NodeRunSchema, RunSchema, RunsService } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
63
|
+
import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunOrigin, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
64
|
+
import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
|
|
65
|
+
import { FakeStoredPipelines, takesRunsAbout } from "./pipelines.js";
|
|
66
|
+
import { pickQueued } from "./queues.js";
|
|
67
|
+
import { FakeClock, FakeSchedules } from "./schedules.js";
|
|
68
|
+
import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
|
|
69
|
+
export { FakeClock, FakeSchedules } from "./schedules.js";
|
|
70
|
+
export { parseCron, nextOccurrence } from "./cron.js";
|
|
71
|
+
export { pickQueued } from "./queues.js";
|
|
72
|
+
export { FakeStoredPipelines, resolveVariables } from "./pipelines.js";
|
|
47
73
|
import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
|
|
48
74
|
/** The tenant the default account acts for. */
|
|
49
75
|
export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
|
|
@@ -58,6 +84,12 @@ export const ALL_SCOPES = [
|
|
|
58
84
|
"pipelines:pool:read",
|
|
59
85
|
"pipelines:pool:edit",
|
|
60
86
|
"pipelines:worker:edit",
|
|
87
|
+
"pipelines:queue:read",
|
|
88
|
+
"pipelines:queue:edit",
|
|
89
|
+
"pipelines:schedule:read",
|
|
90
|
+
"pipelines:schedule:edit",
|
|
91
|
+
"pipelines:pipeline:read",
|
|
92
|
+
"pipelines:pipeline:edit",
|
|
61
93
|
];
|
|
62
94
|
const TRANSIENT = [
|
|
63
95
|
FailureClass.ABANDONED,
|
|
@@ -68,9 +100,11 @@ const TRANSIENT = [
|
|
|
68
100
|
FailureClass.INTERNAL,
|
|
69
101
|
];
|
|
70
102
|
const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
71
|
-
/** How many interruptions an interruptible node gets without spending a retry. */
|
|
72
|
-
const FREE_INTERRUPTIONS =
|
|
103
|
+
/** How many interruptions an interruptible node gets without spending a retry (the api's default). */
|
|
104
|
+
const FREE_INTERRUPTIONS = 3;
|
|
73
105
|
const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
|
|
106
|
+
/** How deeply runs nest through sub-pipelines, counting the outermost (the api's MaxDepth). */
|
|
107
|
+
const MAX_DEPTH = 3;
|
|
74
108
|
/**
|
|
75
109
|
* An in-memory pipelines-api for tests. Call {@link close} when done (it
|
|
76
110
|
* stops the timer that looks at leases and delays).
|
|
@@ -88,6 +122,18 @@ export class FakePipelines {
|
|
|
88
122
|
attemptsById = new Map();
|
|
89
123
|
idempotency = new Map();
|
|
90
124
|
injected = new Map();
|
|
125
|
+
/** Queues by id. */
|
|
126
|
+
queues = new Map();
|
|
127
|
+
/**
|
|
128
|
+
* The clock schedules run on: real time until a test calls
|
|
129
|
+
* `clock.set()` or `clock.advance()`, each of which fires what is due.
|
|
130
|
+
*/
|
|
131
|
+
clock = new FakeClock();
|
|
132
|
+
/** The schedules, with their recorded occurrences. */
|
|
133
|
+
schedules;
|
|
134
|
+
/** Stored pipelines, their versions and the snapshots runs execute. */
|
|
135
|
+
pipelines;
|
|
136
|
+
queueClaims = new Map();
|
|
91
137
|
timer;
|
|
92
138
|
changed;
|
|
93
139
|
constructor(opts = {}) {
|
|
@@ -102,8 +148,54 @@ export class FakePipelines {
|
|
|
102
148
|
retryBackoffMs: opts.retryBackoffMs ?? 0,
|
|
103
149
|
logMaxBytes: opts.logMaxBytes ?? 0,
|
|
104
150
|
tickMs: opts.tickMs ?? 10,
|
|
151
|
+
issueCap: opts.issueCap ?? 1000,
|
|
152
|
+
spanCap: opts.spanCap ?? 10_000,
|
|
105
153
|
};
|
|
106
154
|
this.changed = signal();
|
|
155
|
+
this.pipelines = new FakeStoredPipelines({
|
|
156
|
+
enter: (method, ctx, scope) => this.enter(method, ctx, scope),
|
|
157
|
+
freeze: (def, env) => this.freeze(def, env),
|
|
158
|
+
});
|
|
159
|
+
this.schedules = new FakeSchedules({
|
|
160
|
+
createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
|
|
161
|
+
tenant: owner.tenant,
|
|
162
|
+
handler: spec.handler,
|
|
163
|
+
key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
|
|
164
|
+
payload: spec.payload,
|
|
165
|
+
delayMs: 0,
|
|
166
|
+
resource: spec.resource,
|
|
167
|
+
retry: spec.retry,
|
|
168
|
+
timeoutMs: spec.timeoutMs,
|
|
169
|
+
queue: spec.queue,
|
|
170
|
+
interruptible: spec.interruptible,
|
|
171
|
+
trigger: spec.trigger,
|
|
172
|
+
schedule: spec.schedule,
|
|
173
|
+
}),
|
|
174
|
+
startPipeline: (owner, spec) => {
|
|
175
|
+
const st = this.pipelines.get(spec.target.pipelineId);
|
|
176
|
+
if (!st || st.pipeline.tenant !== owner.tenant)
|
|
177
|
+
throw new ConnectError("no such pipeline", Code.NotFound);
|
|
178
|
+
return this.startStored({ tenant: owner.tenant, scopes: ALL_SCOPES }, {
|
|
179
|
+
tenant: owner.tenant,
|
|
180
|
+
key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
|
|
181
|
+
resource: spec.resource ?? (st.pipeline.scope ? { kind: st.pipeline.scope.kind, id: st.pipeline.scope.id } : undefined),
|
|
182
|
+
variables: spec.variables,
|
|
183
|
+
facts: spec.target.attestedFacts,
|
|
184
|
+
stored: st,
|
|
185
|
+
version: spec.target.version,
|
|
186
|
+
trigger: spec.trigger,
|
|
187
|
+
schedule: spec.schedule,
|
|
188
|
+
});
|
|
189
|
+
},
|
|
190
|
+
checkPipeline: (tenant, id) => {
|
|
191
|
+
const st = this.pipelines.get(id);
|
|
192
|
+
if (!st || st.pipeline.tenant !== tenant)
|
|
193
|
+
throw new ConnectError(`schedule.target.pipeline_id: no pipeline ${id} of the tenant`, Code.NotFound);
|
|
194
|
+
},
|
|
195
|
+
runs: () => this.runs.values(),
|
|
196
|
+
runMessage: (id) => this.runMessage(this.run(id)),
|
|
197
|
+
checkHandler: (handler) => this.requireHandler(handler),
|
|
198
|
+
}, this.clock);
|
|
107
199
|
this.timer = setInterval(() => this.tick(), this.opts.tickMs);
|
|
108
200
|
this.timer.unref();
|
|
109
201
|
}
|
|
@@ -208,7 +300,7 @@ export class FakePipelines {
|
|
|
208
300
|
completeNode(runId, nodeId, result = {}) {
|
|
209
301
|
const run = this.run(runId);
|
|
210
302
|
const node = run.nodes.get(nodeId);
|
|
211
|
-
if (!node || node.handler)
|
|
303
|
+
if (!node || node.handler || node.pipeline)
|
|
212
304
|
throw new Error(`FakePipelines.completeNode: ${nodeId} is not an external node of ${runId}`);
|
|
213
305
|
if (node.state !== NodeState.READY)
|
|
214
306
|
throw new Error(`FakePipelines.completeNode: ${nodeId} is not ready`);
|
|
@@ -371,50 +463,145 @@ export class FakePipelines {
|
|
|
371
463
|
this.notify();
|
|
372
464
|
return { state: a.state };
|
|
373
465
|
},
|
|
466
|
+
appendIssues: (req, ctx) => {
|
|
467
|
+
const who = this.enter("AppendIssues", ctx, "pipelines:worker:edit");
|
|
468
|
+
const a = this.held(who, req.ref);
|
|
469
|
+
if (a.state !== AttemptState.RUNNING)
|
|
470
|
+
throw stale();
|
|
471
|
+
if (req.issues.length < 1 || req.issues.length > 100)
|
|
472
|
+
throw new ConnectError("issues: 1 to 100", Code.InvalidArgument);
|
|
473
|
+
return { accepted: this.appendIssues(a, req.issues) };
|
|
474
|
+
},
|
|
475
|
+
setProgress: (req, ctx) => {
|
|
476
|
+
const who = this.enter("SetProgress", ctx, "pipelines:worker:edit");
|
|
477
|
+
const a = this.held(who, req.ref);
|
|
478
|
+
if (a.state !== AttemptState.RUNNING)
|
|
479
|
+
throw stale();
|
|
480
|
+
if (!(req.fraction >= 0 && req.fraction <= 1))
|
|
481
|
+
throw new ConnectError("fraction: 0 to 1", Code.InvalidArgument);
|
|
482
|
+
if (req.stageState === StageState.PENDING)
|
|
483
|
+
throw new ConnectError("stage_state: a stage never moves back to PENDING", Code.InvalidArgument);
|
|
484
|
+
if (req.stage && !/^[a-z][a-z0-9_.-]{0,63}$/.test(req.stage))
|
|
485
|
+
throw new ConnectError("stage: invalid key", Code.InvalidArgument);
|
|
486
|
+
if (req.message.length > 255)
|
|
487
|
+
throw new ConnectError("message: at most 255", Code.InvalidArgument);
|
|
488
|
+
this.setProgress(a, req);
|
|
489
|
+
return {};
|
|
490
|
+
},
|
|
491
|
+
reportSpans: (req, ctx) => {
|
|
492
|
+
const who = this.enter("ReportSpans", ctx, "pipelines:worker:edit");
|
|
493
|
+
const a = this.held(who, req.ref);
|
|
494
|
+
if (a.state !== AttemptState.RUNNING)
|
|
495
|
+
throw stale();
|
|
496
|
+
if (req.spans.length < 1 || req.spans.length > 1000)
|
|
497
|
+
throw new ConnectError("spans: 1 to 1000", Code.InvalidArgument);
|
|
498
|
+
for (const sp of req.spans) {
|
|
499
|
+
if (!/^[0-9a-f]{32}$/.test(sp.traceId) || !/^[0-9a-f]{16}$/.test(sp.spanId) || !sp.startedAt || !sp.endedAt || !sp.name) {
|
|
500
|
+
throw new ConnectError("spans: trace_id, span_id, name, started_at and ended_at are required", Code.InvalidArgument);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
const room = Math.max(0, this.opts.spanCap - a.spans.length);
|
|
504
|
+
a.spans.push(...req.spans.slice(0, room));
|
|
505
|
+
return { accepted: Math.min(room, req.spans.length) };
|
|
506
|
+
},
|
|
374
507
|
});
|
|
508
|
+
router.service(QueuesService, this.queuesService());
|
|
509
|
+
router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
|
|
510
|
+
router.service(PipelinesService, this.pipelines.service());
|
|
375
511
|
router.service(RunsService, {
|
|
512
|
+
listIssues: (req, ctx) => {
|
|
513
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
514
|
+
this.visible(who, req.runId);
|
|
515
|
+
const issues = this.issuesOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId, minSeverity: req.minSeverity });
|
|
516
|
+
return {
|
|
517
|
+
issues: issues.map((i) => create(RunIssueSchema, { ...i, detail: i.detail, createdAt: timestampFromMs(i.createdAt) })),
|
|
518
|
+
nextCursor: "",
|
|
519
|
+
};
|
|
520
|
+
},
|
|
521
|
+
getLogs: (req, ctx) => {
|
|
522
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
523
|
+
const a = this.attemptsById.get(req.attemptId);
|
|
524
|
+
if (!a)
|
|
525
|
+
throw new ConnectError("no such attempt", Code.NotFound);
|
|
526
|
+
this.visible(who, a.runId);
|
|
527
|
+
const all = Buffer.concat(a.chunks);
|
|
528
|
+
const from = Math.min(Number(req.offset), all.length);
|
|
529
|
+
const to = req.limitBytes ? Math.min(all.length, from + req.limitBytes) : all.length;
|
|
530
|
+
const complete = to === all.length && (a.logFinal || a.completedAt !== undefined);
|
|
531
|
+
return { data: new Uint8Array(all.subarray(from, to)), nextOffset: BigInt(to), complete, truncated: a.logTruncated };
|
|
532
|
+
},
|
|
533
|
+
getRunTrace: (req, ctx) => {
|
|
534
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
535
|
+
this.visible(who, req.runId);
|
|
536
|
+
return { spans: this.traceOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId }), truncated: false };
|
|
537
|
+
},
|
|
376
538
|
createJob: (req, ctx) => {
|
|
377
539
|
const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
|
|
378
540
|
this.actsFor(who, req.tenant);
|
|
379
|
-
const existing = this.idempotency.get(`${
|
|
541
|
+
const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
|
|
380
542
|
if (existing)
|
|
381
543
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
382
|
-
this.
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
|
|
386
|
-
if (req.retry?.when.includes(FailureClass.CANCELLED))
|
|
387
|
-
throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
|
|
388
|
-
this.addNode(run, {
|
|
389
|
-
id: "job",
|
|
544
|
+
const run = this.createJobRun(who, {
|
|
545
|
+
fairnessKey: req.fairnessKey,
|
|
546
|
+
tenant: req.tenant,
|
|
390
547
|
handler: req.handler,
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
548
|
+
key: req.idempotencyKey,
|
|
549
|
+
payload: req.payload ?? {},
|
|
550
|
+
delayMs: req.delay ? durationMs(req.delay) : 0,
|
|
551
|
+
resource: req.resource,
|
|
552
|
+
retry: req.retry,
|
|
395
553
|
timeoutMs: req.timeout ? durationMs(req.timeout) : 0,
|
|
396
|
-
|
|
554
|
+
queue: req.queue,
|
|
555
|
+
interruptible: req.interruptible,
|
|
397
556
|
});
|
|
398
|
-
this.advance(run);
|
|
399
557
|
return { run: this.runMessage(run), deduplicated: false };
|
|
400
558
|
},
|
|
559
|
+
retryNode: (req, ctx) => {
|
|
560
|
+
const who = this.enter("RetryNode", ctx, "pipelines:run:edit");
|
|
561
|
+
const run = this.visible(who, req.runId);
|
|
562
|
+
this.retryNode(run, req.nodeId);
|
|
563
|
+
return { run: this.runMessage(run) };
|
|
564
|
+
},
|
|
401
565
|
createRun: (req, ctx) => {
|
|
402
566
|
const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
|
|
403
567
|
this.actsFor(who, req.tenant);
|
|
404
|
-
const existing = this.idempotency.get(`${
|
|
568
|
+
const existing = this.idempotency.get(`${req.tenant}\u0000${req.idempotencyKey}`);
|
|
405
569
|
if (existing)
|
|
406
570
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
571
|
+
const start = { tenant: req.tenant, key: req.idempotencyKey, resource: req.resource, variables: req.variables, facts: req.attestedFacts };
|
|
572
|
+
let run;
|
|
573
|
+
switch (req.source.case) {
|
|
574
|
+
case "inlineDefinition": {
|
|
575
|
+
const about = req.resource ? { kind: req.resource.kind, id: req.resource.id } : undefined;
|
|
576
|
+
const def = this.freeze(req.source.value, { tenant: req.tenant, scope: about, about });
|
|
577
|
+
run = this.startSnapshot(who, { ...start, snapshot: this.pipelines.inlineSnapshot(req.tenant, def) });
|
|
578
|
+
break;
|
|
579
|
+
}
|
|
580
|
+
case "snapshotId": {
|
|
581
|
+
const snap = this.pipelines.snapshot(req.source.value);
|
|
582
|
+
if (!snap || snap.tenant !== req.tenant)
|
|
583
|
+
throw new ConnectError("no such snapshot", Code.NotFound);
|
|
584
|
+
run = this.startSnapshot(who, { ...start, snapshot: snap });
|
|
585
|
+
break;
|
|
586
|
+
}
|
|
587
|
+
case "pipeline": {
|
|
588
|
+
const src = req.source.value;
|
|
589
|
+
const sel = src.pipeline;
|
|
590
|
+
const st = sel.case === "pipelineId"
|
|
591
|
+
? this.pipelines.get(sel.value)
|
|
592
|
+
: sel.case === "byName" && sel.value.tenant === req.tenant
|
|
593
|
+
? this.pipelines.find(sel.value.tenant, sel.value.scope, sel.value.name)
|
|
594
|
+
: undefined;
|
|
595
|
+
if (!st || st.pipeline.tenant !== req.tenant)
|
|
596
|
+
throw new ConnectError("no such pipeline", Code.NotFound);
|
|
597
|
+
if (!req.resource)
|
|
598
|
+
throw new ConnectError("resource: required", Code.InvalidArgument);
|
|
599
|
+
run = this.startStored(who, { ...start, stored: st, version: src.version, trigger: RunTrigger.API });
|
|
600
|
+
break;
|
|
601
|
+
}
|
|
602
|
+
default:
|
|
603
|
+
throw new ConnectError("source: required", Code.InvalidArgument);
|
|
604
|
+
}
|
|
418
605
|
return { run: this.runMessage(run), deduplicated: false };
|
|
419
606
|
},
|
|
420
607
|
getRun: (req, ctx) => {
|
|
@@ -424,7 +611,11 @@ export class FakePipelines {
|
|
|
424
611
|
getRunGraph: (req, ctx) => {
|
|
425
612
|
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
426
613
|
const run = this.visible(who, req.runId);
|
|
427
|
-
return {
|
|
614
|
+
return {
|
|
615
|
+
run: this.runMessage(run),
|
|
616
|
+
nodes: [...run.nodes.values()].map((n) => this.nodeMessage(n)),
|
|
617
|
+
...(run.definition ? { definition: run.definition } : {}),
|
|
618
|
+
};
|
|
428
619
|
},
|
|
429
620
|
listRuns: (req, ctx) => {
|
|
430
621
|
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
@@ -537,14 +728,40 @@ export class FakePipelines {
|
|
|
537
728
|
throw new ConnectError(`handler ${handler} is not a published worker package`, Code.InvalidArgument);
|
|
538
729
|
}
|
|
539
730
|
}
|
|
731
|
+
/**
|
|
732
|
+
* Holds a handler node's `with` to its manifest, as the api does when a run
|
|
733
|
+
* is created: every key must be a declared input, every required input
|
|
734
|
+
* given; defaults fill the rest. Returns the inputs with defaults.
|
|
735
|
+
*/
|
|
736
|
+
checkInputs(nodeId, handler, given) {
|
|
737
|
+
const pkg = this.packages.get(handler);
|
|
738
|
+
const version = pkg ? [...pkg.versions.keys()].at(-1) : "";
|
|
739
|
+
const manifest = pkg?.versions.get(version);
|
|
740
|
+
const inputs = manifest?.inputs ?? [];
|
|
741
|
+
for (const k of Object.keys(given)) {
|
|
742
|
+
if (!inputs.some((i) => i.name === k)) {
|
|
743
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": input "${k}" is not an input of ${handler}@${version}`, Code.InvalidArgument);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
const out = { ...given };
|
|
747
|
+
for (const i of inputs) {
|
|
748
|
+
if (out[i.name] !== undefined)
|
|
749
|
+
continue;
|
|
750
|
+
if (i.default)
|
|
751
|
+
out[i.name] = toJson(ValueSchema, i.default);
|
|
752
|
+
else if (i.required)
|
|
753
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": required input "${i.name}" is not given`, Code.InvalidArgument);
|
|
754
|
+
}
|
|
755
|
+
return out;
|
|
756
|
+
}
|
|
540
757
|
// ---- the scheduler ----------------------------------------------------
|
|
541
|
-
newRun(who, tenant, key, resource, variables, delayMs) {
|
|
758
|
+
newRun(who, tenant, key, resource, variables, delayMs, extra = {}) {
|
|
542
759
|
const now = Date.now();
|
|
543
760
|
const run = {
|
|
544
761
|
id: randomUUID(),
|
|
545
762
|
tenant,
|
|
546
|
-
product: who.product ?? who.tenant,
|
|
547
763
|
idempotencyKey: key,
|
|
764
|
+
fairnessKey: "",
|
|
548
765
|
createdBy: `urn:fake:sa:${who.tenant}`,
|
|
549
766
|
resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
|
|
550
767
|
variables,
|
|
@@ -558,9 +775,27 @@ export class FakePipelines {
|
|
|
558
775
|
cancelRequestedAt: undefined,
|
|
559
776
|
cancelReason: "",
|
|
560
777
|
nodes: new Map(),
|
|
778
|
+
queueId: "",
|
|
779
|
+
queue: "",
|
|
780
|
+
trigger: RunTrigger.API,
|
|
781
|
+
scheduleId: "",
|
|
782
|
+
scheduledFor: undefined,
|
|
783
|
+
scheduleTarget: "",
|
|
784
|
+
issues: [],
|
|
785
|
+
issuesTruncated: false,
|
|
786
|
+
attestedFacts: undefined,
|
|
787
|
+
snapshotId: "",
|
|
788
|
+
origin: RunOrigin.INLINE,
|
|
789
|
+
pipelineId: "",
|
|
790
|
+
pipelineVersion: 0,
|
|
791
|
+
parentRunId: "",
|
|
792
|
+
parentNodeId: "",
|
|
793
|
+
definition: undefined,
|
|
794
|
+
maxConcurrentRuns: 0,
|
|
795
|
+
...extra,
|
|
561
796
|
};
|
|
562
797
|
this.runs.set(run.id, run);
|
|
563
|
-
this.idempotency.set(`${run.
|
|
798
|
+
this.idempotency.set(`${run.tenant}\u0000${key}`, run.id);
|
|
564
799
|
return run;
|
|
565
800
|
}
|
|
566
801
|
addNode(run, spec) {
|
|
@@ -577,7 +812,170 @@ export class FakePipelines {
|
|
|
577
812
|
retry: spec.retry ?? { max: this.opts.defaultRetries, when: [] },
|
|
578
813
|
timeoutMs: spec.timeoutMs || this.opts.defaultTimeoutMs,
|
|
579
814
|
interruptible: spec.interruptible,
|
|
815
|
+
pipeline: spec.pipeline,
|
|
816
|
+
forEach: spec.forEach,
|
|
817
|
+
elements: undefined,
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
// ---- definitions, snapshots and starts ---------------------------------
|
|
821
|
+
/**
|
|
822
|
+
* Checks a definition as CreateRun does and freezes it as its snapshot
|
|
823
|
+
* holds it: the policy's node defaults filled, each sub-pipeline pinned to
|
|
824
|
+
* the version it names (0, the latest now), a child of another tenant, an
|
|
825
|
+
* unpublished or archived one, one not taking runs about `env.about`, a
|
|
826
|
+
* cycle back to `env.self` or nesting over 3 refused.
|
|
827
|
+
*/
|
|
828
|
+
freeze(def, env) {
|
|
829
|
+
if (def.nodes.length === 0)
|
|
830
|
+
throw new ConnectError("definition: no nodes", Code.InvalidArgument);
|
|
831
|
+
const out = clone(DefinitionSchema, def);
|
|
832
|
+
for (const n of out.nodes) {
|
|
833
|
+
if (!n.retry && env.defaultRetry)
|
|
834
|
+
n.retry = clone(RetrySchema, env.defaultRetry);
|
|
835
|
+
if (!n.timeout && env.defaultTimeout)
|
|
836
|
+
n.timeout = clone(DurationSchema, env.defaultTimeout);
|
|
837
|
+
if (n.action.case === "pipeline") {
|
|
838
|
+
const ref = n.action.value;
|
|
839
|
+
const st = ref.pipelineId ? this.pipelines.get(ref.pipelineId) : this.pipelines.find(env.tenant, ref.scope ?? env.scope, ref.name);
|
|
840
|
+
const label = ref.pipelineId || ref.name;
|
|
841
|
+
if (!st || st.pipeline.tenant !== env.tenant)
|
|
842
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": no pipeline ${label} of this tenant`, Code.InvalidArgument);
|
|
843
|
+
const v = this.pipelines.version(st, ref.version);
|
|
844
|
+
if (st.pipeline.latestVersion === 0 || !v)
|
|
845
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} has no published version ${ref.version || ""}`.trimEnd(), Code.InvalidArgument);
|
|
846
|
+
if (st.pipeline.state === PipelineState.ARCHIVED)
|
|
847
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} is archived`, Code.InvalidArgument);
|
|
848
|
+
if (env.about)
|
|
849
|
+
this.checkChild(n.id, st, env.about);
|
|
850
|
+
if (env.self && (st.pipeline.id === env.self || this.reaches(this.pipelines.snapshot(v.snapshotId).definition, env.self))) {
|
|
851
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": pipeline ${st.pipeline.id} reaches this pipeline again, a cycle`, Code.InvalidArgument);
|
|
852
|
+
}
|
|
853
|
+
const child = this.pipelines.snapshot(v.snapshotId).definition;
|
|
854
|
+
for (const o of ref.outputs) {
|
|
855
|
+
if (!child.nodes.some((c) => c.id === o.node))
|
|
856
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": output ${o.name} names node ${o.node}, which pipeline ${st.pipeline.id} does not have`, Code.InvalidArgument);
|
|
857
|
+
}
|
|
858
|
+
const declared = new Map((v.spec?.variables ?? []).map((x) => [x.name, x]));
|
|
859
|
+
for (const k of Object.keys(n.with)) {
|
|
860
|
+
if (!declared.has(k))
|
|
861
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": ${JSON.stringify(k)} is not a variable pipeline ${st.pipeline.id} declares`, Code.InvalidArgument);
|
|
862
|
+
}
|
|
863
|
+
for (const x of declared.values()) {
|
|
864
|
+
if (x.required && !x.default && !(x.name in n.with))
|
|
865
|
+
throw new ConnectError(`the definition is invalid: node "${n.id}": variable ${JSON.stringify(x.name)} of pipeline ${st.pipeline.id} is required`, Code.InvalidArgument);
|
|
866
|
+
}
|
|
867
|
+
n.action.value = { ...ref, pipelineId: st.pipeline.id, version: v.version };
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
this.compile(out);
|
|
871
|
+
const depth = this.depth(out);
|
|
872
|
+
if (depth > MAX_DEPTH)
|
|
873
|
+
throw new ConnectError(`the definition is invalid: it nests ${depth} deep, and runs nest at most ${MAX_DEPTH} deep counting this one`, Code.InvalidArgument);
|
|
874
|
+
return out;
|
|
875
|
+
}
|
|
876
|
+
/** A frozen definition's nodes, checked: handlers' inputs, the graph, for_each's rules. */
|
|
877
|
+
compile(def) {
|
|
878
|
+
const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
|
|
879
|
+
for (const sp of specs)
|
|
880
|
+
if (sp.handler)
|
|
881
|
+
sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
|
|
882
|
+
checkGraph(specs);
|
|
883
|
+
return specs;
|
|
884
|
+
}
|
|
885
|
+
/** How deep runs of a frozen definition nest through its pinned sub-pipelines, counting its own. */
|
|
886
|
+
depth(def) {
|
|
887
|
+
let most = 0;
|
|
888
|
+
for (const n of def.nodes) {
|
|
889
|
+
if (n.action.case !== "pipeline")
|
|
890
|
+
continue;
|
|
891
|
+
const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
|
|
892
|
+
if (snap)
|
|
893
|
+
most = Math.max(most, this.depth(snap.definition));
|
|
894
|
+
}
|
|
895
|
+
return 1 + most;
|
|
896
|
+
}
|
|
897
|
+
/** Whether a frozen definition reaches pipeline `id` through its pinned sub-pipelines. */
|
|
898
|
+
reaches(def, id) {
|
|
899
|
+
return def.nodes.some((n) => {
|
|
900
|
+
if (n.action.case !== "pipeline")
|
|
901
|
+
return false;
|
|
902
|
+
if (n.action.value.pipelineId === id)
|
|
903
|
+
return true;
|
|
904
|
+
const snap = this.childSnapshot(n.action.value.pipelineId, n.action.value.version);
|
|
905
|
+
return snap ? this.reaches(snap.definition, id) : false;
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
childSnapshot(pipelineId, version) {
|
|
909
|
+
const st = this.pipelines.get(pipelineId);
|
|
910
|
+
const v = st && this.pipelines.version(st, version);
|
|
911
|
+
return v ? this.pipelines.snapshot(v.snapshotId) : undefined;
|
|
912
|
+
}
|
|
913
|
+
/** A sub-pipeline's child must take runs about the parent's resource, as if started directly. */
|
|
914
|
+
checkChild(nodeId, st, about) {
|
|
915
|
+
if (!takesRunsAbout(st.pipeline, about)) {
|
|
916
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": pipeline ${st.pipeline.id} (${st.pipeline.name}) takes runs about its scope or a resource of a kind in its resource_kinds, not ${JSON.stringify(about?.kind ?? "")}`, Code.InvalidArgument);
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
/** Every pinned child, at any depth, against the resource a run is about. */
|
|
920
|
+
checkChildren(def, about) {
|
|
921
|
+
for (const n of def.nodes) {
|
|
922
|
+
if (n.action.case !== "pipeline")
|
|
923
|
+
continue;
|
|
924
|
+
const st = this.pipelines.get(n.action.value.pipelineId);
|
|
925
|
+
this.checkChild(n.id, st, about);
|
|
926
|
+
const snap = this.childSnapshot(st.pipeline.id, n.action.value.version);
|
|
927
|
+
if (snap)
|
|
928
|
+
this.checkChildren(snap.definition, about);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
/** Starts a run of a snapshot: an inline definition's, an earlier one by id, or a version's. */
|
|
932
|
+
startSnapshot(who, spec) {
|
|
933
|
+
this.checkChildren(spec.snapshot.definition, spec.resource);
|
|
934
|
+
const specs = this.compile(spec.snapshot.definition);
|
|
935
|
+
const run = this.newRun(who, spec.tenant, spec.key, spec.resource, spec.variables ?? {}, 0, {
|
|
936
|
+
attestedFacts: spec.facts,
|
|
937
|
+
snapshotId: spec.snapshot.id,
|
|
938
|
+
origin: spec.snapshot.origin,
|
|
939
|
+
definition: spec.snapshot.definition,
|
|
940
|
+
...spec.extra,
|
|
941
|
+
});
|
|
942
|
+
for (const s of specs)
|
|
943
|
+
this.addNode(run, s);
|
|
944
|
+
this.advance(run);
|
|
945
|
+
return run;
|
|
946
|
+
}
|
|
947
|
+
/** Starts a version of a stored pipeline: CreateRun's `pipeline` source, a schedule's target, a sub-pipeline's child. */
|
|
948
|
+
startStored(who, spec) {
|
|
949
|
+
const { version, snapshot, variables } = this.pipelines.resolveStart(spec.stored, spec.version, spec.resource, spec.variables);
|
|
950
|
+
const run = this.startSnapshot(who, {
|
|
951
|
+
...spec,
|
|
952
|
+
variables,
|
|
953
|
+
snapshot,
|
|
954
|
+
extra: {
|
|
955
|
+
pipelineId: spec.stored.pipeline.id,
|
|
956
|
+
pipelineVersion: version.version,
|
|
957
|
+
maxConcurrentRuns: version.spec?.policy?.maxConcurrentRuns ?? 0,
|
|
958
|
+
trigger: spec.trigger,
|
|
959
|
+
...(spec.parent ? { parentRunId: spec.parent.runId, parentNodeId: spec.parent.nodeId } : {}),
|
|
960
|
+
},
|
|
580
961
|
});
|
|
962
|
+
if (spec.schedule) {
|
|
963
|
+
run.scheduleId = spec.schedule.id;
|
|
964
|
+
run.scheduledFor = spec.schedule.scheduledFor;
|
|
965
|
+
run.scheduleTarget = spec.schedule.target;
|
|
966
|
+
}
|
|
967
|
+
return run;
|
|
968
|
+
}
|
|
969
|
+
/** Whether a run of a pipeline must wait for its policy's `max_concurrent_runs`. */
|
|
970
|
+
atCap(run) {
|
|
971
|
+
if (run.maxConcurrentRuns <= 0)
|
|
972
|
+
return false;
|
|
973
|
+
let active = 0;
|
|
974
|
+
for (const r of this.runs.values()) {
|
|
975
|
+
if (r.id !== run.id && r.pipelineId === run.pipelineId && r.state !== RunState.QUEUED && !terminal(r.state))
|
|
976
|
+
active++;
|
|
977
|
+
}
|
|
978
|
+
return active >= run.maxConcurrentRuns;
|
|
581
979
|
}
|
|
582
980
|
/** Makes ready what can be, skips what cannot run, and settles the run. */
|
|
583
981
|
advance(run) {
|
|
@@ -585,6 +983,9 @@ export class FakePipelines {
|
|
|
585
983
|
return;
|
|
586
984
|
if (run.notBefore > Date.now() && run.cancelRequestedAt === undefined)
|
|
587
985
|
return;
|
|
986
|
+
// A stored pipeline's run waits QUEUED under its policy's cap (the tick looks again).
|
|
987
|
+
if (run.state === RunState.QUEUED && run.cancelRequestedAt === undefined && this.atCap(run))
|
|
988
|
+
return;
|
|
588
989
|
for (let changed = true; changed;) {
|
|
589
990
|
changed = false;
|
|
590
991
|
for (const n of run.nodes.values()) {
|
|
@@ -604,16 +1005,7 @@ export class FakePipelines {
|
|
|
604
1005
|
run.startedAt ??= Date.now();
|
|
605
1006
|
if (run.state === RunState.QUEUED)
|
|
606
1007
|
run.state = RunState.RUNNING;
|
|
607
|
-
|
|
608
|
-
const inputs = this.resolve(run, n);
|
|
609
|
-
if (typeof inputs === "string") {
|
|
610
|
-
n.state = NodeState.FAILED;
|
|
611
|
-
n.failureClass = FailureClass.INVALID_IO;
|
|
612
|
-
}
|
|
613
|
-
else {
|
|
614
|
-
this.offer(run, n, inputs, 0);
|
|
615
|
-
}
|
|
616
|
-
}
|
|
1008
|
+
this.dispatch(run, n);
|
|
617
1009
|
changed = true;
|
|
618
1010
|
}
|
|
619
1011
|
}
|
|
@@ -633,38 +1025,208 @@ export class FakePipelines {
|
|
|
633
1025
|
else {
|
|
634
1026
|
run.state = RunState.SUCCEEDED;
|
|
635
1027
|
}
|
|
1028
|
+
this.runFinished(run);
|
|
636
1029
|
}
|
|
637
1030
|
}
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
1031
|
+
/**
|
|
1032
|
+
* A node's `with` for one attempt (of one element): whole-value
|
|
1033
|
+
* references, also inside lists and objects, resolved. A string is why
|
|
1034
|
+
* one does not resolve.
|
|
1035
|
+
*/
|
|
1036
|
+
resolve(run, n, el) {
|
|
1037
|
+
let failed = "";
|
|
1038
|
+
const walk = (v) => {
|
|
1039
|
+
if (Array.isArray(v))
|
|
1040
|
+
return v.map(walk);
|
|
1041
|
+
if (v !== null && typeof v === "object")
|
|
1042
|
+
return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
|
|
641
1043
|
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
642
|
-
if (!m)
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
1044
|
+
if (!m)
|
|
1045
|
+
return v;
|
|
1046
|
+
const value = this.lookup(run, m[1], el);
|
|
1047
|
+
if (value === undefined)
|
|
1048
|
+
failed ||= `${m[1]} does not resolve`;
|
|
1049
|
+
return value ?? null;
|
|
1050
|
+
};
|
|
1051
|
+
const out = walk(n.with);
|
|
1052
|
+
return failed || out;
|
|
1053
|
+
}
|
|
1054
|
+
/** One reference's value: an upstream kv output, a variable, or the element. */
|
|
1055
|
+
lookup(run, expr, el) {
|
|
1056
|
+
const path = expr.split(".");
|
|
1057
|
+
if ((path[0] === "steps" || path[0] === "needs") && path[2] === "outputs" && path[3] === "kv" && path[4])
|
|
1058
|
+
return run.nodes.get(path[1])?.kv[path[4]];
|
|
1059
|
+
if (path[0] === "variables" && path[1])
|
|
1060
|
+
return run.variables[path[1]];
|
|
1061
|
+
if (el && path[0] === "item_index" && path.length === 1)
|
|
1062
|
+
return el.index;
|
|
1063
|
+
if (el && path[0] === "item") {
|
|
1064
|
+
let v = el.item;
|
|
1065
|
+
for (const k of path.slice(1))
|
|
1066
|
+
v = v !== null && typeof v === "object" && !Array.isArray(v) ? v[k] : undefined;
|
|
1067
|
+
return v;
|
|
1068
|
+
}
|
|
1069
|
+
return undefined;
|
|
1070
|
+
}
|
|
1071
|
+
/** A node made ready: its attempt (or child run), its elements, or, for an external node, nothing yet. */
|
|
1072
|
+
dispatch(run, n) {
|
|
1073
|
+
if (n.forEach) {
|
|
1074
|
+
const items = this.items(run, n);
|
|
1075
|
+
if (typeof items === "string") {
|
|
1076
|
+
n.state = NodeState.FAILED;
|
|
1077
|
+
n.failureClass = FailureClass.INVALID_IO;
|
|
1078
|
+
return;
|
|
652
1079
|
}
|
|
653
|
-
|
|
654
|
-
|
|
1080
|
+
n.elements = items.map((item, index) => ({ index, item, state: "pending", failureClass: FailureClass.UNSPECIFIED, kv: {} }));
|
|
1081
|
+
n.state = NodeState.RUNNING;
|
|
1082
|
+
this.launchElements(run, n);
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
1085
|
+
if (!n.handler && !n.pipeline)
|
|
1086
|
+
return;
|
|
1087
|
+
const inputs = this.resolve(run, n);
|
|
1088
|
+
if (typeof inputs === "string") {
|
|
1089
|
+
n.state = NodeState.FAILED;
|
|
1090
|
+
n.failureClass = FailureClass.INVALID_IO;
|
|
1091
|
+
}
|
|
1092
|
+
else {
|
|
1093
|
+
this.launch(run, n, inputs, undefined, 0);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
/** A for_each node's list: a JSON array, or a reference resolving to one, within `max_items`. */
|
|
1097
|
+
items(run, n) {
|
|
1098
|
+
const fe = n.forEach;
|
|
1099
|
+
const m = typeof fe.items === "string" ? REF.exec(fe.items) : null;
|
|
1100
|
+
const v = m ? this.lookup(run, m[1], undefined) : fe.items;
|
|
1101
|
+
if (!Array.isArray(v))
|
|
1102
|
+
return "for_each.items does not resolve to a list";
|
|
1103
|
+
if (v.length > (fe.maxItems || 1000))
|
|
1104
|
+
return `for_each.items has ${v.length} elements, over max_items`;
|
|
1105
|
+
return v;
|
|
1106
|
+
}
|
|
1107
|
+
/** One attempt of a node (of one element): offered to a worker, or a sub-pipeline's child run started. */
|
|
1108
|
+
launch(run, n, inputs, el, delayMs) {
|
|
1109
|
+
if (n.pipeline)
|
|
1110
|
+
this.startChild(run, n, inputs, el);
|
|
1111
|
+
else
|
|
1112
|
+
this.offer(run, n, inputs, delayMs, el);
|
|
1113
|
+
}
|
|
1114
|
+
/**
|
|
1115
|
+
* Starts elements while `max_parallel` has room, and settles the node
|
|
1116
|
+
* once none is live and none will start: under FAIL_FAST nothing starts
|
|
1117
|
+
* after a failure.
|
|
1118
|
+
*/
|
|
1119
|
+
launchElements(run, n) {
|
|
1120
|
+
const fe = n.forEach;
|
|
1121
|
+
const els = n.elements;
|
|
1122
|
+
const policy = fe.onFailure || ForEachFailure.FAIL_FAST;
|
|
1123
|
+
const stopped = () => run.cancelRequestedAt !== undefined || (policy === ForEachFailure.FAIL_FAST && els.some((e) => e.state === "failed"));
|
|
1124
|
+
let live = els.filter((e) => e.state === "running").length;
|
|
1125
|
+
for (const el of els) {
|
|
1126
|
+
if (stopped() || live >= (fe.maxParallel || 10))
|
|
1127
|
+
break;
|
|
1128
|
+
if (el.state !== "pending")
|
|
1129
|
+
continue;
|
|
1130
|
+
const inputs = this.resolve(run, n, el);
|
|
1131
|
+
if (typeof inputs === "string") {
|
|
1132
|
+
el.state = "failed";
|
|
1133
|
+
el.failureClass = FailureClass.INVALID_IO;
|
|
1134
|
+
continue;
|
|
655
1135
|
}
|
|
656
|
-
|
|
657
|
-
|
|
1136
|
+
el.state = "running";
|
|
1137
|
+
live++;
|
|
1138
|
+
this.launch(run, n, inputs, el, 0);
|
|
1139
|
+
}
|
|
1140
|
+
if (live > 0 || (!stopped() && els.some((e) => e.state === "pending")))
|
|
1141
|
+
return;
|
|
1142
|
+
// Settled: outputs as lists in element order, null where an element gave none.
|
|
1143
|
+
const slots = [...new Set(els.flatMap((e) => (e.state === "succeeded" ? Object.keys(e.kv) : [])))];
|
|
1144
|
+
n.kv = Object.fromEntries(slots.map((k) => [k, els.map((e) => (e.state === "succeeded" ? (e.kv[k] ?? null) : null))]));
|
|
1145
|
+
const failed = els.find((e) => e.state === "failed");
|
|
1146
|
+
if (run.cancelRequestedAt !== undefined) {
|
|
1147
|
+
n.state = NodeState.CANCELLED;
|
|
1148
|
+
n.failureClass = FailureClass.CANCELLED;
|
|
1149
|
+
}
|
|
1150
|
+
else if (failed && policy !== ForEachFailure.ALLOW) {
|
|
1151
|
+
n.state = NodeState.FAILED;
|
|
1152
|
+
n.failureClass = failed.failureClass;
|
|
1153
|
+
}
|
|
1154
|
+
else {
|
|
1155
|
+
n.state = NodeState.SUCCEEDED;
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
/** A sub-pipeline node's attempt: one child run of the pinned version, about the parent's resource. */
|
|
1159
|
+
startChild(run, n, inputs, el) {
|
|
1160
|
+
const a = this.newAttempt(run, n, inputs, 0, el);
|
|
1161
|
+
a.state = AttemptState.RUNNING;
|
|
1162
|
+
a.leaseExpiresAt = Number.POSITIVE_INFINITY; // held by the platform, not a worker
|
|
1163
|
+
a.startedAt = Date.now();
|
|
1164
|
+
n.attempts.push(a);
|
|
1165
|
+
this.attemptsById.set(a.id, a);
|
|
1166
|
+
if (!n.forEach)
|
|
1167
|
+
n.state = NodeState.RUNNING;
|
|
1168
|
+
try {
|
|
1169
|
+
const child = this.startStored({ tenant: run.tenant, scopes: ALL_SCOPES }, {
|
|
1170
|
+
tenant: run.tenant,
|
|
1171
|
+
key: `parent:${run.id}:${n.id}:${a.id}`,
|
|
1172
|
+
resource: run.resource,
|
|
1173
|
+
variables: inputs,
|
|
1174
|
+
facts: run.attestedFacts,
|
|
1175
|
+
stored: this.pipelines.get(n.pipeline.id),
|
|
1176
|
+
version: n.pipeline.version,
|
|
1177
|
+
trigger: RunTrigger.PARENT,
|
|
1178
|
+
parent: { runId: run.id, nodeId: n.id },
|
|
1179
|
+
});
|
|
1180
|
+
a.childRunId = child.id;
|
|
1181
|
+
}
|
|
1182
|
+
catch (err) {
|
|
1183
|
+
// Its variables, resolved now, refused: the attempt fails, as the child would.
|
|
1184
|
+
a.state = AttemptState.FAILED;
|
|
1185
|
+
a.failureClass = FailureClass.DOWNSTREAM_FAILED;
|
|
1186
|
+
a.message = ConnectError.from(err).rawMessage;
|
|
1187
|
+
a.completedAt = Date.now();
|
|
1188
|
+
queueMicrotask(() => this.settle(a));
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
/** A child run ended: its parent's attempt takes its result. */
|
|
1192
|
+
childFinished(child) {
|
|
1193
|
+
const parent = this.runs.get(child.parentRunId);
|
|
1194
|
+
const n = parent?.nodes.get(child.parentNodeId);
|
|
1195
|
+
const a = n?.attempts.find((x) => x.childRunId === child.id);
|
|
1196
|
+
if (!n || !a || a.state !== AttemptState.RUNNING)
|
|
1197
|
+
return;
|
|
1198
|
+
a.completedAt = Date.now();
|
|
1199
|
+
if (child.state === RunState.SUCCEEDED) {
|
|
1200
|
+
a.state = AttemptState.SUCCEEDED;
|
|
1201
|
+
for (const o of n.pipeline.outputs) {
|
|
1202
|
+
const v = child.nodes.get(o.node)?.kv[o.slot];
|
|
1203
|
+
if (v !== undefined)
|
|
1204
|
+
a.kv[o.name] = v;
|
|
658
1205
|
}
|
|
659
1206
|
}
|
|
660
|
-
|
|
1207
|
+
else if (child.state === RunState.CANCELLED) {
|
|
1208
|
+
a.state = AttemptState.CANCELLED;
|
|
1209
|
+
a.failureClass = FailureClass.CANCELLED;
|
|
1210
|
+
}
|
|
1211
|
+
else {
|
|
1212
|
+
a.state = AttemptState.FAILED;
|
|
1213
|
+
a.failureClass = FailureClass.DOWNSTREAM_FAILED;
|
|
1214
|
+
a.message = `the child run ${child.id} failed`;
|
|
1215
|
+
}
|
|
1216
|
+
this.settle(a);
|
|
1217
|
+
}
|
|
1218
|
+
offer(run, n, inputs, delayMs, el) {
|
|
1219
|
+
const a = this.newAttempt(run, n, inputs, delayMs, el);
|
|
1220
|
+
n.attempts.push(a);
|
|
1221
|
+
this.attemptsById.set(a.id, a);
|
|
1222
|
+
this.notify();
|
|
661
1223
|
}
|
|
662
|
-
|
|
663
|
-
|
|
1224
|
+
newAttempt(run, n, inputs, delayMs, el) {
|
|
1225
|
+
return {
|
|
664
1226
|
id: randomUUID(),
|
|
665
1227
|
runId: run.id,
|
|
666
1228
|
nodeId: n.id,
|
|
667
|
-
number: n.attempts.length + 1,
|
|
1229
|
+
number: n.attempts.filter((x) => x.itemIndex === el?.index).length + 1,
|
|
668
1230
|
state: AttemptState.OFFERED,
|
|
669
1231
|
epoch: 0n,
|
|
670
1232
|
workerId: "",
|
|
@@ -683,14 +1245,23 @@ export class FakePipelines {
|
|
|
683
1245
|
completedAt: undefined,
|
|
684
1246
|
handlerFailureKind: HandlerFailureKind.UNSPECIFIED,
|
|
685
1247
|
releases: 0,
|
|
1248
|
+
offeredAt: Date.now(),
|
|
1249
|
+
progress: undefined,
|
|
1250
|
+
spans: [],
|
|
1251
|
+
itemIndex: el?.index,
|
|
1252
|
+
item: el?.item,
|
|
1253
|
+
childRunId: "",
|
|
686
1254
|
};
|
|
687
|
-
n.attempts.push(a);
|
|
688
|
-
this.attemptsById.set(a.id, a);
|
|
689
|
-
this.notify();
|
|
690
1255
|
}
|
|
1256
|
+
/**
|
|
1257
|
+
* Claims for a worker: the attempts of handler nodes of full runs first,
|
|
1258
|
+
* in creation order, as they always were; then jobs, through their
|
|
1259
|
+
* queues' limits and fairness rule (see ./queues.ts).
|
|
1260
|
+
*/
|
|
691
1261
|
claim(pool, handlers, workerId, max) {
|
|
692
1262
|
const now = Date.now();
|
|
693
1263
|
const out = [];
|
|
1264
|
+
const queued = [];
|
|
694
1265
|
for (const run of this.runs.values()) {
|
|
695
1266
|
if (run.tenant !== pool.owner?.tenant || run.cancelRequestedAt !== undefined || run.notBefore > now)
|
|
696
1267
|
continue;
|
|
@@ -698,21 +1269,48 @@ export class FakePipelines {
|
|
|
698
1269
|
if (!handlers.includes(n.handler))
|
|
699
1270
|
continue;
|
|
700
1271
|
for (const a of n.attempts) {
|
|
701
|
-
if (out.length >= max)
|
|
702
|
-
return out;
|
|
703
1272
|
if (a.state !== AttemptState.OFFERED || a.notBefore > now)
|
|
704
1273
|
continue;
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
a.leaseExpiresAt = now + this.opts.leaseMs;
|
|
710
|
-
out.push(a);
|
|
1274
|
+
if (run.queueId)
|
|
1275
|
+
queued.push({ item: a, queueId: run.queueId, lane: run.fairnessKey, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
|
|
1276
|
+
else if (out.length < max)
|
|
1277
|
+
out.push(a);
|
|
711
1278
|
}
|
|
712
1279
|
}
|
|
713
1280
|
}
|
|
1281
|
+
if (out.length < max && queued.length > 0) {
|
|
1282
|
+
const limits = new Map();
|
|
1283
|
+
for (const q of this.queues.values())
|
|
1284
|
+
limits.set(q.id, queueLimits(q));
|
|
1285
|
+
const picks = pickQueued(queued, limits, { running: (q, t) => this.queueRunning(q, t), claimedSince: (q, since) => (this.queueClaims.get(q) ?? []).filter((x) => x >= since).length }, now, max - out.length);
|
|
1286
|
+
for (const p of picks) {
|
|
1287
|
+
out.push(p.item);
|
|
1288
|
+
this.queueClaims.set(p.queueId, [...(this.queueClaims.get(p.queueId) ?? []).filter((x) => x > now - 86_400_000), now]);
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
for (const a of out) {
|
|
1292
|
+
a.state = AttemptState.CLAIMED;
|
|
1293
|
+
a.epoch += 1n;
|
|
1294
|
+
a.workerId = workerId;
|
|
1295
|
+
a.poolId = pool.id;
|
|
1296
|
+
a.leaseExpiresAt = now + this.opts.leaseMs;
|
|
1297
|
+
this.initProgress(a);
|
|
1298
|
+
}
|
|
714
1299
|
return out;
|
|
715
1300
|
}
|
|
1301
|
+
/** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
|
|
1302
|
+
queueRunning(queueId, lane) {
|
|
1303
|
+
let n = 0;
|
|
1304
|
+
for (const run of this.runs.values()) {
|
|
1305
|
+
if (run.queueId !== queueId || (lane !== undefined && run.fairnessKey !== lane))
|
|
1306
|
+
continue;
|
|
1307
|
+
for (const node of run.nodes.values())
|
|
1308
|
+
for (const a of node.attempts)
|
|
1309
|
+
if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING)
|
|
1310
|
+
n++;
|
|
1311
|
+
}
|
|
1312
|
+
return n;
|
|
1313
|
+
}
|
|
716
1314
|
describe(a) {
|
|
717
1315
|
const run = this.run(a.runId);
|
|
718
1316
|
const n = run.nodes.get(a.nodeId);
|
|
@@ -754,13 +1352,28 @@ export class FakePipelines {
|
|
|
754
1352
|
settle(a) {
|
|
755
1353
|
const run = this.run(a.runId);
|
|
756
1354
|
const n = run.nodes.get(a.nodeId);
|
|
1355
|
+
// An element's attempt settles its element; the node settles with the last one.
|
|
1356
|
+
const el = a.itemIndex !== undefined ? n.elements?.[a.itemIndex] : undefined;
|
|
1357
|
+
const mine = n.attempts.filter((x) => x.itemIndex === a.itemIndex);
|
|
757
1358
|
if (a.state === AttemptState.SUCCEEDED) {
|
|
758
|
-
|
|
759
|
-
|
|
1359
|
+
if (el) {
|
|
1360
|
+
el.state = "succeeded";
|
|
1361
|
+
el.kv = a.kv;
|
|
1362
|
+
}
|
|
1363
|
+
else {
|
|
1364
|
+
n.state = NodeState.SUCCEEDED;
|
|
1365
|
+
n.kv = a.kv;
|
|
1366
|
+
}
|
|
760
1367
|
}
|
|
761
1368
|
else if (a.state === AttemptState.CANCELLED || run.cancelRequestedAt !== undefined) {
|
|
762
|
-
|
|
763
|
-
|
|
1369
|
+
if (el) {
|
|
1370
|
+
el.state = "failed";
|
|
1371
|
+
el.failureClass = FailureClass.CANCELLED;
|
|
1372
|
+
}
|
|
1373
|
+
else {
|
|
1374
|
+
n.state = NodeState.CANCELLED;
|
|
1375
|
+
n.failureClass = FailureClass.CANCELLED;
|
|
1376
|
+
}
|
|
764
1377
|
}
|
|
765
1378
|
else {
|
|
766
1379
|
const permanent = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.PERMANENT;
|
|
@@ -768,19 +1381,32 @@ export class FakePipelines {
|
|
|
768
1381
|
const when = n.retry.when.length > 0 ? n.retry.when : transientHandler ? [...TRANSIENT, FailureClass.HANDLER_ERROR] : TRANSIENT;
|
|
769
1382
|
// On an interruptible node an interruption is free, up to a bound.
|
|
770
1383
|
const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
|
|
771
|
-
const freeUsed =
|
|
1384
|
+
const freeUsed = mine.filter(free).length;
|
|
772
1385
|
if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
|
|
773
|
-
|
|
1386
|
+
// Up to the bound, an interruption is offered again at once.
|
|
1387
|
+
this.launch(run, n, a.inputs, el, 0);
|
|
774
1388
|
return;
|
|
775
1389
|
}
|
|
776
|
-
const retries =
|
|
777
|
-
|
|
778
|
-
|
|
1390
|
+
const retries = mine.slice(0, -1).filter((x) => !free(x)).length;
|
|
1391
|
+
// As the api: a worker that vanished (ABANDONED) may have had outside
|
|
1392
|
+
// effects, so its node runs again automatically only when it declares
|
|
1393
|
+
// itself interruptible.
|
|
1394
|
+
const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
|
|
1395
|
+
if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
|
|
1396
|
+
this.launch(run, n, a.inputs, el, this.opts.retryBackoffMs);
|
|
779
1397
|
return;
|
|
780
1398
|
}
|
|
781
|
-
|
|
782
|
-
|
|
1399
|
+
if (el) {
|
|
1400
|
+
el.state = "failed";
|
|
1401
|
+
el.failureClass = a.failureClass;
|
|
1402
|
+
}
|
|
1403
|
+
else {
|
|
1404
|
+
n.state = NodeState.FAILED;
|
|
1405
|
+
n.failureClass = a.failureClass;
|
|
1406
|
+
}
|
|
783
1407
|
}
|
|
1408
|
+
if (el)
|
|
1409
|
+
this.launchElements(run, n);
|
|
784
1410
|
this.advance(run);
|
|
785
1411
|
this.notify();
|
|
786
1412
|
}
|
|
@@ -790,8 +1416,11 @@ export class FakePipelines {
|
|
|
790
1416
|
run.cancelRequestedAt = Date.now();
|
|
791
1417
|
run.cancelReason = reason;
|
|
792
1418
|
run.state = RunState.CANCELLING;
|
|
1419
|
+
const children = [];
|
|
793
1420
|
for (const n of run.nodes.values()) {
|
|
794
1421
|
for (const a of n.attempts) {
|
|
1422
|
+
if (a.state === AttemptState.RUNNING && a.childRunId)
|
|
1423
|
+
children.push(this.run(a.childRunId));
|
|
795
1424
|
if (a.state === AttemptState.OFFERED) {
|
|
796
1425
|
a.state = AttemptState.CANCELLED;
|
|
797
1426
|
a.failureClass = FailureClass.CANCELLED;
|
|
@@ -803,6 +1432,9 @@ export class FakePipelines {
|
|
|
803
1432
|
n.state = n.state === NodeState.PENDING ? NodeState.SKIPPED : NodeState.CANCELLED;
|
|
804
1433
|
}
|
|
805
1434
|
}
|
|
1435
|
+
// Cancelling a run cancels its sub-pipelines' children; each one's end settles its attempt.
|
|
1436
|
+
for (const c of children)
|
|
1437
|
+
this.cancel(c, reason);
|
|
806
1438
|
this.advance(run);
|
|
807
1439
|
this.notify();
|
|
808
1440
|
}
|
|
@@ -836,6 +1468,7 @@ export class FakePipelines {
|
|
|
836
1468
|
}
|
|
837
1469
|
}
|
|
838
1470
|
}
|
|
1471
|
+
this.schedules.evaluate();
|
|
839
1472
|
this.notify();
|
|
840
1473
|
}
|
|
841
1474
|
notify() {
|
|
@@ -843,13 +1476,449 @@ export class FakePipelines {
|
|
|
843
1476
|
this.changed = signal();
|
|
844
1477
|
prev.resolve();
|
|
845
1478
|
}
|
|
1479
|
+
/** QueuesService, as the contract states it. */
|
|
1480
|
+
queuesService() {
|
|
1481
|
+
const readable = (who, q) => {
|
|
1482
|
+
if (!q || q.owner?.tenant !== who.tenant)
|
|
1483
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1484
|
+
return q;
|
|
1485
|
+
};
|
|
1486
|
+
const byId = (who, id) => readable(who, this.queues.get(id));
|
|
1487
|
+
return {
|
|
1488
|
+
setQueue: (req, ctx) => {
|
|
1489
|
+
const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
|
|
1490
|
+
const q = req.queue;
|
|
1491
|
+
if (!q?.owner?.tenant)
|
|
1492
|
+
throw new ConnectError("queue.owner.tenant: required", Code.InvalidArgument);
|
|
1493
|
+
if (q.owner.tenant !== who.tenant)
|
|
1494
|
+
throw new ConnectError("queue.owner.tenant: the caller is not a service-account member of it", Code.PermissionDenied);
|
|
1495
|
+
if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
|
|
1496
|
+
throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
|
|
1497
|
+
if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
|
|
1498
|
+
throw new ConnectError("max_concurrency_per_tenant must not exceed a non-zero max_concurrency", Code.InvalidArgument);
|
|
1499
|
+
}
|
|
1500
|
+
if (req.updateMask)
|
|
1501
|
+
throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
|
|
1502
|
+
const existing = [...this.queues.values()].find((x) => x.owner?.tenant === q.owner.tenant && x.name === q.name);
|
|
1503
|
+
const now = Date.now();
|
|
1504
|
+
const next = create(QueueSchema, {
|
|
1505
|
+
...q,
|
|
1506
|
+
id: existing?.id ?? randomUUID(),
|
|
1507
|
+
paused: existing?.paused ?? false,
|
|
1508
|
+
pausedReason: existing?.pausedReason ?? "",
|
|
1509
|
+
pausedBy: existing?.pausedBy ?? "",
|
|
1510
|
+
...(existing?.pausedAt ? { pausedAt: existing.pausedAt } : {}),
|
|
1511
|
+
createdAt: existing?.createdAt ?? timestampFromMs(now),
|
|
1512
|
+
updatedAt: timestampFromMs(now),
|
|
1513
|
+
});
|
|
1514
|
+
this.queues.set(next.id, next);
|
|
1515
|
+
this.notify();
|
|
1516
|
+
return { queue: next, created: !existing };
|
|
1517
|
+
},
|
|
1518
|
+
getQueue: (req, ctx) => {
|
|
1519
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1520
|
+
if (req.queue.case === "queueId")
|
|
1521
|
+
return { queue: byId(who, req.queue.value) };
|
|
1522
|
+
if (req.queue.case !== "byName")
|
|
1523
|
+
throw new ConnectError("queue: required", Code.InvalidArgument);
|
|
1524
|
+
const { owner, name } = req.queue.value;
|
|
1525
|
+
if (!owner?.tenant || owner.tenant !== who.tenant)
|
|
1526
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1527
|
+
const found = [...this.queues.values()].find((x) => x.owner?.tenant === owner.tenant && x.name === name);
|
|
1528
|
+
if (found)
|
|
1529
|
+
return { queue: readable(who, found) };
|
|
1530
|
+
if (name === "default")
|
|
1531
|
+
return { queue: this.defaultQueue(owner.tenant) };
|
|
1532
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1533
|
+
},
|
|
1534
|
+
listQueues: (req, ctx) => {
|
|
1535
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1536
|
+
if (!req.tenant)
|
|
1537
|
+
throw new ConnectError("tenant: required", Code.InvalidArgument);
|
|
1538
|
+
if (req.tenant !== who.tenant)
|
|
1539
|
+
return { queues: [], nextCursor: "" };
|
|
1540
|
+
const queues = [...this.queues.values()]
|
|
1541
|
+
.filter((q) => q.owner?.tenant === req.tenant)
|
|
1542
|
+
.sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
|
|
1543
|
+
return { queues, nextCursor: "" };
|
|
1544
|
+
},
|
|
1545
|
+
markQueuePaused: (req, ctx) => {
|
|
1546
|
+
const who = this.enter("MarkQueuePaused", ctx, "pipelines:queue:edit");
|
|
1547
|
+
const q = byId(who, req.queueId);
|
|
1548
|
+
if (!q.paused) {
|
|
1549
|
+
q.paused = true;
|
|
1550
|
+
q.pausedAt = timestampFromMs(Date.now());
|
|
1551
|
+
q.pausedBy = `urn:fake:sa:${who.tenant}`;
|
|
1552
|
+
q.pausedReason = req.reason;
|
|
1553
|
+
}
|
|
1554
|
+
return { queue: q };
|
|
1555
|
+
},
|
|
1556
|
+
markQueueResumed: (req, ctx) => {
|
|
1557
|
+
const who = this.enter("MarkQueueResumed", ctx, "pipelines:queue:edit");
|
|
1558
|
+
const q = byId(who, req.queueId);
|
|
1559
|
+
q.paused = false;
|
|
1560
|
+
q.pausedReason = "";
|
|
1561
|
+
q.pausedBy = "";
|
|
1562
|
+
delete q.pausedAt;
|
|
1563
|
+
this.notify();
|
|
1564
|
+
return { queue: q };
|
|
1565
|
+
},
|
|
1566
|
+
deleteQueue: (req, ctx) => {
|
|
1567
|
+
const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
|
|
1568
|
+
const q = byId(who, req.queueId);
|
|
1569
|
+
const waiting = [...this.runs.values()].filter((r) => r.queueId === q.id && (r.state === RunState.QUEUED || (r.state === RunState.RUNNING && this.attemptsOf(r.id).some((a) => a.state === AttemptState.OFFERED))));
|
|
1570
|
+
if (waiting.length > 0 && !req.cancelWaiting)
|
|
1571
|
+
throw new ConnectError(`queue ${q.name} has ${waiting.length} waiting jobs`, Code.FailedPrecondition);
|
|
1572
|
+
for (const r of waiting)
|
|
1573
|
+
this.cancel(r, "queue deleted");
|
|
1574
|
+
if (q.name === "default" && !q.owner?.tenant) {
|
|
1575
|
+
this.queues.set(q.id, create(QueueSchema, { id: q.id, owner: q.owner, name: "default", createdAt: q.createdAt, updatedAt: timestampFromMs(Date.now()) }));
|
|
1576
|
+
}
|
|
1577
|
+
else {
|
|
1578
|
+
this.queues.delete(q.id);
|
|
1579
|
+
}
|
|
1580
|
+
return { cancelledJobs: waiting.length };
|
|
1581
|
+
},
|
|
1582
|
+
getQueueStats: (req, ctx) => {
|
|
1583
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1584
|
+
const q = byId(who, req.queueId);
|
|
1585
|
+
const now = Date.now();
|
|
1586
|
+
const windowMs = req.window ? durationMs(req.window) : 3_600_000;
|
|
1587
|
+
const runs = [...this.runs.values()].filter((r) => r.queueId === q.id);
|
|
1588
|
+
let waiting = 0;
|
|
1589
|
+
let delayed = 0;
|
|
1590
|
+
let running = 0;
|
|
1591
|
+
let started = 0n;
|
|
1592
|
+
let oldest;
|
|
1593
|
+
for (const r of runs) {
|
|
1594
|
+
for (const a of this.attemptsOf(r.id)) {
|
|
1595
|
+
if (a.state === AttemptState.OFFERED) {
|
|
1596
|
+
if (a.notBefore > now || r.notBefore > now)
|
|
1597
|
+
delayed++;
|
|
1598
|
+
else {
|
|
1599
|
+
waiting++;
|
|
1600
|
+
const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
|
|
1601
|
+
oldest = oldest === undefined ? since : Math.min(oldest, since);
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
|
|
1605
|
+
running++;
|
|
1606
|
+
}
|
|
1607
|
+
if (a.startedAt && a.startedAt >= now - windowMs)
|
|
1608
|
+
started++;
|
|
1609
|
+
}
|
|
1610
|
+
if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
|
|
1611
|
+
delayed++;
|
|
1612
|
+
}
|
|
1613
|
+
const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
|
|
1614
|
+
return {
|
|
1615
|
+
waiting,
|
|
1616
|
+
delayed,
|
|
1617
|
+
running,
|
|
1618
|
+
...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
|
|
1619
|
+
started,
|
|
1620
|
+
failed,
|
|
1621
|
+
paused: q.paused,
|
|
1622
|
+
at: timestampFromMs(now),
|
|
1623
|
+
};
|
|
1624
|
+
},
|
|
1625
|
+
listQueueJobs: (req, ctx) => {
|
|
1626
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1627
|
+
const q = byId(who, req.queueId);
|
|
1628
|
+
const runs = [...this.runs.values()]
|
|
1629
|
+
.filter((r) => r.queueId === q.id && (req.states.length === 0 || req.states.includes(r.state)))
|
|
1630
|
+
.reverse();
|
|
1631
|
+
return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
|
|
1632
|
+
},
|
|
1633
|
+
};
|
|
1634
|
+
}
|
|
1635
|
+
// ---- jobs and queues --------------------------------------------------
|
|
1636
|
+
/** CreateJob's work, shared with schedules: resolve the queue, apply its limits, make the one-node run. */
|
|
1637
|
+
createJobRun(who, spec) {
|
|
1638
|
+
this.requireHandler(spec.handler);
|
|
1639
|
+
if (spec.retry && spec.retry.max > 10)
|
|
1640
|
+
throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
|
|
1641
|
+
if (spec.retry?.when.includes(FailureClass.CANCELLED))
|
|
1642
|
+
throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
|
|
1643
|
+
const q = this.resolveQueue(spec.tenant, spec.queue);
|
|
1644
|
+
if (!q)
|
|
1645
|
+
throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among the tenant's queues`, Code.NotFound);
|
|
1646
|
+
const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
|
|
1647
|
+
const payload = this.checkInputs("job", spec.handler, spec.payload);
|
|
1648
|
+
const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
|
|
1649
|
+
run.queueId = q.id;
|
|
1650
|
+
run.queue = q.name;
|
|
1651
|
+
run.fairnessKey = spec.fairnessKey ?? "";
|
|
1652
|
+
run.trigger = spec.trigger ?? RunTrigger.API;
|
|
1653
|
+
if (spec.schedule) {
|
|
1654
|
+
run.scheduleId = spec.schedule.id;
|
|
1655
|
+
run.scheduledFor = spec.schedule.scheduledFor;
|
|
1656
|
+
run.scheduleTarget = spec.schedule.target;
|
|
1657
|
+
}
|
|
1658
|
+
this.addNode(run, {
|
|
1659
|
+
id: "job",
|
|
1660
|
+
handler: spec.handler,
|
|
1661
|
+
uses: "",
|
|
1662
|
+
with: payload,
|
|
1663
|
+
needs: [],
|
|
1664
|
+
retry,
|
|
1665
|
+
timeoutMs,
|
|
1666
|
+
interruptible: spec.interruptible,
|
|
1667
|
+
});
|
|
1668
|
+
this.advance(run);
|
|
1669
|
+
return run;
|
|
1670
|
+
}
|
|
1671
|
+
/** A job's queue: the tenant's of that name; empty is `default`. */
|
|
1672
|
+
resolveQueue(tenant, name) {
|
|
1673
|
+
const want = name || "default";
|
|
1674
|
+
const found = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === want);
|
|
1675
|
+
if (found)
|
|
1676
|
+
return found;
|
|
1677
|
+
return want === "default" ? this.defaultQueue(tenant) : undefined;
|
|
1678
|
+
}
|
|
1679
|
+
/** The tenant's `default` queue, made on first use with no limits. */
|
|
1680
|
+
defaultQueue(tenant) {
|
|
1681
|
+
const existing = [...this.queues.values()].find((q) => q.owner?.tenant === tenant && q.name === "default");
|
|
1682
|
+
if (existing)
|
|
1683
|
+
return existing;
|
|
1684
|
+
const now = Date.now();
|
|
1685
|
+
const q = create(QueueSchema, { id: randomUUID(), owner: { tenant }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
|
|
1686
|
+
this.queues.set(q.id, q);
|
|
1687
|
+
return q;
|
|
1688
|
+
}
|
|
1689
|
+
/** RetryNode: reopens a finished run and offers the node again; on a job, a requeue. */
|
|
1690
|
+
retryNode(run, nodeId) {
|
|
1691
|
+
const n = run.nodes.get(nodeId);
|
|
1692
|
+
if (!n)
|
|
1693
|
+
throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
|
|
1694
|
+
if (!terminal(run.state))
|
|
1695
|
+
throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
|
|
1696
|
+
if ((!n.handler && !n.pipeline) || (n.attempts.length === 0 && !n.forEach) || !terminalNode(n.state))
|
|
1697
|
+
throw new ConnectError("the node never settled", Code.FailedPrecondition);
|
|
1698
|
+
run.state = RunState.RUNNING;
|
|
1699
|
+
run.finishedAt = undefined;
|
|
1700
|
+
run.failureClass = FailureClass.UNSPECIFIED;
|
|
1701
|
+
run.failedNodeId = "";
|
|
1702
|
+
run.cancelRequestedAt = undefined;
|
|
1703
|
+
// The node and everything downstream of it run again.
|
|
1704
|
+
const reset = new Set([nodeId]);
|
|
1705
|
+
for (let grew = true; grew;) {
|
|
1706
|
+
grew = false;
|
|
1707
|
+
for (const x of run.nodes.values()) {
|
|
1708
|
+
if (!reset.has(x.id) && x.needs.some((d) => reset.has(d))) {
|
|
1709
|
+
reset.add(x.id);
|
|
1710
|
+
grew = true;
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
for (const id of reset) {
|
|
1715
|
+
const x = run.nodes.get(id);
|
|
1716
|
+
x.state = id === nodeId ? NodeState.READY : NodeState.PENDING;
|
|
1717
|
+
x.failureClass = FailureClass.UNSPECIFIED;
|
|
1718
|
+
x.kv = {};
|
|
1719
|
+
}
|
|
1720
|
+
if (n.forEach && n.elements) {
|
|
1721
|
+
// Only the failed elements run again; the succeeded ones are kept.
|
|
1722
|
+
for (const el of n.elements) {
|
|
1723
|
+
if (el.state === "succeeded")
|
|
1724
|
+
continue;
|
|
1725
|
+
el.state = "pending";
|
|
1726
|
+
el.failureClass = FailureClass.UNSPECIFIED;
|
|
1727
|
+
el.kv = {};
|
|
1728
|
+
}
|
|
1729
|
+
n.state = NodeState.RUNNING;
|
|
1730
|
+
this.launchElements(run, n);
|
|
1731
|
+
this.advance(run);
|
|
1732
|
+
return;
|
|
1733
|
+
}
|
|
1734
|
+
if (n.forEach) {
|
|
1735
|
+
this.dispatch(run, n);
|
|
1736
|
+
this.advance(run);
|
|
1737
|
+
return;
|
|
1738
|
+
}
|
|
1739
|
+
const inputs = this.resolve(run, n);
|
|
1740
|
+
this.launch(run, n, typeof inputs === "string" ? {} : inputs, undefined, 0);
|
|
1741
|
+
}
|
|
1742
|
+
// ---- issues, progress and spans ---------------------------------------
|
|
1743
|
+
/** Every declared stage PENDING from the claim, as the platform does. */
|
|
1744
|
+
initProgress(a) {
|
|
1745
|
+
if (a.progress)
|
|
1746
|
+
return;
|
|
1747
|
+
const n = this.run(a.runId).nodes.get(a.nodeId);
|
|
1748
|
+
const pkg = this.packages.get(n.handler);
|
|
1749
|
+
const manifest = pkg ? [...pkg.versions.values()].at(-1) : undefined;
|
|
1750
|
+
const stages = manifest?.stages ?? [];
|
|
1751
|
+
if (stages.length === 0)
|
|
1752
|
+
return;
|
|
1753
|
+
a.progress = {
|
|
1754
|
+
fraction: 0,
|
|
1755
|
+
message: "",
|
|
1756
|
+
processed: 0n,
|
|
1757
|
+
total: 0n,
|
|
1758
|
+
failed: 0n,
|
|
1759
|
+
updatedAt: Date.now(),
|
|
1760
|
+
reports: 0,
|
|
1761
|
+
stages: stages.map((st) => ({ key: st.key, name: st.name, weight: st.weight || 1, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n })),
|
|
1762
|
+
};
|
|
1763
|
+
}
|
|
1764
|
+
setProgress(a, req) {
|
|
1765
|
+
const now = Date.now();
|
|
1766
|
+
const p = (a.progress ??= { fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n, stages: [], updatedAt: now, reports: 0 });
|
|
1767
|
+
p.reports++;
|
|
1768
|
+
p.updatedAt = now;
|
|
1769
|
+
p.message = req.message;
|
|
1770
|
+
p.processed = req.processed;
|
|
1771
|
+
p.total = req.total;
|
|
1772
|
+
p.failed = req.failed;
|
|
1773
|
+
if (!req.stage) {
|
|
1774
|
+
p.fraction = Math.max(p.fraction, req.fraction);
|
|
1775
|
+
return;
|
|
1776
|
+
}
|
|
1777
|
+
let st = p.stages.find((x) => x.key === req.stage);
|
|
1778
|
+
if (!st) {
|
|
1779
|
+
st = { key: req.stage, name: req.stageName || req.stage, weight: 0, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n };
|
|
1780
|
+
p.stages.push(st);
|
|
1781
|
+
}
|
|
1782
|
+
// Reporting a stage finishes the one before it.
|
|
1783
|
+
for (const other of p.stages) {
|
|
1784
|
+
if (other !== st && other.state === StageState.RUNNING) {
|
|
1785
|
+
other.state = StageState.SUCCEEDED;
|
|
1786
|
+
other.fraction = 1;
|
|
1787
|
+
other.finishedAt = now;
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
const state = req.stageState === StageState.UNSPECIFIED ? StageState.RUNNING : req.stageState;
|
|
1791
|
+
if (st.state === StageState.PENDING || st.state === StageState.RUNNING) {
|
|
1792
|
+
st.state = state;
|
|
1793
|
+
st.startedAt ??= now;
|
|
1794
|
+
if (state !== StageState.RUNNING)
|
|
1795
|
+
st.finishedAt = now;
|
|
1796
|
+
}
|
|
1797
|
+
st.fraction = state === StageState.SUCCEEDED ? 1 : req.fraction;
|
|
1798
|
+
st.message = req.message;
|
|
1799
|
+
st.processed = req.processed;
|
|
1800
|
+
st.total = req.total;
|
|
1801
|
+
st.failed = req.failed;
|
|
1802
|
+
const total = p.stages.reduce((s, x) => s + x.weight, 0);
|
|
1803
|
+
if (total > 0) {
|
|
1804
|
+
const done = p.stages.reduce((s, x) => s + (x.state === StageState.SUCCEEDED || x.state === StageState.FAILED || x.state === StageState.SKIPPED ? x.weight : x.state === StageState.RUNNING ? x.weight * x.fraction : 0), 0);
|
|
1805
|
+
p.fraction = Math.max(p.fraction, Math.min(1, done / total));
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
appendIssues(a, issues) {
|
|
1809
|
+
const run = this.run(a.runId);
|
|
1810
|
+
let accepted = 0;
|
|
1811
|
+
for (const i of issues) {
|
|
1812
|
+
if (!i.message)
|
|
1813
|
+
throw new ConnectError("issues.message: required", Code.InvalidArgument);
|
|
1814
|
+
if (i.severity === 0)
|
|
1815
|
+
throw new ConnectError("issues.severity: required", Code.InvalidArgument);
|
|
1816
|
+
}
|
|
1817
|
+
for (const i of issues) {
|
|
1818
|
+
if (run.issues.length >= this.opts.issueCap) {
|
|
1819
|
+
run.issuesTruncated = true;
|
|
1820
|
+
continue;
|
|
1821
|
+
}
|
|
1822
|
+
run.issues.push({
|
|
1823
|
+
id: randomUUID(),
|
|
1824
|
+
runId: run.id,
|
|
1825
|
+
nodeId: a.nodeId,
|
|
1826
|
+
attemptId: a.id,
|
|
1827
|
+
stage: i.stage,
|
|
1828
|
+
severity: i.severity,
|
|
1829
|
+
subject: i.subject || i.subjectId,
|
|
1830
|
+
subjectKind: i.subjectKind,
|
|
1831
|
+
subjectId: i.subjectId,
|
|
1832
|
+
message: i.message,
|
|
1833
|
+
detail: i.detail,
|
|
1834
|
+
createdAt: Date.now(),
|
|
1835
|
+
});
|
|
1836
|
+
accepted++;
|
|
1837
|
+
}
|
|
1838
|
+
return accepted;
|
|
1839
|
+
}
|
|
1840
|
+
/** Issues of a run, filtered as ListIssues filters them. */
|
|
1841
|
+
issuesOf(runId, filter = {}) {
|
|
1842
|
+
return this.run(runId).issues.filter((i) => (!filter.nodeId || i.nodeId === filter.nodeId) && (!filter.attemptId || i.attemptId === filter.attemptId) && (!filter.minSeverity || i.severity >= filter.minSeverity));
|
|
1843
|
+
}
|
|
1844
|
+
/** The run's trace: the platform's spans for the run, its nodes and attempts, and the step spans reported. */
|
|
1845
|
+
traceOf(runId, filter = {}) {
|
|
1846
|
+
const run = this.run(runId);
|
|
1847
|
+
const traceId = run.id.replaceAll("-", "");
|
|
1848
|
+
const sid = (s) => s.replaceAll("-", "").slice(0, 16);
|
|
1849
|
+
const out = [
|
|
1850
|
+
create(TraceSpanSchema, {
|
|
1851
|
+
traceId,
|
|
1852
|
+
spanId: sid(run.id),
|
|
1853
|
+
name: `run ${run.id}`,
|
|
1854
|
+
source: TraceSpanSource.PLATFORM,
|
|
1855
|
+
startedAt: timestampFromMs(run.createdAt),
|
|
1856
|
+
...(run.finishedAt ? { endedAt: timestampFromMs(run.finishedAt) } : {}),
|
|
1857
|
+
}),
|
|
1858
|
+
];
|
|
1859
|
+
for (const n of run.nodes.values()) {
|
|
1860
|
+
if (filter.nodeId && n.id !== filter.nodeId)
|
|
1861
|
+
continue;
|
|
1862
|
+
const nodeSpan = sid(randomUUIDFrom(run.id + n.id));
|
|
1863
|
+
const first = n.attempts[0];
|
|
1864
|
+
out.push(create(TraceSpanSchema, {
|
|
1865
|
+
traceId,
|
|
1866
|
+
spanId: nodeSpan,
|
|
1867
|
+
parentSpanId: sid(run.id),
|
|
1868
|
+
name: `node ${n.id}`,
|
|
1869
|
+
source: TraceSpanSource.PLATFORM,
|
|
1870
|
+
nodeId: n.id,
|
|
1871
|
+
startedAt: timestampFromMs(first?.offeredAt ?? run.createdAt),
|
|
1872
|
+
}));
|
|
1873
|
+
for (const a of n.attempts) {
|
|
1874
|
+
if (filter.attemptId && a.id !== filter.attemptId)
|
|
1875
|
+
continue;
|
|
1876
|
+
out.push(create(TraceSpanSchema, {
|
|
1877
|
+
traceId,
|
|
1878
|
+
spanId: sid(a.id),
|
|
1879
|
+
parentSpanId: nodeSpan,
|
|
1880
|
+
name: `attempt ${a.number}`,
|
|
1881
|
+
source: TraceSpanSource.PLATFORM,
|
|
1882
|
+
nodeId: n.id,
|
|
1883
|
+
attemptId: a.id,
|
|
1884
|
+
startedAt: timestampFromMs(a.offeredAt),
|
|
1885
|
+
...(a.completedAt ? { endedAt: timestampFromMs(a.completedAt) } : {}),
|
|
1886
|
+
}));
|
|
1887
|
+
for (const sp of a.spans) {
|
|
1888
|
+
out.push(create(TraceSpanSchema, {
|
|
1889
|
+
traceId: sp.traceId,
|
|
1890
|
+
spanId: sp.spanId,
|
|
1891
|
+
parentSpanId: sp.parentSpanId || sid(a.id),
|
|
1892
|
+
name: sp.name,
|
|
1893
|
+
source: TraceSpanSource.STEP,
|
|
1894
|
+
nodeId: n.id,
|
|
1895
|
+
attemptId: a.id,
|
|
1896
|
+
...(sp.startedAt ? { startedAt: sp.startedAt } : {}),
|
|
1897
|
+
...(sp.endedAt ? { endedAt: sp.endedAt } : {}),
|
|
1898
|
+
status: sp.status,
|
|
1899
|
+
statusMessage: sp.statusMessage,
|
|
1900
|
+
...(sp.attributes ? { attributes: sp.attributes } : {}),
|
|
1901
|
+
}));
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
}
|
|
1905
|
+
return out.sort((x, y) => Number(x.startedAt?.seconds ?? 0n) - Number(y.startedAt?.seconds ?? 0n));
|
|
1906
|
+
}
|
|
1907
|
+
/** Runs whenever a run becomes terminal. */
|
|
1908
|
+
runFinished(run) {
|
|
1909
|
+
// Deferred, so a parent never settles inside the child's own advance.
|
|
1910
|
+
if (run.parentRunId)
|
|
1911
|
+
queueMicrotask(() => this.childFinished(run));
|
|
1912
|
+
// The schedules are made after the router; a run cannot finish before.
|
|
1913
|
+
this.schedules?.runFinished(run);
|
|
1914
|
+
}
|
|
846
1915
|
// ---- messages ---------------------------------------------------------
|
|
847
1916
|
runMessage(r) {
|
|
848
1917
|
return create(RunSchema, {
|
|
849
1918
|
id: r.id,
|
|
850
1919
|
tenant: r.tenant,
|
|
851
|
-
product: r.product,
|
|
852
1920
|
idempotencyKey: r.idempotencyKey,
|
|
1921
|
+
fairnessKey: r.fairnessKey,
|
|
853
1922
|
...(r.resource ? { resource: r.resource } : {}),
|
|
854
1923
|
variables: r.variables,
|
|
855
1924
|
state: r.state,
|
|
@@ -862,6 +1931,23 @@ export class FakePipelines {
|
|
|
862
1931
|
...(r.finishedAt ? { finishedAt: timestampFromMs(r.finishedAt) } : {}),
|
|
863
1932
|
...(r.cancelRequestedAt ? { cancelRequestedAt: timestampFromMs(r.cancelRequestedAt) } : {}),
|
|
864
1933
|
cancelReason: r.cancelReason,
|
|
1934
|
+
queueId: r.queueId,
|
|
1935
|
+
queue: r.queue,
|
|
1936
|
+
trigger: r.trigger,
|
|
1937
|
+
scheduleId: r.scheduleId,
|
|
1938
|
+
...(r.scheduledFor !== undefined ? { scheduledFor: timestampFromMs(r.scheduledFor) } : {}),
|
|
1939
|
+
scheduleTarget: r.scheduleTarget,
|
|
1940
|
+
issueInfoCount: r.issues.filter((i) => i.severity === IssueSeverity.INFO).length,
|
|
1941
|
+
issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
|
|
1942
|
+
issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
|
|
1943
|
+
issuesTruncated: r.issuesTruncated,
|
|
1944
|
+
...(r.attestedFacts ? { attestedFacts: r.attestedFacts } : {}),
|
|
1945
|
+
snapshotId: r.snapshotId,
|
|
1946
|
+
origin: r.origin,
|
|
1947
|
+
pipelineId: r.pipelineId,
|
|
1948
|
+
pipelineVersion: r.pipelineVersion,
|
|
1949
|
+
parentRunId: r.parentRunId,
|
|
1950
|
+
parentNodeId: r.parentNodeId,
|
|
865
1951
|
});
|
|
866
1952
|
}
|
|
867
1953
|
nodeMessage(n) {
|
|
@@ -872,6 +1958,18 @@ export class FakePipelines {
|
|
|
872
1958
|
failureClass: n.failureClass,
|
|
873
1959
|
settledAttemptId: terminalNode(n.state) ? (settled?.id ?? "") : "",
|
|
874
1960
|
kv: Object.fromEntries(Object.entries(n.kv).map(([k, v]) => [k, jsonValue(v)])),
|
|
1961
|
+
downstreamRunId: n.attempts.findLast((a) => a.childRunId)?.childRunId ?? "",
|
|
1962
|
+
...(n.elements
|
|
1963
|
+
? {
|
|
1964
|
+
forEach: {
|
|
1965
|
+
total: n.elements.length,
|
|
1966
|
+
succeeded: n.elements.filter((e) => e.state === "succeeded").length,
|
|
1967
|
+
failed: n.elements.filter((e) => e.state === "failed").length,
|
|
1968
|
+
running: n.elements.filter((e) => e.state === "running").length,
|
|
1969
|
+
pending: n.elements.filter((e) => e.state === "pending").length,
|
|
1970
|
+
},
|
|
1971
|
+
}
|
|
1972
|
+
: {}),
|
|
875
1973
|
attempts: n.attempts.map((a) => create(AttemptSchema, {
|
|
876
1974
|
id: a.id,
|
|
877
1975
|
runId: a.runId,
|
|
@@ -884,16 +1982,54 @@ export class FakePipelines {
|
|
|
884
1982
|
termination: a.termination,
|
|
885
1983
|
...(a.exitCode !== undefined ? { exitCode: a.exitCode } : {}),
|
|
886
1984
|
message: a.message,
|
|
1985
|
+
childRunId: a.childRunId,
|
|
1986
|
+
...(a.itemIndex !== undefined ? { itemIndex: a.itemIndex, item: jsonValue(a.item ?? null) } : {}),
|
|
887
1987
|
logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
|
|
888
1988
|
logTruncated: a.logTruncated,
|
|
1989
|
+
...(a.progress
|
|
1990
|
+
? {
|
|
1991
|
+
progress: {
|
|
1992
|
+
fraction: a.progress.fraction,
|
|
1993
|
+
message: a.progress.message,
|
|
1994
|
+
processed: a.progress.processed,
|
|
1995
|
+
total: a.progress.total,
|
|
1996
|
+
failed: a.progress.failed,
|
|
1997
|
+
updatedAt: timestampFromMs(a.progress.updatedAt),
|
|
1998
|
+
stages: a.progress.stages.map((st) => ({
|
|
1999
|
+
key: st.key,
|
|
2000
|
+
name: st.name,
|
|
2001
|
+
weight: st.weight,
|
|
2002
|
+
state: st.state,
|
|
2003
|
+
fraction: st.fraction,
|
|
2004
|
+
message: st.message,
|
|
2005
|
+
processed: st.processed,
|
|
2006
|
+
total: st.total,
|
|
2007
|
+
failed: st.failed,
|
|
2008
|
+
...(st.startedAt ? { startedAt: timestampFromMs(st.startedAt) } : {}),
|
|
2009
|
+
...(st.finishedAt ? { finishedAt: timestampFromMs(st.finishedAt) } : {}),
|
|
2010
|
+
})),
|
|
2011
|
+
},
|
|
2012
|
+
}
|
|
2013
|
+
: {}),
|
|
889
2014
|
})),
|
|
890
2015
|
});
|
|
891
2016
|
}
|
|
892
2017
|
}
|
|
2018
|
+
/** Every whole-value reference in a value, also inside lists and objects. */
|
|
2019
|
+
function refs(v) {
|
|
2020
|
+
if (Array.isArray(v))
|
|
2021
|
+
return v.flatMap(refs);
|
|
2022
|
+
if (v !== null && typeof v === "object")
|
|
2023
|
+
return Object.values(v).flatMap(refs);
|
|
2024
|
+
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
2025
|
+
return m ? [m[1]] : [];
|
|
2026
|
+
}
|
|
2027
|
+
const ITEM_REF = /^item(_index$|$|\.)/;
|
|
893
2028
|
/**
|
|
894
2029
|
* Holds a definition to the server's rules the fake models: unique ids,
|
|
895
|
-
* known needs, no cycle,
|
|
896
|
-
* node's dependency closure (a reference adds no edge)
|
|
2030
|
+
* known needs, no cycle, a reference names only a node in the referring
|
|
2031
|
+
* node's dependency closure (a reference adds no edge), and `item` is read
|
|
2032
|
+
* only in a for_each node's `with`.
|
|
897
2033
|
*/
|
|
898
2034
|
function checkGraph(specs) {
|
|
899
2035
|
const byId = new Map();
|
|
@@ -917,18 +2053,22 @@ function checkGraph(specs) {
|
|
|
917
2053
|
};
|
|
918
2054
|
for (const s of specs) {
|
|
919
2055
|
const reachable = closure(s.id);
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
const node =
|
|
923
|
-
if (node && !reachable.has(node))
|
|
924
|
-
throw new ConnectError(`node ${s.id}: ${
|
|
925
|
-
}
|
|
2056
|
+
const items = s.forEach ? refs(s.forEach.items) : [];
|
|
2057
|
+
for (const r of [...refs(s.with), ...items]) {
|
|
2058
|
+
const node = r.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
|
|
2059
|
+
if (node && !reachable.has(node))
|
|
2060
|
+
throw new ConnectError(`node ${s.id}: ${r} names a node outside its needs`, Code.InvalidArgument);
|
|
926
2061
|
}
|
|
2062
|
+
if (!s.forEach && refs(s.with).some((r) => ITEM_REF.test(r)))
|
|
2063
|
+
throw new ConnectError(`node ${s.id}: item is read only in a for_each node`, Code.InvalidArgument);
|
|
2064
|
+
if (items.some((r) => ITEM_REF.test(r)))
|
|
2065
|
+
throw new ConnectError(`node ${s.id}: for_each.items cannot read item`, Code.InvalidArgument);
|
|
927
2066
|
}
|
|
928
2067
|
}
|
|
929
2068
|
function nodeSpec(n, requireHandler) {
|
|
930
2069
|
let handler = "";
|
|
931
2070
|
let uses = "";
|
|
2071
|
+
let pipeline;
|
|
932
2072
|
if (n.action.case === "handler") {
|
|
933
2073
|
handler = n.action.value;
|
|
934
2074
|
requireHandler(handler);
|
|
@@ -936,8 +2076,31 @@ function nodeSpec(n, requireHandler) {
|
|
|
936
2076
|
else if (n.action.case === "uses") {
|
|
937
2077
|
uses = `${n.action.value.name}@${n.action.value.version}`;
|
|
938
2078
|
}
|
|
2079
|
+
else if (n.action.case === "pipeline") {
|
|
2080
|
+
// Pinned by freeze() before a node is compiled.
|
|
2081
|
+
const p = n.action.value;
|
|
2082
|
+
pipeline = { id: p.pipelineId, version: p.version, outputs: p.outputs.map((o) => ({ name: o.name, node: o.node, slot: o.slot })) };
|
|
2083
|
+
}
|
|
939
2084
|
else {
|
|
940
|
-
throw new ConnectError(`node ${n.id}: only handler and
|
|
2085
|
+
throw new ConnectError(`node ${n.id}: only handler, uses and pipeline nodes`, Code.Unimplemented);
|
|
2086
|
+
}
|
|
2087
|
+
let forEach;
|
|
2088
|
+
if (n.forEach) {
|
|
2089
|
+
if (n.manual)
|
|
2090
|
+
throw new ConnectError(`node ${n.id}: for_each is not for a manual node`, Code.InvalidArgument);
|
|
2091
|
+
if (uses)
|
|
2092
|
+
throw new ConnectError(`node ${n.id}: the fake runs for_each on handler and pipeline nodes only`, Code.Unimplemented);
|
|
2093
|
+
const fe = n.forEach;
|
|
2094
|
+
if (fe.maxItems > 10_000)
|
|
2095
|
+
throw new ConnectError(`node ${n.id}: for_each.max_items is at most 10000`, Code.InvalidArgument);
|
|
2096
|
+
const items = fe.items ? toJson(ValueSchema, fe.items) : null;
|
|
2097
|
+
if (typeof items === "string" ? !REF.test(items) : !Array.isArray(items)) {
|
|
2098
|
+
throw new ConnectError(`node ${n.id}: for_each.items is a list or a whole-value reference to one`, Code.InvalidArgument);
|
|
2099
|
+
}
|
|
2100
|
+
if (Array.isArray(items) && items.length > (fe.maxItems || 1000)) {
|
|
2101
|
+
throw new ConnectError(`node ${n.id}: for_each.items has ${items.length} elements, over max_items ${fe.maxItems || 1000}`, Code.InvalidArgument);
|
|
2102
|
+
}
|
|
2103
|
+
forEach = { items, maxItems: fe.maxItems, maxParallel: fe.maxParallel, onFailure: fe.onFailure };
|
|
941
2104
|
}
|
|
942
2105
|
return {
|
|
943
2106
|
id: n.id,
|
|
@@ -948,6 +2111,8 @@ function nodeSpec(n, requireHandler) {
|
|
|
948
2111
|
retry: n.retry ? { max: n.retry.max, when: n.retry.when } : undefined,
|
|
949
2112
|
timeoutMs: n.timeout ? durationMs(n.timeout) : 0,
|
|
950
2113
|
interruptible: n.interruptible,
|
|
2114
|
+
...(pipeline ? { pipeline } : {}),
|
|
2115
|
+
...(forEach ? { forEach } : {}),
|
|
951
2116
|
};
|
|
952
2117
|
}
|
|
953
2118
|
/** The server's dispatch.Outcome. */
|
|
@@ -1012,4 +2177,40 @@ function aborted(s) {
|
|
|
1012
2177
|
s.addEventListener("abort", () => r(), { once: true });
|
|
1013
2178
|
});
|
|
1014
2179
|
}
|
|
2180
|
+
/** A queue's limits as the selection reads them. */
|
|
2181
|
+
function queueLimits(q) {
|
|
2182
|
+
return {
|
|
2183
|
+
id: q.id,
|
|
2184
|
+
priority: q.priority || 1,
|
|
2185
|
+
paused: q.paused,
|
|
2186
|
+
maxConcurrency: q.maxConcurrency,
|
|
2187
|
+
maxConcurrencyPerTenant: q.maxConcurrencyPerTenant,
|
|
2188
|
+
...(q.rateLimit?.window ? { rateLimit: { limit: q.rateLimit.limit, windowMs: durationMs(q.rateLimit.window) } } : {}),
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
/**
|
|
2192
|
+
* A job's retry and timeout under its queue: the queue's defaults when the
|
|
2193
|
+
* job gives none, and its ceilings, past which the job is refused.
|
|
2194
|
+
*/
|
|
2195
|
+
function jobLimits(q, retry, timeoutMs) {
|
|
2196
|
+
const ceiling = q.defaultRetry;
|
|
2197
|
+
if (retry && ceiling) {
|
|
2198
|
+
if (retry.max > ceiling.max)
|
|
2199
|
+
throw new ConnectError(`retry.max ${retry.max} is above queue ${q.name}'s ${ceiling.max}`, Code.InvalidArgument);
|
|
2200
|
+
const allowed = ceiling.when.length > 0 ? ceiling.when : TRANSIENT;
|
|
2201
|
+
const outside = retry.when.filter((c) => !allowed.includes(c));
|
|
2202
|
+
if (outside.length > 0)
|
|
2203
|
+
throw new ConnectError(`retry.when lists ${outside.map((c) => FailureClass[c]).join(", ")}, which queue ${q.name} does not retry`, Code.InvalidArgument);
|
|
2204
|
+
}
|
|
2205
|
+
const maxTimeout = q.defaultTimeout ? durationMs(q.defaultTimeout) : 0;
|
|
2206
|
+
if (timeoutMs > 0 && maxTimeout > 0 && timeoutMs > maxTimeout) {
|
|
2207
|
+
throw new ConnectError(`timeout ${timeoutMs} ms is above queue ${q.name}'s ${maxTimeout} ms`, Code.InvalidArgument);
|
|
2208
|
+
}
|
|
2209
|
+
const r = retry ?? ceiling;
|
|
2210
|
+
return { retry: r ? { max: r.max, when: r.when } : undefined, timeoutMs: timeoutMs || maxTimeout };
|
|
2211
|
+
}
|
|
2212
|
+
/** A stable UUID-shaped id from a string, for the fake's platform spans. */
|
|
2213
|
+
function randomUUIDFrom(s) {
|
|
2214
|
+
return createHash("sha256").update(s).digest("hex").slice(0, 32);
|
|
2215
|
+
}
|
|
1015
2216
|
//# sourceMappingURL=index.js.map
|