@xeonr/pipelines-sdk 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +247 -24
- package/dist/client.d.ts +73 -3
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +48 -1
- package/dist/client.js.map +1 -1
- package/dist/errors.d.ts +42 -6
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +45 -2
- package/dist/errors.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +173 -12
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +45 -28
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +136 -17
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +42 -35
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +346 -15
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +105 -10
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +38 -6
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.js +37 -5
- package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +280 -5
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js +100 -7
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +1100 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +298 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +120 -1
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js +36 -11
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +769 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js +190 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +60 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js +21 -16
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +867 -4
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js +243 -27
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +1110 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +295 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +8 -3
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +1 -1
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts +123 -1
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/platform/v1/options_pb.js +90 -1
- package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
- package/dist/handler.d.ts +50 -0
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +27 -0
- package/dist/handler.js.map +1 -1
- package/dist/index.d.ts +13 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/logship.d.ts +60 -10
- package/dist/logship.d.ts.map +1 -1
- package/dist/logship.js +171 -28
- package/dist/logship.js.map +1 -1
- package/dist/queues.d.ts +171 -0
- package/dist/queues.d.ts.map +1 -0
- package/dist/queues.js +135 -0
- package/dist/queues.js.map +1 -0
- package/dist/reporter.d.ts +120 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporter.js +323 -0
- package/dist/reporter.js.map +1 -0
- package/dist/schedules.d.ts +201 -0
- package/dist/schedules.d.ts.map +1 -0
- package/dist/schedules.js +127 -0
- package/dist/schedules.js.map +1 -0
- package/dist/testing/cron.d.ts +33 -0
- package/dist/testing/cron.d.ts.map +1 -0
- package/dist/testing/cron.js +177 -0
- package/dist/testing/cron.js.map +1 -0
- package/dist/testing/index.d.ts +142 -4
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +843 -44
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/queues.d.ts +46 -0
- package/dist/testing/queues.d.ts.map +1 -0
- package/dist/testing/queues.js +70 -0
- package/dist/testing/queues.js.map +1 -0
- package/dist/testing/schedules.d.ts +113 -0
- package/dist/testing/schedules.d.ts.map +1 -0
- package/dist/testing/schedules.js +419 -0
- package/dist/testing/schedules.js.map +1 -0
- package/dist/worker.d.ts +78 -8
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +163 -33
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
package/dist/testing/index.js
CHANGED
|
@@ -19,17 +19,22 @@
|
|
|
19
19
|
*
|
|
20
20
|
* What it models: auth by bearer token (accounts with a tenant and scopes),
|
|
21
21
|
* namespace and pool ownership, `PublishVersion`, `SetPool`, `CreateJob`
|
|
22
|
-
* (with delay and
|
|
23
|
-
* (image) nodes that a test settles with
|
|
24
|
-
* `${{ steps.<node>.outputs.kv.<slot> }}`
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
22
|
+
* (with delay, idempotency, retry and timeout), `CreateRun` with handler
|
|
23
|
+
* nodes and external (image) nodes that a test settles with
|
|
24
|
+
* {@link FakePipelines.completeNode}, `${{ steps.<node>.outputs.kv.<slot> }}`
|
|
25
|
+
* references (only to a node in the referrer's needs closure), the claim
|
|
26
|
+
* long-poll, the worker id on every ref, leases and their lapse,
|
|
27
|
+
* `ReleaseAttempt`, heartbeats with cancel, logs (idempotent on seq,
|
|
28
|
+
* `out_of_range` naming the expected seq for a gap, and
|
|
29
|
+
* {@link FakePipelines.loseLogChunks} to play a server that lost chunks),
|
|
30
|
+
* completion with the server's termination to failure-class mapping,
|
|
31
|
+
* `handler_failure_kind`, retries by class, free interruptions on an
|
|
32
|
+
* `interruptible` node, cancel, and error injection.
|
|
28
33
|
*
|
|
29
34
|
* What it does not: secrets, blobs, tiers, routing between pools, the
|
|
30
35
|
* outbox, and the platform's own deadline enforcement.
|
|
31
36
|
*/
|
|
32
|
-
import { randomUUID } from "node:crypto";
|
|
37
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
33
38
|
import { create, fromJson, toJson } from "@bufbuild/protobuf";
|
|
34
39
|
import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
35
40
|
import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
|
|
@@ -38,8 +43,15 @@ import { authInterceptor } from "../transport.js";
|
|
|
38
43
|
import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
|
|
39
44
|
import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
|
|
40
45
|
import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
|
|
41
|
-
import { AttemptSchema, NodeRunSchema, RunSchema, RunsService } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
42
|
-
import {
|
|
46
|
+
import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
47
|
+
import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
|
|
48
|
+
import { pickQueued } from "./queues.js";
|
|
49
|
+
import { FakeClock, FakeSchedules } from "./schedules.js";
|
|
50
|
+
import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
|
|
51
|
+
export { FakeClock, FakeSchedules } from "./schedules.js";
|
|
52
|
+
export { parseCron, nextOccurrence } from "./cron.js";
|
|
53
|
+
export { pickQueued } from "./queues.js";
|
|
54
|
+
import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
|
|
43
55
|
/** The tenant the default account acts for. */
|
|
44
56
|
export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
|
|
45
57
|
/** The default account's token. */
|
|
@@ -53,15 +65,24 @@ export const ALL_SCOPES = [
|
|
|
53
65
|
"pipelines:pool:read",
|
|
54
66
|
"pipelines:pool:edit",
|
|
55
67
|
"pipelines:worker:edit",
|
|
68
|
+
"pipelines:queue:read",
|
|
69
|
+
"pipelines:queue:edit",
|
|
70
|
+
"pipelines:schedule:read",
|
|
71
|
+
"pipelines:schedule:edit",
|
|
56
72
|
];
|
|
73
|
+
/** The product the default account belongs to. */
|
|
74
|
+
export const FAKE_PRODUCT = "00000000-0000-4000-8000-0000000000a1";
|
|
57
75
|
const TRANSIENT = [
|
|
58
76
|
FailureClass.ABANDONED,
|
|
59
77
|
FailureClass.RUNNER_CRASHED,
|
|
60
78
|
FailureClass.IMAGE_PULL,
|
|
61
79
|
FailureClass.STAGING_FAILED,
|
|
80
|
+
FailureClass.INTERRUPTED,
|
|
62
81
|
FailureClass.INTERNAL,
|
|
63
82
|
];
|
|
64
83
|
const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
84
|
+
/** How many interruptions an interruptible node gets without spending a retry (the api's default). */
|
|
85
|
+
const FREE_INTERRUPTIONS = 3;
|
|
65
86
|
const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
|
|
66
87
|
/**
|
|
67
88
|
* An in-memory pipelines-api for tests. Call {@link close} when done (it
|
|
@@ -80,6 +101,16 @@ export class FakePipelines {
|
|
|
80
101
|
attemptsById = new Map();
|
|
81
102
|
idempotency = new Map();
|
|
82
103
|
injected = new Map();
|
|
104
|
+
/** Queues by id. */
|
|
105
|
+
queues = new Map();
|
|
106
|
+
/**
|
|
107
|
+
* The clock schedules run on: real time until a test calls
|
|
108
|
+
* `clock.set()` or `clock.advance()`, each of which fires what is due.
|
|
109
|
+
*/
|
|
110
|
+
clock = new FakeClock();
|
|
111
|
+
/** The schedules, with their recorded occurrences. */
|
|
112
|
+
schedules;
|
|
113
|
+
queueClaims = new Map();
|
|
83
114
|
timer;
|
|
84
115
|
changed;
|
|
85
116
|
constructor(opts = {}) {
|
|
@@ -94,8 +125,29 @@ export class FakePipelines {
|
|
|
94
125
|
retryBackoffMs: opts.retryBackoffMs ?? 0,
|
|
95
126
|
logMaxBytes: opts.logMaxBytes ?? 0,
|
|
96
127
|
tickMs: opts.tickMs ?? 10,
|
|
128
|
+
issueCap: opts.issueCap ?? 1000,
|
|
129
|
+
spanCap: opts.spanCap ?? 10_000,
|
|
97
130
|
};
|
|
98
131
|
this.changed = signal();
|
|
132
|
+
this.schedules = new FakeSchedules({
|
|
133
|
+
createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES, product: owner.product }, {
|
|
134
|
+
tenant: owner.tenant,
|
|
135
|
+
handler: spec.handler,
|
|
136
|
+
key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
|
|
137
|
+
payload: spec.payload,
|
|
138
|
+
delayMs: 0,
|
|
139
|
+
resource: spec.resource,
|
|
140
|
+
retry: spec.retry,
|
|
141
|
+
timeoutMs: spec.timeoutMs,
|
|
142
|
+
queue: spec.queue,
|
|
143
|
+
interruptible: spec.interruptible,
|
|
144
|
+
trigger: spec.trigger,
|
|
145
|
+
schedule: spec.schedule,
|
|
146
|
+
}),
|
|
147
|
+
runs: () => this.runs.values(),
|
|
148
|
+
runMessage: (id) => this.runMessage(this.run(id)),
|
|
149
|
+
checkHandler: (_product, handler) => this.requireHandler(handler),
|
|
150
|
+
}, this.clock, FAKE_PRODUCT);
|
|
99
151
|
this.timer = setInterval(() => this.tick(), this.opts.tickMs);
|
|
100
152
|
this.timer.unref();
|
|
101
153
|
}
|
|
@@ -180,6 +232,15 @@ export class FakePipelines {
|
|
|
180
232
|
}
|
|
181
233
|
}
|
|
182
234
|
/** Lapses an attempt's lease now, as the reaper would at its expiry. */
|
|
235
|
+
/**
|
|
236
|
+
* Forgets an attempt's log chunks from `fromSeq` on, as a server that lost
|
|
237
|
+
* chunks it had acknowledged would: the worker's next chunk is refused as
|
|
238
|
+
* ahead (`out_of_range` naming `fromSeq`).
|
|
239
|
+
*/
|
|
240
|
+
loseLogChunks(attemptId, fromSeq) {
|
|
241
|
+
const a = this.attempt(attemptId);
|
|
242
|
+
a.chunks.length = Math.min(a.chunks.length, fromSeq);
|
|
243
|
+
}
|
|
183
244
|
expireLease(attemptId) {
|
|
184
245
|
this.attempt(attemptId).leaseExpiresAt = 0;
|
|
185
246
|
this.tick();
|
|
@@ -279,7 +340,7 @@ export class FakePipelines {
|
|
|
279
340
|
throw stale();
|
|
280
341
|
const next = BigInt(a.chunks.length);
|
|
281
342
|
if (req.seq > next)
|
|
282
|
-
throw new ConnectError(`the next expected seq is ${next}`, Code.OutOfRange);
|
|
343
|
+
throw new ConnectError(`seq: the next expected seq is ${next}`, Code.OutOfRange);
|
|
283
344
|
if (req.data.length < 1 || req.data.length > 1 << 20)
|
|
284
345
|
throw new ConnectError("data: 1 byte to 1 MiB", Code.InvalidArgument);
|
|
285
346
|
if (req.seq === next && !a.logTruncated) {
|
|
@@ -302,6 +363,9 @@ export class FakePipelines {
|
|
|
302
363
|
throw new ConnectError("termination: required", Code.InvalidArgument);
|
|
303
364
|
if (req.message.length > 2048)
|
|
304
365
|
throw new ConnectError("message: at most 2048", Code.InvalidArgument);
|
|
366
|
+
if (req.handlerFailureKind !== HandlerFailureKind.UNSPECIFIED && req.termination !== Termination.HANDLER_ERROR) {
|
|
367
|
+
throw new ConnectError("handler_failure_kind is only for termination TERMINATION_HANDLER_ERROR", Code.InvalidArgument);
|
|
368
|
+
}
|
|
305
369
|
const kv = {};
|
|
306
370
|
const entries = Object.entries(req.kv);
|
|
307
371
|
if (entries.length > 100)
|
|
@@ -317,29 +381,141 @@ export class FakePipelines {
|
|
|
317
381
|
a.termination = req.termination;
|
|
318
382
|
a.exitCode = req.exitCode;
|
|
319
383
|
a.message = req.message;
|
|
384
|
+
a.handlerFailureKind = req.handlerFailureKind;
|
|
320
385
|
a.kv = state === AttemptState.SUCCEEDED ? kv : {};
|
|
321
386
|
a.completedAt = Date.now();
|
|
322
387
|
this.settle(a);
|
|
323
388
|
return { state: a.state };
|
|
324
389
|
},
|
|
390
|
+
releaseAttempt: (req, ctx) => {
|
|
391
|
+
const who = this.enter("ReleaseAttempt", ctx, "pipelines:worker:edit");
|
|
392
|
+
if (req.reason.length > 256)
|
|
393
|
+
throw new ConnectError("reason: at most 256", Code.InvalidArgument);
|
|
394
|
+
const a = this.held(who, req.ref);
|
|
395
|
+
if (a.state === AttemptState.RUNNING)
|
|
396
|
+
return { state: a.state };
|
|
397
|
+
if (a.state !== AttemptState.CLAIMED)
|
|
398
|
+
throw stale();
|
|
399
|
+
// CLAIMED -> OFFERED at once: the epoch is bumped, nothing spent.
|
|
400
|
+
a.epoch += 1n;
|
|
401
|
+
a.workerId = "";
|
|
402
|
+
a.poolId = "";
|
|
403
|
+
a.leaseExpiresAt = 0;
|
|
404
|
+
a.releases++;
|
|
405
|
+
if (this.run(a.runId).cancelRequestedAt !== undefined) {
|
|
406
|
+
a.state = AttemptState.CANCELLED;
|
|
407
|
+
a.failureClass = FailureClass.CANCELLED;
|
|
408
|
+
a.completedAt = Date.now();
|
|
409
|
+
this.settle(a);
|
|
410
|
+
}
|
|
411
|
+
else {
|
|
412
|
+
a.state = AttemptState.OFFERED;
|
|
413
|
+
a.notBefore = 0;
|
|
414
|
+
}
|
|
415
|
+
this.notify();
|
|
416
|
+
return { state: a.state };
|
|
417
|
+
},
|
|
418
|
+
appendIssues: (req, ctx) => {
|
|
419
|
+
const who = this.enter("AppendIssues", ctx, "pipelines:worker:edit");
|
|
420
|
+
const a = this.held(who, req.ref);
|
|
421
|
+
if (a.state !== AttemptState.RUNNING)
|
|
422
|
+
throw stale();
|
|
423
|
+
if (req.issues.length < 1 || req.issues.length > 100)
|
|
424
|
+
throw new ConnectError("issues: 1 to 100", Code.InvalidArgument);
|
|
425
|
+
return { accepted: this.appendIssues(a, req.issues) };
|
|
426
|
+
},
|
|
427
|
+
setProgress: (req, ctx) => {
|
|
428
|
+
const who = this.enter("SetProgress", ctx, "pipelines:worker:edit");
|
|
429
|
+
const a = this.held(who, req.ref);
|
|
430
|
+
if (a.state !== AttemptState.RUNNING)
|
|
431
|
+
throw stale();
|
|
432
|
+
if (!(req.fraction >= 0 && req.fraction <= 1))
|
|
433
|
+
throw new ConnectError("fraction: 0 to 1", Code.InvalidArgument);
|
|
434
|
+
if (req.stageState === StageState.PENDING)
|
|
435
|
+
throw new ConnectError("stage_state: a stage never moves back to PENDING", Code.InvalidArgument);
|
|
436
|
+
if (req.stage && !/^[a-z][a-z0-9_.-]{0,63}$/.test(req.stage))
|
|
437
|
+
throw new ConnectError("stage: invalid key", Code.InvalidArgument);
|
|
438
|
+
if (req.message.length > 255)
|
|
439
|
+
throw new ConnectError("message: at most 255", Code.InvalidArgument);
|
|
440
|
+
this.setProgress(a, req);
|
|
441
|
+
return {};
|
|
442
|
+
},
|
|
443
|
+
reportSpans: (req, ctx) => {
|
|
444
|
+
const who = this.enter("ReportSpans", ctx, "pipelines:worker:edit");
|
|
445
|
+
const a = this.held(who, req.ref);
|
|
446
|
+
if (a.state !== AttemptState.RUNNING)
|
|
447
|
+
throw stale();
|
|
448
|
+
if (req.spans.length < 1 || req.spans.length > 1000)
|
|
449
|
+
throw new ConnectError("spans: 1 to 1000", Code.InvalidArgument);
|
|
450
|
+
for (const sp of req.spans) {
|
|
451
|
+
if (!/^[0-9a-f]{32}$/.test(sp.traceId) || !/^[0-9a-f]{16}$/.test(sp.spanId) || !sp.startedAt || !sp.endedAt || !sp.name) {
|
|
452
|
+
throw new ConnectError("spans: trace_id, span_id, name, started_at and ended_at are required", Code.InvalidArgument);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
const room = Math.max(0, this.opts.spanCap - a.spans.length);
|
|
456
|
+
a.spans.push(...req.spans.slice(0, room));
|
|
457
|
+
return { accepted: Math.min(room, req.spans.length) };
|
|
458
|
+
},
|
|
325
459
|
});
|
|
460
|
+
router.service(QueuesService, this.queuesService());
|
|
461
|
+
router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
|
|
326
462
|
router.service(RunsService, {
|
|
463
|
+
listIssues: (req, ctx) => {
|
|
464
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
465
|
+
this.visible(who, req.runId);
|
|
466
|
+
const issues = this.issuesOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId, minSeverity: req.minSeverity });
|
|
467
|
+
return {
|
|
468
|
+
issues: issues.map((i) => create(RunIssueSchema, { ...i, detail: i.detail, createdAt: timestampFromMs(i.createdAt) })),
|
|
469
|
+
nextCursor: "",
|
|
470
|
+
};
|
|
471
|
+
},
|
|
472
|
+
getLogs: (req, ctx) => {
|
|
473
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
474
|
+
const a = this.attemptsById.get(req.attemptId);
|
|
475
|
+
if (!a)
|
|
476
|
+
throw new ConnectError("no such attempt", Code.NotFound);
|
|
477
|
+
this.visible(who, a.runId);
|
|
478
|
+
const all = Buffer.concat(a.chunks);
|
|
479
|
+
const from = Math.min(Number(req.offset), all.length);
|
|
480
|
+
const to = req.limitBytes ? Math.min(all.length, from + req.limitBytes) : all.length;
|
|
481
|
+
const complete = to === all.length && (a.logFinal || a.completedAt !== undefined);
|
|
482
|
+
return { data: new Uint8Array(all.subarray(from, to)), nextOffset: BigInt(to), complete, truncated: a.logTruncated };
|
|
483
|
+
},
|
|
484
|
+
getRunTrace: (req, ctx) => {
|
|
485
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
486
|
+
this.visible(who, req.runId);
|
|
487
|
+
return { spans: this.traceOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId }), truncated: false };
|
|
488
|
+
},
|
|
327
489
|
createJob: (req, ctx) => {
|
|
328
490
|
const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
|
|
329
491
|
this.actsFor(who, req.tenant);
|
|
330
|
-
const existing = this.idempotency.get(`${who
|
|
492
|
+
const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
|
|
331
493
|
if (existing)
|
|
332
494
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
333
|
-
this.
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
495
|
+
const run = this.createJobRun(who, {
|
|
496
|
+
tenant: req.tenant,
|
|
497
|
+
handler: req.handler,
|
|
498
|
+
key: req.idempotencyKey,
|
|
499
|
+
payload: req.payload ?? {},
|
|
500
|
+
delayMs: req.delay ? durationMs(req.delay) : 0,
|
|
501
|
+
resource: req.resource,
|
|
502
|
+
retry: req.retry,
|
|
503
|
+
timeoutMs: req.timeout ? durationMs(req.timeout) : 0,
|
|
504
|
+
queue: req.queue,
|
|
505
|
+
interruptible: req.interruptible,
|
|
506
|
+
});
|
|
337
507
|
return { run: this.runMessage(run), deduplicated: false };
|
|
338
508
|
},
|
|
509
|
+
retryNode: (req, ctx) => {
|
|
510
|
+
const who = this.enter("RetryNode", ctx, "pipelines:run:edit");
|
|
511
|
+
const run = this.visible(who, req.runId);
|
|
512
|
+
this.retryNode(run, req.nodeId);
|
|
513
|
+
return { run: this.runMessage(run) };
|
|
514
|
+
},
|
|
339
515
|
createRun: (req, ctx) => {
|
|
340
516
|
const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
|
|
341
517
|
this.actsFor(who, req.tenant);
|
|
342
|
-
const existing = this.idempotency.get(`${who
|
|
518
|
+
const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
|
|
343
519
|
if (existing)
|
|
344
520
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
345
521
|
if (req.source.case !== "inlineDefinition")
|
|
@@ -347,14 +523,14 @@ export class FakePipelines {
|
|
|
347
523
|
const def = req.source.value;
|
|
348
524
|
if (def.nodes.length === 0)
|
|
349
525
|
throw new ConnectError("definition: no nodes", Code.InvalidArgument);
|
|
526
|
+
const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
|
|
527
|
+
for (const sp of specs)
|
|
528
|
+
if (sp.handler)
|
|
529
|
+
sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
|
|
530
|
+
checkGraph(specs);
|
|
350
531
|
const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, req.variables ?? {}, 0);
|
|
351
|
-
for (const
|
|
352
|
-
this.addNode(run,
|
|
353
|
-
for (const n of run.nodes.values()) {
|
|
354
|
-
for (const need of n.needs)
|
|
355
|
-
if (!run.nodes.has(need))
|
|
356
|
-
throw new ConnectError(`node ${n.id}: unknown need ${need}`, Code.InvalidArgument);
|
|
357
|
-
}
|
|
532
|
+
for (const spec of specs)
|
|
533
|
+
this.addNode(run, spec);
|
|
358
534
|
this.advance(run);
|
|
359
535
|
return { run: this.runMessage(run), deduplicated: false };
|
|
360
536
|
},
|
|
@@ -465,6 +641,9 @@ export class FakePipelines {
|
|
|
465
641
|
const a = this.attemptsById.get(ref.attemptId);
|
|
466
642
|
if (!a || a.epoch !== ref.claimEpoch || !a.poolId)
|
|
467
643
|
throw stale();
|
|
644
|
+
// Claimed by handler: the ref must name the worker recorded at the claim.
|
|
645
|
+
if (!ref.workerId || ref.workerId !== a.workerId)
|
|
646
|
+
throw stale();
|
|
468
647
|
if (this.pools.get(a.poolId)?.owner?.tenant !== who.tenant)
|
|
469
648
|
throw stale();
|
|
470
649
|
return a;
|
|
@@ -475,13 +654,39 @@ export class FakePipelines {
|
|
|
475
654
|
throw new ConnectError(`handler ${handler} is not a published worker package`, Code.InvalidArgument);
|
|
476
655
|
}
|
|
477
656
|
}
|
|
657
|
+
/**
|
|
658
|
+
* Holds a handler node's `with` to its manifest, as the api does when a run
|
|
659
|
+
* is created: every key must be a declared input, every required input
|
|
660
|
+
* given; defaults fill the rest. Returns the inputs with defaults.
|
|
661
|
+
*/
|
|
662
|
+
checkInputs(nodeId, handler, given) {
|
|
663
|
+
const pkg = this.packages.get(handler);
|
|
664
|
+
const version = pkg ? [...pkg.versions.keys()].at(-1) : "";
|
|
665
|
+
const manifest = pkg?.versions.get(version);
|
|
666
|
+
const inputs = manifest?.inputs ?? [];
|
|
667
|
+
for (const k of Object.keys(given)) {
|
|
668
|
+
if (!inputs.some((i) => i.name === k)) {
|
|
669
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": input "${k}" is not an input of ${handler}@${version}`, Code.InvalidArgument);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
const out = { ...given };
|
|
673
|
+
for (const i of inputs) {
|
|
674
|
+
if (out[i.name] !== undefined)
|
|
675
|
+
continue;
|
|
676
|
+
if (i.default)
|
|
677
|
+
out[i.name] = toJson(ValueSchema, i.default);
|
|
678
|
+
else if (i.required)
|
|
679
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": required input "${i.name}" is not given`, Code.InvalidArgument);
|
|
680
|
+
}
|
|
681
|
+
return out;
|
|
682
|
+
}
|
|
478
683
|
// ---- the scheduler ----------------------------------------------------
|
|
479
684
|
newRun(who, tenant, key, resource, variables, delayMs) {
|
|
480
685
|
const now = Date.now();
|
|
481
686
|
const run = {
|
|
482
687
|
id: randomUUID(),
|
|
483
688
|
tenant,
|
|
484
|
-
product: who
|
|
689
|
+
product: productOf(who),
|
|
485
690
|
idempotencyKey: key,
|
|
486
691
|
createdBy: `urn:fake:sa:${who.tenant}`,
|
|
487
692
|
resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
|
|
@@ -496,19 +701,20 @@ export class FakePipelines {
|
|
|
496
701
|
cancelRequestedAt: undefined,
|
|
497
702
|
cancelReason: "",
|
|
498
703
|
nodes: new Map(),
|
|
704
|
+
queueId: "",
|
|
705
|
+
queue: "",
|
|
706
|
+
trigger: RunTrigger.API,
|
|
707
|
+
scheduleId: "",
|
|
708
|
+
scheduledFor: undefined,
|
|
709
|
+
scheduleTarget: "",
|
|
710
|
+
issues: [],
|
|
711
|
+
issuesTruncated: false,
|
|
499
712
|
};
|
|
500
713
|
this.runs.set(run.id, run);
|
|
501
714
|
this.idempotency.set(`${run.product}\u0000${key}`, run.id);
|
|
502
715
|
return run;
|
|
503
716
|
}
|
|
504
717
|
addNode(run, spec) {
|
|
505
|
-
const refs = new Set(spec.needs);
|
|
506
|
-
for (const v of Object.values(spec.with)) {
|
|
507
|
-
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
508
|
-
const node = m?.[1]?.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
|
|
509
|
-
if (node)
|
|
510
|
-
refs.add(node);
|
|
511
|
-
}
|
|
512
718
|
run.nodes.set(spec.id, {
|
|
513
719
|
id: spec.id,
|
|
514
720
|
handler: spec.handler,
|
|
@@ -517,10 +723,11 @@ export class FakePipelines {
|
|
|
517
723
|
failureClass: FailureClass.UNSPECIFIED,
|
|
518
724
|
kv: {},
|
|
519
725
|
attempts: [],
|
|
520
|
-
needs:
|
|
726
|
+
needs: spec.needs,
|
|
521
727
|
with: spec.with,
|
|
522
728
|
retry: spec.retry ?? { max: this.opts.defaultRetries, when: [] },
|
|
523
729
|
timeoutMs: spec.timeoutMs || this.opts.defaultTimeoutMs,
|
|
730
|
+
interruptible: spec.interruptible,
|
|
524
731
|
});
|
|
525
732
|
}
|
|
526
733
|
/** Makes ready what can be, skips what cannot run, and settles the run. */
|
|
@@ -577,6 +784,7 @@ export class FakePipelines {
|
|
|
577
784
|
else {
|
|
578
785
|
run.state = RunState.SUCCEEDED;
|
|
579
786
|
}
|
|
787
|
+
this.runFinished(run);
|
|
580
788
|
}
|
|
581
789
|
}
|
|
582
790
|
resolve(run, n) {
|
|
@@ -625,14 +833,25 @@ export class FakePipelines {
|
|
|
625
833
|
logFinal: false,
|
|
626
834
|
logTruncated: false,
|
|
627
835
|
completedAt: undefined,
|
|
836
|
+
handlerFailureKind: HandlerFailureKind.UNSPECIFIED,
|
|
837
|
+
releases: 0,
|
|
838
|
+
offeredAt: Date.now(),
|
|
839
|
+
progress: undefined,
|
|
840
|
+
spans: [],
|
|
628
841
|
};
|
|
629
842
|
n.attempts.push(a);
|
|
630
843
|
this.attemptsById.set(a.id, a);
|
|
631
844
|
this.notify();
|
|
632
845
|
}
|
|
846
|
+
/**
|
|
847
|
+
* Claims for a worker: the attempts of handler nodes of full runs first,
|
|
848
|
+
* in creation order, as they always were; then jobs, through their
|
|
849
|
+
* queues' limits and fairness rule (see ./queues.ts).
|
|
850
|
+
*/
|
|
633
851
|
claim(pool, handlers, workerId, max) {
|
|
634
852
|
const now = Date.now();
|
|
635
853
|
const out = [];
|
|
854
|
+
const queued = [];
|
|
636
855
|
for (const run of this.runs.values()) {
|
|
637
856
|
if (run.tenant !== pool.owner?.tenant || run.cancelRequestedAt !== undefined || run.notBefore > now)
|
|
638
857
|
continue;
|
|
@@ -640,28 +859,55 @@ export class FakePipelines {
|
|
|
640
859
|
if (!handlers.includes(n.handler))
|
|
641
860
|
continue;
|
|
642
861
|
for (const a of n.attempts) {
|
|
643
|
-
if (out.length >= max)
|
|
644
|
-
return out;
|
|
645
862
|
if (a.state !== AttemptState.OFFERED || a.notBefore > now)
|
|
646
863
|
continue;
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
a.leaseExpiresAt = now + this.opts.leaseMs;
|
|
652
|
-
out.push(a);
|
|
864
|
+
if (run.queueId)
|
|
865
|
+
queued.push({ item: a, queueId: run.queueId, tenant: run.tenant, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
|
|
866
|
+
else if (out.length < max)
|
|
867
|
+
out.push(a);
|
|
653
868
|
}
|
|
654
869
|
}
|
|
655
870
|
}
|
|
871
|
+
if (out.length < max && queued.length > 0) {
|
|
872
|
+
const limits = new Map();
|
|
873
|
+
for (const q of this.queues.values())
|
|
874
|
+
limits.set(q.id, queueLimits(q));
|
|
875
|
+
const picks = pickQueued(queued, limits, { running: (q, t) => this.queueRunning(q, t), claimedSince: (q, since) => (this.queueClaims.get(q) ?? []).filter((x) => x >= since).length }, now, max - out.length);
|
|
876
|
+
for (const p of picks) {
|
|
877
|
+
out.push(p.item);
|
|
878
|
+
this.queueClaims.set(p.queueId, [...(this.queueClaims.get(p.queueId) ?? []).filter((x) => x > now - 86_400_000), now]);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
for (const a of out) {
|
|
882
|
+
a.state = AttemptState.CLAIMED;
|
|
883
|
+
a.epoch += 1n;
|
|
884
|
+
a.workerId = workerId;
|
|
885
|
+
a.poolId = pool.id;
|
|
886
|
+
a.leaseExpiresAt = now + this.opts.leaseMs;
|
|
887
|
+
this.initProgress(a);
|
|
888
|
+
}
|
|
656
889
|
return out;
|
|
657
890
|
}
|
|
891
|
+
/** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
|
|
892
|
+
queueRunning(queueId, tenant) {
|
|
893
|
+
let n = 0;
|
|
894
|
+
for (const run of this.runs.values()) {
|
|
895
|
+
if (run.queueId !== queueId || (tenant !== undefined && run.tenant !== tenant))
|
|
896
|
+
continue;
|
|
897
|
+
for (const node of run.nodes.values())
|
|
898
|
+
for (const a of node.attempts)
|
|
899
|
+
if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING)
|
|
900
|
+
n++;
|
|
901
|
+
}
|
|
902
|
+
return n;
|
|
903
|
+
}
|
|
658
904
|
describe(a) {
|
|
659
905
|
const run = this.run(a.runId);
|
|
660
906
|
const n = run.nodes.get(a.nodeId);
|
|
661
907
|
const pkg = this.packages.get(n.handler);
|
|
662
908
|
const version = pkg ? [...pkg.versions.keys()].at(-1) : "0.0.0";
|
|
663
909
|
return create(ClaimedAttemptSchema, {
|
|
664
|
-
ref: { attemptId: a.id, claimEpoch: a.epoch },
|
|
910
|
+
ref: { attemptId: a.id, claimEpoch: a.epoch, workerId: a.workerId },
|
|
665
911
|
runId: run.id,
|
|
666
912
|
nodeId: n.id,
|
|
667
913
|
number: a.number,
|
|
@@ -705,9 +951,23 @@ export class FakePipelines {
|
|
|
705
951
|
n.failureClass = FailureClass.CANCELLED;
|
|
706
952
|
}
|
|
707
953
|
else {
|
|
708
|
-
const
|
|
709
|
-
const
|
|
710
|
-
|
|
954
|
+
const permanent = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.PERMANENT;
|
|
955
|
+
const transientHandler = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.TRANSIENT;
|
|
956
|
+
const when = n.retry.when.length > 0 ? n.retry.when : transientHandler ? [...TRANSIENT, FailureClass.HANDLER_ERROR] : TRANSIENT;
|
|
957
|
+
// On an interruptible node an interruption is free, up to a bound.
|
|
958
|
+
const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
|
|
959
|
+
const freeUsed = n.attempts.filter(free).length;
|
|
960
|
+
if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
|
|
961
|
+
// Up to the bound, an interruption is offered again at once.
|
|
962
|
+
this.offer(run, n, a.inputs, 0);
|
|
963
|
+
return;
|
|
964
|
+
}
|
|
965
|
+
const retries = n.attempts.slice(0, -1).filter((x) => !free(x)).length;
|
|
966
|
+
// As the api: a worker that vanished (ABANDONED) may have had outside
|
|
967
|
+
// effects, so its node runs again automatically only when it declares
|
|
968
|
+
// itself interruptible.
|
|
969
|
+
const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
|
|
970
|
+
if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
|
|
711
971
|
this.offer(run, n, a.inputs, this.opts.retryBackoffMs);
|
|
712
972
|
return;
|
|
713
973
|
}
|
|
@@ -769,6 +1029,7 @@ export class FakePipelines {
|
|
|
769
1029
|
}
|
|
770
1030
|
}
|
|
771
1031
|
}
|
|
1032
|
+
this.schedules.evaluate();
|
|
772
1033
|
this.notify();
|
|
773
1034
|
}
|
|
774
1035
|
notify() {
|
|
@@ -776,6 +1037,430 @@ export class FakePipelines {
|
|
|
776
1037
|
this.changed = signal();
|
|
777
1038
|
prev.resolve();
|
|
778
1039
|
}
|
|
1040
|
+
/** QueuesService, as the contract states it. */
|
|
1041
|
+
queuesService() {
|
|
1042
|
+
const readable = (who, q) => {
|
|
1043
|
+
if (!q || q.owner?.product !== productOf(who) || (q.owner.tenant && q.owner.tenant !== who.tenant))
|
|
1044
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1045
|
+
return q;
|
|
1046
|
+
};
|
|
1047
|
+
const byId = (who, id) => readable(who, this.queues.get(id));
|
|
1048
|
+
return {
|
|
1049
|
+
setQueue: (req, ctx) => {
|
|
1050
|
+
const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
|
|
1051
|
+
const q = req.queue;
|
|
1052
|
+
if (!q?.owner)
|
|
1053
|
+
throw new ConnectError("queue.owner: required", Code.InvalidArgument);
|
|
1054
|
+
if (q.owner.product !== productOf(who))
|
|
1055
|
+
throw new ConnectError("queue.owner.product: a product sets only its own queues", Code.PermissionDenied);
|
|
1056
|
+
if (q.owner.tenant && q.owner.tenant !== who.tenant)
|
|
1057
|
+
throw new ConnectError("queue.owner.tenant: the caller does not act for it", Code.PermissionDenied);
|
|
1058
|
+
if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
|
|
1059
|
+
throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
|
|
1060
|
+
if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
|
|
1061
|
+
throw new ConnectError("max_concurrency_per_tenant must not exceed a non-zero max_concurrency", Code.InvalidArgument);
|
|
1062
|
+
}
|
|
1063
|
+
if (req.updateMask)
|
|
1064
|
+
throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
|
|
1065
|
+
const existing = [...this.queues.values()].find((x) => x.owner?.product === q.owner.product && x.owner.tenant === q.owner.tenant && x.name === q.name);
|
|
1066
|
+
const now = Date.now();
|
|
1067
|
+
const next = create(QueueSchema, {
|
|
1068
|
+
...q,
|
|
1069
|
+
id: existing?.id ?? randomUUID(),
|
|
1070
|
+
paused: existing?.paused ?? false,
|
|
1071
|
+
pausedReason: existing?.pausedReason ?? "",
|
|
1072
|
+
pausedBy: existing?.pausedBy ?? "",
|
|
1073
|
+
...(existing?.pausedAt ? { pausedAt: existing.pausedAt } : {}),
|
|
1074
|
+
createdAt: existing?.createdAt ?? timestampFromMs(now),
|
|
1075
|
+
updatedAt: timestampFromMs(now),
|
|
1076
|
+
});
|
|
1077
|
+
this.queues.set(next.id, next);
|
|
1078
|
+
this.notify();
|
|
1079
|
+
return { queue: next, created: !existing };
|
|
1080
|
+
},
|
|
1081
|
+
getQueue: (req, ctx) => {
|
|
1082
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1083
|
+
if (req.queue.case === "queueId")
|
|
1084
|
+
return { queue: byId(who, req.queue.value) };
|
|
1085
|
+
if (req.queue.case !== "byName")
|
|
1086
|
+
throw new ConnectError("queue: required", Code.InvalidArgument);
|
|
1087
|
+
const { owner, name } = req.queue.value;
|
|
1088
|
+
if (owner?.product !== productOf(who))
|
|
1089
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1090
|
+
const found = [...this.queues.values()].find((x) => x.owner?.product === owner.product && x.owner.tenant === owner.tenant && x.name === name);
|
|
1091
|
+
if (found)
|
|
1092
|
+
return { queue: readable(who, found) };
|
|
1093
|
+
if (name === "default" && !owner.tenant)
|
|
1094
|
+
return { queue: this.defaultQueue(owner.product) };
|
|
1095
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1096
|
+
},
|
|
1097
|
+
listQueues: (req, ctx) => {
|
|
1098
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1099
|
+
const product = req.product || productOf(who);
|
|
1100
|
+
if (product !== productOf(who))
|
|
1101
|
+
return { queues: [], nextCursor: "" };
|
|
1102
|
+
const queues = [...this.queues.values()]
|
|
1103
|
+
.filter((q) => q.owner?.product === product && (!req.tenant || q.owner.tenant === req.tenant) && (!q.owner.tenant || q.owner.tenant === who.tenant))
|
|
1104
|
+
.sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
|
|
1105
|
+
return { queues, nextCursor: "" };
|
|
1106
|
+
},
|
|
1107
|
+
markQueuePaused: (req, ctx) => {
|
|
1108
|
+
const who = this.enter("MarkQueuePaused", ctx, "pipelines:queue:edit");
|
|
1109
|
+
const q = byId(who, req.queueId);
|
|
1110
|
+
if (!q.paused) {
|
|
1111
|
+
q.paused = true;
|
|
1112
|
+
q.pausedAt = timestampFromMs(Date.now());
|
|
1113
|
+
q.pausedBy = `urn:fake:sa:${who.tenant}`;
|
|
1114
|
+
q.pausedReason = req.reason;
|
|
1115
|
+
}
|
|
1116
|
+
return { queue: q };
|
|
1117
|
+
},
|
|
1118
|
+
markQueueResumed: (req, ctx) => {
|
|
1119
|
+
const who = this.enter("MarkQueueResumed", ctx, "pipelines:queue:edit");
|
|
1120
|
+
const q = byId(who, req.queueId);
|
|
1121
|
+
q.paused = false;
|
|
1122
|
+
q.pausedReason = "";
|
|
1123
|
+
q.pausedBy = "";
|
|
1124
|
+
delete q.pausedAt;
|
|
1125
|
+
this.notify();
|
|
1126
|
+
return { queue: q };
|
|
1127
|
+
},
|
|
1128
|
+
deleteQueue: (req, ctx) => {
|
|
1129
|
+
const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
|
|
1130
|
+
const q = byId(who, req.queueId);
|
|
1131
|
+
const waiting = [...this.runs.values()].filter((r) => r.queueId === q.id && (r.state === RunState.QUEUED || (r.state === RunState.RUNNING && this.attemptsOf(r.id).some((a) => a.state === AttemptState.OFFERED))));
|
|
1132
|
+
if (waiting.length > 0 && !req.cancelWaiting)
|
|
1133
|
+
throw new ConnectError(`queue ${q.name} has ${waiting.length} waiting jobs`, Code.FailedPrecondition);
|
|
1134
|
+
for (const r of waiting)
|
|
1135
|
+
this.cancel(r, "queue deleted");
|
|
1136
|
+
if (q.name === "default" && !q.owner?.tenant) {
|
|
1137
|
+
this.queues.set(q.id, create(QueueSchema, { id: q.id, owner: q.owner, name: "default", createdAt: q.createdAt, updatedAt: timestampFromMs(Date.now()) }));
|
|
1138
|
+
}
|
|
1139
|
+
else {
|
|
1140
|
+
this.queues.delete(q.id);
|
|
1141
|
+
}
|
|
1142
|
+
return { cancelledJobs: waiting.length };
|
|
1143
|
+
},
|
|
1144
|
+
getQueueStats: (req, ctx) => {
|
|
1145
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1146
|
+
const q = byId(who, req.queueId);
|
|
1147
|
+
const now = Date.now();
|
|
1148
|
+
const windowMs = req.window ? durationMs(req.window) : 3_600_000;
|
|
1149
|
+
const runs = [...this.runs.values()].filter((r) => r.queueId === q.id);
|
|
1150
|
+
let waiting = 0;
|
|
1151
|
+
let delayed = 0;
|
|
1152
|
+
let running = 0;
|
|
1153
|
+
let started = 0n;
|
|
1154
|
+
let oldest;
|
|
1155
|
+
const tenants = new Map();
|
|
1156
|
+
for (const r of runs) {
|
|
1157
|
+
const t = tenants.get(r.tenant) ?? { waiting: 0, running: 0 };
|
|
1158
|
+
for (const a of this.attemptsOf(r.id)) {
|
|
1159
|
+
if (a.state === AttemptState.OFFERED) {
|
|
1160
|
+
if (a.notBefore > now || r.notBefore > now)
|
|
1161
|
+
delayed++;
|
|
1162
|
+
else {
|
|
1163
|
+
waiting++;
|
|
1164
|
+
t.waiting++;
|
|
1165
|
+
const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
|
|
1166
|
+
oldest = oldest === undefined ? since : Math.min(oldest, since);
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
|
|
1170
|
+
running++;
|
|
1171
|
+
t.running++;
|
|
1172
|
+
}
|
|
1173
|
+
if (a.startedAt && a.startedAt >= now - windowMs)
|
|
1174
|
+
started++;
|
|
1175
|
+
}
|
|
1176
|
+
if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
|
|
1177
|
+
delayed++;
|
|
1178
|
+
tenants.set(r.tenant, t);
|
|
1179
|
+
}
|
|
1180
|
+
const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
|
|
1181
|
+
return {
|
|
1182
|
+
waiting,
|
|
1183
|
+
delayed,
|
|
1184
|
+
running,
|
|
1185
|
+
...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
|
|
1186
|
+
tenants: [...tenants.entries()]
|
|
1187
|
+
.map(([tenant, v]) => ({ tenant, ...v }))
|
|
1188
|
+
.sort((a, b) => b.waiting - a.waiting || b.running - a.running)
|
|
1189
|
+
.slice(0, req.topTenants || 10),
|
|
1190
|
+
started,
|
|
1191
|
+
failed,
|
|
1192
|
+
paused: q.paused,
|
|
1193
|
+
at: timestampFromMs(now),
|
|
1194
|
+
};
|
|
1195
|
+
},
|
|
1196
|
+
listQueueJobs: (req, ctx) => {
|
|
1197
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1198
|
+
const q = byId(who, req.queueId);
|
|
1199
|
+
const runs = [...this.runs.values()]
|
|
1200
|
+
.filter((r) => r.queueId === q.id && (!req.tenant || r.tenant === req.tenant) && (req.states.length === 0 || req.states.includes(r.state)))
|
|
1201
|
+
.reverse();
|
|
1202
|
+
return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
|
|
1203
|
+
},
|
|
1204
|
+
};
|
|
1205
|
+
}
|
|
1206
|
+
// ---- jobs and queues --------------------------------------------------
|
|
1207
|
+
/** CreateJob's work, shared with schedules: resolve the queue, apply its limits, make the one-node run. */
|
|
1208
|
+
createJobRun(who, spec) {
|
|
1209
|
+
this.requireHandler(spec.handler);
|
|
1210
|
+
if (spec.retry && spec.retry.max > 10)
|
|
1211
|
+
throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
|
|
1212
|
+
if (spec.retry?.when.includes(FailureClass.CANCELLED))
|
|
1213
|
+
throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
|
|
1214
|
+
const q = this.resolveQueue(productOf(who), spec.tenant, spec.queue);
|
|
1215
|
+
if (!q)
|
|
1216
|
+
throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among this product's queues`, Code.NotFound);
|
|
1217
|
+
const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
|
|
1218
|
+
const payload = this.checkInputs("job", spec.handler, spec.payload);
|
|
1219
|
+
const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
|
|
1220
|
+
run.queueId = q.id;
|
|
1221
|
+
run.queue = q.name;
|
|
1222
|
+
run.trigger = spec.trigger ?? RunTrigger.API;
|
|
1223
|
+
if (spec.schedule) {
|
|
1224
|
+
run.scheduleId = spec.schedule.id;
|
|
1225
|
+
run.scheduledFor = spec.schedule.scheduledFor;
|
|
1226
|
+
run.scheduleTarget = spec.schedule.target;
|
|
1227
|
+
}
|
|
1228
|
+
this.addNode(run, {
|
|
1229
|
+
id: "job",
|
|
1230
|
+
handler: spec.handler,
|
|
1231
|
+
uses: "",
|
|
1232
|
+
with: payload,
|
|
1233
|
+
needs: [],
|
|
1234
|
+
retry,
|
|
1235
|
+
timeoutMs,
|
|
1236
|
+
interruptible: spec.interruptible,
|
|
1237
|
+
});
|
|
1238
|
+
this.advance(run);
|
|
1239
|
+
return run;
|
|
1240
|
+
}
|
|
1241
|
+
/** A job's queue: the tenant's own of that name, else the product-wide one; empty is `default`. */
|
|
1242
|
+
resolveQueue(product, tenant, name) {
|
|
1243
|
+
const want = name || "default";
|
|
1244
|
+
const all = [...this.queues.values()].filter((q) => q.owner?.product === product && q.name === want);
|
|
1245
|
+
const found = all.find((q) => q.owner?.tenant === tenant) ?? all.find((q) => !q.owner?.tenant);
|
|
1246
|
+
if (found)
|
|
1247
|
+
return found;
|
|
1248
|
+
return want === "default" ? this.defaultQueue(product) : undefined;
|
|
1249
|
+
}
|
|
1250
|
+
/** The product's `default` queue, made on first use with no limits. */
|
|
1251
|
+
defaultQueue(product) {
|
|
1252
|
+
const existing = [...this.queues.values()].find((q) => q.owner?.product === product && !q.owner.tenant && q.name === "default");
|
|
1253
|
+
if (existing)
|
|
1254
|
+
return existing;
|
|
1255
|
+
const now = Date.now();
|
|
1256
|
+
const q = create(QueueSchema, { id: randomUUID(), owner: { product, tenant: "" }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
|
|
1257
|
+
this.queues.set(q.id, q);
|
|
1258
|
+
return q;
|
|
1259
|
+
}
|
|
1260
|
+
/** RetryNode: reopens a finished run and offers the node again; on a job, a requeue. */
|
|
1261
|
+
retryNode(run, nodeId) {
|
|
1262
|
+
const n = run.nodes.get(nodeId);
|
|
1263
|
+
if (!n)
|
|
1264
|
+
throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
|
|
1265
|
+
if (!terminal(run.state))
|
|
1266
|
+
throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
|
|
1267
|
+
if (!n.handler || n.attempts.length === 0 || !terminalNode(n.state))
|
|
1268
|
+
throw new ConnectError("the node never settled", Code.FailedPrecondition);
|
|
1269
|
+
run.state = RunState.RUNNING;
|
|
1270
|
+
run.finishedAt = undefined;
|
|
1271
|
+
run.failureClass = FailureClass.UNSPECIFIED;
|
|
1272
|
+
run.failedNodeId = "";
|
|
1273
|
+
run.cancelRequestedAt = undefined;
|
|
1274
|
+
// The node and everything downstream of it run again.
|
|
1275
|
+
const reset = new Set([nodeId]);
|
|
1276
|
+
for (let grew = true; grew;) {
|
|
1277
|
+
grew = false;
|
|
1278
|
+
for (const x of run.nodes.values()) {
|
|
1279
|
+
if (!reset.has(x.id) && x.needs.some((d) => reset.has(d))) {
|
|
1280
|
+
reset.add(x.id);
|
|
1281
|
+
grew = true;
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
for (const id of reset) {
|
|
1286
|
+
const x = run.nodes.get(id);
|
|
1287
|
+
x.state = id === nodeId ? NodeState.READY : NodeState.PENDING;
|
|
1288
|
+
x.failureClass = FailureClass.UNSPECIFIED;
|
|
1289
|
+
x.kv = {};
|
|
1290
|
+
}
|
|
1291
|
+
const inputs = this.resolve(run, n);
|
|
1292
|
+
this.offer(run, n, typeof inputs === "string" ? {} : inputs, 0);
|
|
1293
|
+
}
|
|
1294
|
+
// ---- issues, progress and spans ---------------------------------------
|
|
1295
|
+
/** Every declared stage PENDING from the claim, as the platform does. */
|
|
1296
|
+
initProgress(a) {
|
|
1297
|
+
if (a.progress)
|
|
1298
|
+
return;
|
|
1299
|
+
const n = this.run(a.runId).nodes.get(a.nodeId);
|
|
1300
|
+
const pkg = this.packages.get(n.handler);
|
|
1301
|
+
const manifest = pkg ? [...pkg.versions.values()].at(-1) : undefined;
|
|
1302
|
+
const stages = manifest?.stages ?? [];
|
|
1303
|
+
if (stages.length === 0)
|
|
1304
|
+
return;
|
|
1305
|
+
a.progress = {
|
|
1306
|
+
fraction: 0,
|
|
1307
|
+
message: "",
|
|
1308
|
+
processed: 0n,
|
|
1309
|
+
total: 0n,
|
|
1310
|
+
failed: 0n,
|
|
1311
|
+
updatedAt: Date.now(),
|
|
1312
|
+
reports: 0,
|
|
1313
|
+
stages: stages.map((st) => ({ key: st.key, name: st.name, weight: st.weight || 1, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n })),
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
setProgress(a, req) {
|
|
1317
|
+
const now = Date.now();
|
|
1318
|
+
const p = (a.progress ??= { fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n, stages: [], updatedAt: now, reports: 0 });
|
|
1319
|
+
p.reports++;
|
|
1320
|
+
p.updatedAt = now;
|
|
1321
|
+
p.message = req.message;
|
|
1322
|
+
p.processed = req.processed;
|
|
1323
|
+
p.total = req.total;
|
|
1324
|
+
p.failed = req.failed;
|
|
1325
|
+
if (!req.stage) {
|
|
1326
|
+
p.fraction = Math.max(p.fraction, req.fraction);
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
let st = p.stages.find((x) => x.key === req.stage);
|
|
1330
|
+
if (!st) {
|
|
1331
|
+
st = { key: req.stage, name: req.stageName || req.stage, weight: 0, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n };
|
|
1332
|
+
p.stages.push(st);
|
|
1333
|
+
}
|
|
1334
|
+
// Reporting a stage finishes the one before it.
|
|
1335
|
+
for (const other of p.stages) {
|
|
1336
|
+
if (other !== st && other.state === StageState.RUNNING) {
|
|
1337
|
+
other.state = StageState.SUCCEEDED;
|
|
1338
|
+
other.fraction = 1;
|
|
1339
|
+
other.finishedAt = now;
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
const state = req.stageState === StageState.UNSPECIFIED ? StageState.RUNNING : req.stageState;
|
|
1343
|
+
if (st.state === StageState.PENDING || st.state === StageState.RUNNING) {
|
|
1344
|
+
st.state = state;
|
|
1345
|
+
st.startedAt ??= now;
|
|
1346
|
+
if (state !== StageState.RUNNING)
|
|
1347
|
+
st.finishedAt = now;
|
|
1348
|
+
}
|
|
1349
|
+
st.fraction = state === StageState.SUCCEEDED ? 1 : req.fraction;
|
|
1350
|
+
st.message = req.message;
|
|
1351
|
+
st.processed = req.processed;
|
|
1352
|
+
st.total = req.total;
|
|
1353
|
+
st.failed = req.failed;
|
|
1354
|
+
const total = p.stages.reduce((s, x) => s + x.weight, 0);
|
|
1355
|
+
if (total > 0) {
|
|
1356
|
+
const done = p.stages.reduce((s, x) => s + (x.state === StageState.SUCCEEDED || x.state === StageState.FAILED || x.state === StageState.SKIPPED ? x.weight : x.state === StageState.RUNNING ? x.weight * x.fraction : 0), 0);
|
|
1357
|
+
p.fraction = Math.max(p.fraction, Math.min(1, done / total));
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
appendIssues(a, issues) {
|
|
1361
|
+
const run = this.run(a.runId);
|
|
1362
|
+
let accepted = 0;
|
|
1363
|
+
for (const i of issues) {
|
|
1364
|
+
if (!i.message)
|
|
1365
|
+
throw new ConnectError("issues.message: required", Code.InvalidArgument);
|
|
1366
|
+
if (i.severity === 0)
|
|
1367
|
+
throw new ConnectError("issues.severity: required", Code.InvalidArgument);
|
|
1368
|
+
}
|
|
1369
|
+
for (const i of issues) {
|
|
1370
|
+
if (run.issues.length >= this.opts.issueCap) {
|
|
1371
|
+
run.issuesTruncated = true;
|
|
1372
|
+
continue;
|
|
1373
|
+
}
|
|
1374
|
+
run.issues.push({
|
|
1375
|
+
id: randomUUID(),
|
|
1376
|
+
runId: run.id,
|
|
1377
|
+
nodeId: a.nodeId,
|
|
1378
|
+
attemptId: a.id,
|
|
1379
|
+
stage: i.stage,
|
|
1380
|
+
severity: i.severity,
|
|
1381
|
+
subject: i.subject || i.subjectId,
|
|
1382
|
+
subjectKind: i.subjectKind,
|
|
1383
|
+
subjectId: i.subjectId,
|
|
1384
|
+
message: i.message,
|
|
1385
|
+
detail: i.detail,
|
|
1386
|
+
createdAt: Date.now(),
|
|
1387
|
+
});
|
|
1388
|
+
accepted++;
|
|
1389
|
+
}
|
|
1390
|
+
return accepted;
|
|
1391
|
+
}
|
|
1392
|
+
/** Issues of a run, filtered as ListIssues filters them. */
|
|
1393
|
+
issuesOf(runId, filter = {}) {
|
|
1394
|
+
return this.run(runId).issues.filter((i) => (!filter.nodeId || i.nodeId === filter.nodeId) && (!filter.attemptId || i.attemptId === filter.attemptId) && (!filter.minSeverity || i.severity >= filter.minSeverity));
|
|
1395
|
+
}
|
|
1396
|
+
/** The run's trace: the platform's spans for the run, its nodes and attempts, and the step spans reported. */
|
|
1397
|
+
traceOf(runId, filter = {}) {
|
|
1398
|
+
const run = this.run(runId);
|
|
1399
|
+
const traceId = run.id.replaceAll("-", "");
|
|
1400
|
+
const sid = (s) => s.replaceAll("-", "").slice(0, 16);
|
|
1401
|
+
const out = [
|
|
1402
|
+
create(TraceSpanSchema, {
|
|
1403
|
+
traceId,
|
|
1404
|
+
spanId: sid(run.id),
|
|
1405
|
+
name: `run ${run.id}`,
|
|
1406
|
+
source: TraceSpanSource.PLATFORM,
|
|
1407
|
+
startedAt: timestampFromMs(run.createdAt),
|
|
1408
|
+
...(run.finishedAt ? { endedAt: timestampFromMs(run.finishedAt) } : {}),
|
|
1409
|
+
}),
|
|
1410
|
+
];
|
|
1411
|
+
for (const n of run.nodes.values()) {
|
|
1412
|
+
if (filter.nodeId && n.id !== filter.nodeId)
|
|
1413
|
+
continue;
|
|
1414
|
+
const nodeSpan = sid(randomUUIDFrom(run.id + n.id));
|
|
1415
|
+
const first = n.attempts[0];
|
|
1416
|
+
out.push(create(TraceSpanSchema, {
|
|
1417
|
+
traceId,
|
|
1418
|
+
spanId: nodeSpan,
|
|
1419
|
+
parentSpanId: sid(run.id),
|
|
1420
|
+
name: `node ${n.id}`,
|
|
1421
|
+
source: TraceSpanSource.PLATFORM,
|
|
1422
|
+
nodeId: n.id,
|
|
1423
|
+
startedAt: timestampFromMs(first?.offeredAt ?? run.createdAt),
|
|
1424
|
+
}));
|
|
1425
|
+
for (const a of n.attempts) {
|
|
1426
|
+
if (filter.attemptId && a.id !== filter.attemptId)
|
|
1427
|
+
continue;
|
|
1428
|
+
out.push(create(TraceSpanSchema, {
|
|
1429
|
+
traceId,
|
|
1430
|
+
spanId: sid(a.id),
|
|
1431
|
+
parentSpanId: nodeSpan,
|
|
1432
|
+
name: `attempt ${a.number}`,
|
|
1433
|
+
source: TraceSpanSource.PLATFORM,
|
|
1434
|
+
nodeId: n.id,
|
|
1435
|
+
attemptId: a.id,
|
|
1436
|
+
startedAt: timestampFromMs(a.offeredAt),
|
|
1437
|
+
...(a.completedAt ? { endedAt: timestampFromMs(a.completedAt) } : {}),
|
|
1438
|
+
}));
|
|
1439
|
+
for (const sp of a.spans) {
|
|
1440
|
+
out.push(create(TraceSpanSchema, {
|
|
1441
|
+
traceId: sp.traceId,
|
|
1442
|
+
spanId: sp.spanId,
|
|
1443
|
+
parentSpanId: sp.parentSpanId || sid(a.id),
|
|
1444
|
+
name: sp.name,
|
|
1445
|
+
source: TraceSpanSource.STEP,
|
|
1446
|
+
nodeId: n.id,
|
|
1447
|
+
attemptId: a.id,
|
|
1448
|
+
...(sp.startedAt ? { startedAt: sp.startedAt } : {}),
|
|
1449
|
+
...(sp.endedAt ? { endedAt: sp.endedAt } : {}),
|
|
1450
|
+
status: sp.status,
|
|
1451
|
+
statusMessage: sp.statusMessage,
|
|
1452
|
+
...(sp.attributes ? { attributes: sp.attributes } : {}),
|
|
1453
|
+
}));
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
return out.sort((x, y) => Number(x.startedAt?.seconds ?? 0n) - Number(y.startedAt?.seconds ?? 0n));
|
|
1458
|
+
}
|
|
1459
|
+
/** Runs whenever a run becomes terminal. */
|
|
1460
|
+
runFinished(run) {
|
|
1461
|
+
// The schedules are made after the router; a run cannot finish before.
|
|
1462
|
+
this.schedules?.runFinished(run);
|
|
1463
|
+
}
|
|
779
1464
|
// ---- messages ---------------------------------------------------------
|
|
780
1465
|
runMessage(r) {
|
|
781
1466
|
return create(RunSchema, {
|
|
@@ -795,6 +1480,16 @@ export class FakePipelines {
|
|
|
795
1480
|
...(r.finishedAt ? { finishedAt: timestampFromMs(r.finishedAt) } : {}),
|
|
796
1481
|
...(r.cancelRequestedAt ? { cancelRequestedAt: timestampFromMs(r.cancelRequestedAt) } : {}),
|
|
797
1482
|
cancelReason: r.cancelReason,
|
|
1483
|
+
queueId: r.queueId,
|
|
1484
|
+
queue: r.queue,
|
|
1485
|
+
trigger: r.trigger,
|
|
1486
|
+
scheduleId: r.scheduleId,
|
|
1487
|
+
...(r.scheduledFor !== undefined ? { scheduledFor: timestampFromMs(r.scheduledFor) } : {}),
|
|
1488
|
+
scheduleTarget: r.scheduleTarget,
|
|
1489
|
+
issueInfoCount: r.issues.filter((i) => i.severity === IssueSeverity.INFO).length,
|
|
1490
|
+
issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
|
|
1491
|
+
issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
|
|
1492
|
+
issuesTruncated: r.issuesTruncated,
|
|
798
1493
|
});
|
|
799
1494
|
}
|
|
800
1495
|
nodeMessage(n) {
|
|
@@ -819,10 +1514,71 @@ export class FakePipelines {
|
|
|
819
1514
|
message: a.message,
|
|
820
1515
|
logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
|
|
821
1516
|
logTruncated: a.logTruncated,
|
|
1517
|
+
...(a.progress
|
|
1518
|
+
? {
|
|
1519
|
+
progress: {
|
|
1520
|
+
fraction: a.progress.fraction,
|
|
1521
|
+
message: a.progress.message,
|
|
1522
|
+
processed: a.progress.processed,
|
|
1523
|
+
total: a.progress.total,
|
|
1524
|
+
failed: a.progress.failed,
|
|
1525
|
+
updatedAt: timestampFromMs(a.progress.updatedAt),
|
|
1526
|
+
stages: a.progress.stages.map((st) => ({
|
|
1527
|
+
key: st.key,
|
|
1528
|
+
name: st.name,
|
|
1529
|
+
weight: st.weight,
|
|
1530
|
+
state: st.state,
|
|
1531
|
+
fraction: st.fraction,
|
|
1532
|
+
message: st.message,
|
|
1533
|
+
processed: st.processed,
|
|
1534
|
+
total: st.total,
|
|
1535
|
+
failed: st.failed,
|
|
1536
|
+
...(st.startedAt ? { startedAt: timestampFromMs(st.startedAt) } : {}),
|
|
1537
|
+
...(st.finishedAt ? { finishedAt: timestampFromMs(st.finishedAt) } : {}),
|
|
1538
|
+
})),
|
|
1539
|
+
},
|
|
1540
|
+
}
|
|
1541
|
+
: {}),
|
|
822
1542
|
})),
|
|
823
1543
|
});
|
|
824
1544
|
}
|
|
825
1545
|
}
|
|
1546
|
+
/**
|
|
1547
|
+
* Holds a definition to the server's rules the fake models: unique ids,
|
|
1548
|
+
* known needs, no cycle, and a reference names only a node in the referring
|
|
1549
|
+
* node's dependency closure (a reference adds no edge).
|
|
1550
|
+
*/
|
|
1551
|
+
function checkGraph(specs) {
|
|
1552
|
+
const byId = new Map();
|
|
1553
|
+
for (const s of specs) {
|
|
1554
|
+
if (byId.has(s.id))
|
|
1555
|
+
throw new ConnectError(`node ${s.id}: duplicate id`, Code.InvalidArgument);
|
|
1556
|
+
byId.set(s.id, s);
|
|
1557
|
+
}
|
|
1558
|
+
const closure = (id, seen = new Set(), path = new Set()) => {
|
|
1559
|
+
for (const need of byId.get(id).needs) {
|
|
1560
|
+
if (!byId.has(need))
|
|
1561
|
+
throw new ConnectError(`node ${id}: unknown need ${need}`, Code.InvalidArgument);
|
|
1562
|
+
if (path.has(need) || need === id)
|
|
1563
|
+
throw new ConnectError(`node ${id}: the needs form a cycle`, Code.InvalidArgument);
|
|
1564
|
+
if (seen.has(need))
|
|
1565
|
+
continue;
|
|
1566
|
+
seen.add(need);
|
|
1567
|
+
closure(need, seen, new Set([...path, id]));
|
|
1568
|
+
}
|
|
1569
|
+
return seen;
|
|
1570
|
+
};
|
|
1571
|
+
for (const s of specs) {
|
|
1572
|
+
const reachable = closure(s.id);
|
|
1573
|
+
for (const v of Object.values(s.with)) {
|
|
1574
|
+
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
1575
|
+
const node = m?.[1]?.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
|
|
1576
|
+
if (node && !reachable.has(node)) {
|
|
1577
|
+
throw new ConnectError(`node ${s.id}: ${m[1]} names a node outside its needs`, Code.InvalidArgument);
|
|
1578
|
+
}
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
826
1582
|
function nodeSpec(n, requireHandler) {
|
|
827
1583
|
let handler = "";
|
|
828
1584
|
let uses = "";
|
|
@@ -844,6 +1600,7 @@ function nodeSpec(n, requireHandler) {
|
|
|
844
1600
|
needs: n.needs.map((x) => x.node),
|
|
845
1601
|
retry: n.retry ? { max: n.retry.max, when: n.retry.when } : undefined,
|
|
846
1602
|
timeoutMs: n.timeout ? durationMs(n.timeout) : 0,
|
|
1603
|
+
interruptible: n.interruptible,
|
|
847
1604
|
};
|
|
848
1605
|
}
|
|
849
1606
|
/** The server's dispatch.Outcome. */
|
|
@@ -867,6 +1624,8 @@ function outcome(t, exitCode) {
|
|
|
867
1624
|
return [AttemptState.FAILED, FailureClass.SECRETS_UNAVAILABLE];
|
|
868
1625
|
case Termination.HANDLER_ERROR:
|
|
869
1626
|
return [AttemptState.FAILED, FailureClass.HANDLER_ERROR];
|
|
1627
|
+
case Termination.INTERRUPTED:
|
|
1628
|
+
return [AttemptState.FAILED, FailureClass.INTERRUPTED];
|
|
870
1629
|
default:
|
|
871
1630
|
return [AttemptState.FAILED, FailureClass.RUNNER_CRASHED];
|
|
872
1631
|
}
|
|
@@ -906,4 +1665,44 @@ function aborted(s) {
|
|
|
906
1665
|
s.addEventListener("abort", () => r(), { once: true });
|
|
907
1666
|
});
|
|
908
1667
|
}
|
|
1668
|
+
/** The product an account belongs to. */
|
|
1669
|
+
function productOf(who) {
|
|
1670
|
+
return who.product ?? FAKE_PRODUCT;
|
|
1671
|
+
}
|
|
1672
|
+
/** A queue's limits as the selection reads them. */
|
|
1673
|
+
function queueLimits(q) {
|
|
1674
|
+
return {
|
|
1675
|
+
id: q.id,
|
|
1676
|
+
priority: q.priority || 1,
|
|
1677
|
+
paused: q.paused,
|
|
1678
|
+
maxConcurrency: q.maxConcurrency,
|
|
1679
|
+
maxConcurrencyPerTenant: q.maxConcurrencyPerTenant,
|
|
1680
|
+
...(q.rateLimit?.window ? { rateLimit: { limit: q.rateLimit.limit, windowMs: durationMs(q.rateLimit.window) } } : {}),
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
/**
|
|
1684
|
+
* A job's retry and timeout under its queue: the queue's defaults when the
|
|
1685
|
+
* job gives none, and its ceilings, past which the job is refused.
|
|
1686
|
+
*/
|
|
1687
|
+
function jobLimits(q, retry, timeoutMs) {
|
|
1688
|
+
const ceiling = q.defaultRetry;
|
|
1689
|
+
if (retry && ceiling) {
|
|
1690
|
+
if (retry.max > ceiling.max)
|
|
1691
|
+
throw new ConnectError(`retry.max ${retry.max} is above queue ${q.name}'s ${ceiling.max}`, Code.InvalidArgument);
|
|
1692
|
+
const allowed = ceiling.when.length > 0 ? ceiling.when : TRANSIENT;
|
|
1693
|
+
const outside = retry.when.filter((c) => !allowed.includes(c));
|
|
1694
|
+
if (outside.length > 0)
|
|
1695
|
+
throw new ConnectError(`retry.when lists ${outside.map((c) => FailureClass[c]).join(", ")}, which queue ${q.name} does not retry`, Code.InvalidArgument);
|
|
1696
|
+
}
|
|
1697
|
+
const maxTimeout = q.defaultTimeout ? durationMs(q.defaultTimeout) : 0;
|
|
1698
|
+
if (timeoutMs > 0 && maxTimeout > 0 && timeoutMs > maxTimeout) {
|
|
1699
|
+
throw new ConnectError(`timeout ${timeoutMs} ms is above queue ${q.name}'s ${maxTimeout} ms`, Code.InvalidArgument);
|
|
1700
|
+
}
|
|
1701
|
+
const r = retry ?? ceiling;
|
|
1702
|
+
return { retry: r ? { max: r.max, when: r.when } : undefined, timeoutMs: timeoutMs || maxTimeout };
|
|
1703
|
+
}
|
|
1704
|
+
/** A stable UUID-shaped id from a string, for the fake's platform spans. */
|
|
1705
|
+
function randomUUIDFrom(s) {
|
|
1706
|
+
return createHash("sha256").update(s).digest("hex").slice(0, 32);
|
|
1707
|
+
}
|
|
909
1708
|
//# sourceMappingURL=index.js.map
|