@xeonr/pipelines-sdk 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +172 -16
- package/dist/client.d.ts +63 -4
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +43 -1
- package/dist/client.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +173 -12
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +45 -28
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +136 -17
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +42 -35
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +187 -2
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +38 -5
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +4 -2
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/common_pb.js +4 -2
- package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +277 -4
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js +100 -7
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +1100 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +298 -0
- package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js +1 -1
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +769 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js +190 -0
- package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +60 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js +21 -16
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +846 -3
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js +243 -27
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +1110 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +295 -0
- package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +1 -1
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts +123 -1
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
- package/dist/gen/xeonr/platform/v1/options_pb.js +90 -1
- package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
- package/dist/handler.d.ts +50 -0
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +27 -0
- package/dist/handler.js.map +1 -1
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/queues.d.ts +171 -0
- package/dist/queues.d.ts.map +1 -0
- package/dist/queues.js +135 -0
- package/dist/queues.js.map +1 -0
- package/dist/reporter.d.ts +120 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporter.js +323 -0
- package/dist/reporter.js.map +1 -0
- package/dist/schedules.d.ts +201 -0
- package/dist/schedules.d.ts.map +1 -0
- package/dist/schedules.js +127 -0
- package/dist/schedules.js.map +1 -0
- package/dist/testing/cron.d.ts +33 -0
- package/dist/testing/cron.d.ts.map +1 -0
- package/dist/testing/cron.js +177 -0
- package/dist/testing/cron.js.map +1 -0
- package/dist/testing/index.d.ts +129 -4
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +723 -30
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/queues.d.ts +46 -0
- package/dist/testing/queues.d.ts.map +1 -0
- package/dist/testing/queues.js +70 -0
- package/dist/testing/queues.js.map +1 -0
- package/dist/testing/schedules.d.ts +113 -0
- package/dist/testing/schedules.d.ts.map +1 -0
- package/dist/testing/schedules.js +419 -0
- package/dist/testing/schedules.js.map +1 -0
- package/dist/worker.d.ts +48 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +67 -6
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
package/dist/testing/index.js
CHANGED
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
* What it does not: secrets, blobs, tiers, routing between pools, the
|
|
35
35
|
* outbox, and the platform's own deadline enforcement.
|
|
36
36
|
*/
|
|
37
|
-
import { randomUUID } from "node:crypto";
|
|
37
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
38
38
|
import { create, fromJson, toJson } from "@bufbuild/protobuf";
|
|
39
39
|
import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
40
40
|
import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
|
|
@@ -43,7 +43,14 @@ import { authInterceptor } from "../transport.js";
|
|
|
43
43
|
import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
|
|
44
44
|
import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
|
|
45
45
|
import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
|
|
46
|
-
import { AttemptSchema, NodeRunSchema, RunSchema, RunsService } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
46
|
+
import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
47
|
+
import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
|
|
48
|
+
import { pickQueued } from "./queues.js";
|
|
49
|
+
import { FakeClock, FakeSchedules } from "./schedules.js";
|
|
50
|
+
import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
|
|
51
|
+
export { FakeClock, FakeSchedules } from "./schedules.js";
|
|
52
|
+
export { parseCron, nextOccurrence } from "./cron.js";
|
|
53
|
+
export { pickQueued } from "./queues.js";
|
|
47
54
|
import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
|
|
48
55
|
/** The tenant the default account acts for. */
|
|
49
56
|
export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
|
|
@@ -58,7 +65,13 @@ export const ALL_SCOPES = [
|
|
|
58
65
|
"pipelines:pool:read",
|
|
59
66
|
"pipelines:pool:edit",
|
|
60
67
|
"pipelines:worker:edit",
|
|
68
|
+
"pipelines:queue:read",
|
|
69
|
+
"pipelines:queue:edit",
|
|
70
|
+
"pipelines:schedule:read",
|
|
71
|
+
"pipelines:schedule:edit",
|
|
61
72
|
];
|
|
73
|
+
/** The product the default account belongs to. */
|
|
74
|
+
export const FAKE_PRODUCT = "00000000-0000-4000-8000-0000000000a1";
|
|
62
75
|
const TRANSIENT = [
|
|
63
76
|
FailureClass.ABANDONED,
|
|
64
77
|
FailureClass.RUNNER_CRASHED,
|
|
@@ -68,8 +81,8 @@ const TRANSIENT = [
|
|
|
68
81
|
FailureClass.INTERNAL,
|
|
69
82
|
];
|
|
70
83
|
const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
71
|
-
/** How many interruptions an interruptible node gets without spending a retry. */
|
|
72
|
-
const FREE_INTERRUPTIONS =
|
|
84
|
+
/** How many interruptions an interruptible node gets without spending a retry (the api's default). */
|
|
85
|
+
const FREE_INTERRUPTIONS = 3;
|
|
73
86
|
const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
|
|
74
87
|
/**
|
|
75
88
|
* An in-memory pipelines-api for tests. Call {@link close} when done (it
|
|
@@ -88,6 +101,16 @@ export class FakePipelines {
|
|
|
88
101
|
attemptsById = new Map();
|
|
89
102
|
idempotency = new Map();
|
|
90
103
|
injected = new Map();
|
|
104
|
+
/** Queues by id. */
|
|
105
|
+
queues = new Map();
|
|
106
|
+
/**
|
|
107
|
+
* The clock schedules run on: real time until a test calls
|
|
108
|
+
* `clock.set()` or `clock.advance()`, each of which fires what is due.
|
|
109
|
+
*/
|
|
110
|
+
clock = new FakeClock();
|
|
111
|
+
/** The schedules, with their recorded occurrences. */
|
|
112
|
+
schedules;
|
|
113
|
+
queueClaims = new Map();
|
|
91
114
|
timer;
|
|
92
115
|
changed;
|
|
93
116
|
constructor(opts = {}) {
|
|
@@ -102,8 +125,29 @@ export class FakePipelines {
|
|
|
102
125
|
retryBackoffMs: opts.retryBackoffMs ?? 0,
|
|
103
126
|
logMaxBytes: opts.logMaxBytes ?? 0,
|
|
104
127
|
tickMs: opts.tickMs ?? 10,
|
|
128
|
+
issueCap: opts.issueCap ?? 1000,
|
|
129
|
+
spanCap: opts.spanCap ?? 10_000,
|
|
105
130
|
};
|
|
106
131
|
this.changed = signal();
|
|
132
|
+
this.schedules = new FakeSchedules({
|
|
133
|
+
createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES, product: owner.product }, {
|
|
134
|
+
tenant: owner.tenant,
|
|
135
|
+
handler: spec.handler,
|
|
136
|
+
key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
|
|
137
|
+
payload: spec.payload,
|
|
138
|
+
delayMs: 0,
|
|
139
|
+
resource: spec.resource,
|
|
140
|
+
retry: spec.retry,
|
|
141
|
+
timeoutMs: spec.timeoutMs,
|
|
142
|
+
queue: spec.queue,
|
|
143
|
+
interruptible: spec.interruptible,
|
|
144
|
+
trigger: spec.trigger,
|
|
145
|
+
schedule: spec.schedule,
|
|
146
|
+
}),
|
|
147
|
+
runs: () => this.runs.values(),
|
|
148
|
+
runMessage: (id) => this.runMessage(this.run(id)),
|
|
149
|
+
checkHandler: (_product, handler) => this.requireHandler(handler),
|
|
150
|
+
}, this.clock, FAKE_PRODUCT);
|
|
107
151
|
this.timer = setInterval(() => this.tick(), this.opts.tickMs);
|
|
108
152
|
this.timer.unref();
|
|
109
153
|
}
|
|
@@ -371,37 +415,107 @@ export class FakePipelines {
|
|
|
371
415
|
this.notify();
|
|
372
416
|
return { state: a.state };
|
|
373
417
|
},
|
|
418
|
+
appendIssues: (req, ctx) => {
|
|
419
|
+
const who = this.enter("AppendIssues", ctx, "pipelines:worker:edit");
|
|
420
|
+
const a = this.held(who, req.ref);
|
|
421
|
+
if (a.state !== AttemptState.RUNNING)
|
|
422
|
+
throw stale();
|
|
423
|
+
if (req.issues.length < 1 || req.issues.length > 100)
|
|
424
|
+
throw new ConnectError("issues: 1 to 100", Code.InvalidArgument);
|
|
425
|
+
return { accepted: this.appendIssues(a, req.issues) };
|
|
426
|
+
},
|
|
427
|
+
setProgress: (req, ctx) => {
|
|
428
|
+
const who = this.enter("SetProgress", ctx, "pipelines:worker:edit");
|
|
429
|
+
const a = this.held(who, req.ref);
|
|
430
|
+
if (a.state !== AttemptState.RUNNING)
|
|
431
|
+
throw stale();
|
|
432
|
+
if (!(req.fraction >= 0 && req.fraction <= 1))
|
|
433
|
+
throw new ConnectError("fraction: 0 to 1", Code.InvalidArgument);
|
|
434
|
+
if (req.stageState === StageState.PENDING)
|
|
435
|
+
throw new ConnectError("stage_state: a stage never moves back to PENDING", Code.InvalidArgument);
|
|
436
|
+
if (req.stage && !/^[a-z][a-z0-9_.-]{0,63}$/.test(req.stage))
|
|
437
|
+
throw new ConnectError("stage: invalid key", Code.InvalidArgument);
|
|
438
|
+
if (req.message.length > 255)
|
|
439
|
+
throw new ConnectError("message: at most 255", Code.InvalidArgument);
|
|
440
|
+
this.setProgress(a, req);
|
|
441
|
+
return {};
|
|
442
|
+
},
|
|
443
|
+
reportSpans: (req, ctx) => {
|
|
444
|
+
const who = this.enter("ReportSpans", ctx, "pipelines:worker:edit");
|
|
445
|
+
const a = this.held(who, req.ref);
|
|
446
|
+
if (a.state !== AttemptState.RUNNING)
|
|
447
|
+
throw stale();
|
|
448
|
+
if (req.spans.length < 1 || req.spans.length > 1000)
|
|
449
|
+
throw new ConnectError("spans: 1 to 1000", Code.InvalidArgument);
|
|
450
|
+
for (const sp of req.spans) {
|
|
451
|
+
if (!/^[0-9a-f]{32}$/.test(sp.traceId) || !/^[0-9a-f]{16}$/.test(sp.spanId) || !sp.startedAt || !sp.endedAt || !sp.name) {
|
|
452
|
+
throw new ConnectError("spans: trace_id, span_id, name, started_at and ended_at are required", Code.InvalidArgument);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
const room = Math.max(0, this.opts.spanCap - a.spans.length);
|
|
456
|
+
a.spans.push(...req.spans.slice(0, room));
|
|
457
|
+
return { accepted: Math.min(room, req.spans.length) };
|
|
458
|
+
},
|
|
374
459
|
});
|
|
460
|
+
router.service(QueuesService, this.queuesService());
|
|
461
|
+
router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
|
|
375
462
|
router.service(RunsService, {
|
|
463
|
+
listIssues: (req, ctx) => {
|
|
464
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
465
|
+
this.visible(who, req.runId);
|
|
466
|
+
const issues = this.issuesOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId, minSeverity: req.minSeverity });
|
|
467
|
+
return {
|
|
468
|
+
issues: issues.map((i) => create(RunIssueSchema, { ...i, detail: i.detail, createdAt: timestampFromMs(i.createdAt) })),
|
|
469
|
+
nextCursor: "",
|
|
470
|
+
};
|
|
471
|
+
},
|
|
472
|
+
getLogs: (req, ctx) => {
|
|
473
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
474
|
+
const a = this.attemptsById.get(req.attemptId);
|
|
475
|
+
if (!a)
|
|
476
|
+
throw new ConnectError("no such attempt", Code.NotFound);
|
|
477
|
+
this.visible(who, a.runId);
|
|
478
|
+
const all = Buffer.concat(a.chunks);
|
|
479
|
+
const from = Math.min(Number(req.offset), all.length);
|
|
480
|
+
const to = req.limitBytes ? Math.min(all.length, from + req.limitBytes) : all.length;
|
|
481
|
+
const complete = to === all.length && (a.logFinal || a.completedAt !== undefined);
|
|
482
|
+
return { data: new Uint8Array(all.subarray(from, to)), nextOffset: BigInt(to), complete, truncated: a.logTruncated };
|
|
483
|
+
},
|
|
484
|
+
getRunTrace: (req, ctx) => {
|
|
485
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
486
|
+
this.visible(who, req.runId);
|
|
487
|
+
return { spans: this.traceOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId }), truncated: false };
|
|
488
|
+
},
|
|
376
489
|
createJob: (req, ctx) => {
|
|
377
490
|
const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
|
|
378
491
|
this.actsFor(who, req.tenant);
|
|
379
|
-
const existing = this.idempotency.get(`${who
|
|
492
|
+
const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
|
|
380
493
|
if (existing)
|
|
381
494
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
382
|
-
this.
|
|
383
|
-
|
|
384
|
-
if (req.retry && req.retry.max > 10)
|
|
385
|
-
throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
|
|
386
|
-
if (req.retry?.when.includes(FailureClass.CANCELLED))
|
|
387
|
-
throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
|
|
388
|
-
this.addNode(run, {
|
|
389
|
-
id: "job",
|
|
495
|
+
const run = this.createJobRun(who, {
|
|
496
|
+
tenant: req.tenant,
|
|
390
497
|
handler: req.handler,
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
498
|
+
key: req.idempotencyKey,
|
|
499
|
+
payload: req.payload ?? {},
|
|
500
|
+
delayMs: req.delay ? durationMs(req.delay) : 0,
|
|
501
|
+
resource: req.resource,
|
|
502
|
+
retry: req.retry,
|
|
395
503
|
timeoutMs: req.timeout ? durationMs(req.timeout) : 0,
|
|
396
|
-
|
|
504
|
+
queue: req.queue,
|
|
505
|
+
interruptible: req.interruptible,
|
|
397
506
|
});
|
|
398
|
-
this.advance(run);
|
|
399
507
|
return { run: this.runMessage(run), deduplicated: false };
|
|
400
508
|
},
|
|
509
|
+
retryNode: (req, ctx) => {
|
|
510
|
+
const who = this.enter("RetryNode", ctx, "pipelines:run:edit");
|
|
511
|
+
const run = this.visible(who, req.runId);
|
|
512
|
+
this.retryNode(run, req.nodeId);
|
|
513
|
+
return { run: this.runMessage(run) };
|
|
514
|
+
},
|
|
401
515
|
createRun: (req, ctx) => {
|
|
402
516
|
const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
|
|
403
517
|
this.actsFor(who, req.tenant);
|
|
404
|
-
const existing = this.idempotency.get(`${who
|
|
518
|
+
const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
|
|
405
519
|
if (existing)
|
|
406
520
|
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
407
521
|
if (req.source.case !== "inlineDefinition")
|
|
@@ -410,6 +524,9 @@ export class FakePipelines {
|
|
|
410
524
|
if (def.nodes.length === 0)
|
|
411
525
|
throw new ConnectError("definition: no nodes", Code.InvalidArgument);
|
|
412
526
|
const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
|
|
527
|
+
for (const sp of specs)
|
|
528
|
+
if (sp.handler)
|
|
529
|
+
sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
|
|
413
530
|
checkGraph(specs);
|
|
414
531
|
const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, req.variables ?? {}, 0);
|
|
415
532
|
for (const spec of specs)
|
|
@@ -537,13 +654,39 @@ export class FakePipelines {
|
|
|
537
654
|
throw new ConnectError(`handler ${handler} is not a published worker package`, Code.InvalidArgument);
|
|
538
655
|
}
|
|
539
656
|
}
|
|
657
|
+
/**
|
|
658
|
+
* Holds a handler node's `with` to its manifest, as the api does when a run
|
|
659
|
+
* is created: every key must be a declared input, every required input
|
|
660
|
+
* given; defaults fill the rest. Returns the inputs with defaults.
|
|
661
|
+
*/
|
|
662
|
+
checkInputs(nodeId, handler, given) {
|
|
663
|
+
const pkg = this.packages.get(handler);
|
|
664
|
+
const version = pkg ? [...pkg.versions.keys()].at(-1) : "";
|
|
665
|
+
const manifest = pkg?.versions.get(version);
|
|
666
|
+
const inputs = manifest?.inputs ?? [];
|
|
667
|
+
for (const k of Object.keys(given)) {
|
|
668
|
+
if (!inputs.some((i) => i.name === k)) {
|
|
669
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": input "${k}" is not an input of ${handler}@${version}`, Code.InvalidArgument);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
const out = { ...given };
|
|
673
|
+
for (const i of inputs) {
|
|
674
|
+
if (out[i.name] !== undefined)
|
|
675
|
+
continue;
|
|
676
|
+
if (i.default)
|
|
677
|
+
out[i.name] = toJson(ValueSchema, i.default);
|
|
678
|
+
else if (i.required)
|
|
679
|
+
throw new ConnectError(`the definition is invalid: node "${nodeId}": required input "${i.name}" is not given`, Code.InvalidArgument);
|
|
680
|
+
}
|
|
681
|
+
return out;
|
|
682
|
+
}
|
|
540
683
|
// ---- the scheduler ----------------------------------------------------
|
|
541
684
|
newRun(who, tenant, key, resource, variables, delayMs) {
|
|
542
685
|
const now = Date.now();
|
|
543
686
|
const run = {
|
|
544
687
|
id: randomUUID(),
|
|
545
688
|
tenant,
|
|
546
|
-
product: who
|
|
689
|
+
product: productOf(who),
|
|
547
690
|
idempotencyKey: key,
|
|
548
691
|
createdBy: `urn:fake:sa:${who.tenant}`,
|
|
549
692
|
resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
|
|
@@ -558,6 +701,14 @@ export class FakePipelines {
|
|
|
558
701
|
cancelRequestedAt: undefined,
|
|
559
702
|
cancelReason: "",
|
|
560
703
|
nodes: new Map(),
|
|
704
|
+
queueId: "",
|
|
705
|
+
queue: "",
|
|
706
|
+
trigger: RunTrigger.API,
|
|
707
|
+
scheduleId: "",
|
|
708
|
+
scheduledFor: undefined,
|
|
709
|
+
scheduleTarget: "",
|
|
710
|
+
issues: [],
|
|
711
|
+
issuesTruncated: false,
|
|
561
712
|
};
|
|
562
713
|
this.runs.set(run.id, run);
|
|
563
714
|
this.idempotency.set(`${run.product}\u0000${key}`, run.id);
|
|
@@ -633,6 +784,7 @@ export class FakePipelines {
|
|
|
633
784
|
else {
|
|
634
785
|
run.state = RunState.SUCCEEDED;
|
|
635
786
|
}
|
|
787
|
+
this.runFinished(run);
|
|
636
788
|
}
|
|
637
789
|
}
|
|
638
790
|
resolve(run, n) {
|
|
@@ -683,14 +835,23 @@ export class FakePipelines {
|
|
|
683
835
|
completedAt: undefined,
|
|
684
836
|
handlerFailureKind: HandlerFailureKind.UNSPECIFIED,
|
|
685
837
|
releases: 0,
|
|
838
|
+
offeredAt: Date.now(),
|
|
839
|
+
progress: undefined,
|
|
840
|
+
spans: [],
|
|
686
841
|
};
|
|
687
842
|
n.attempts.push(a);
|
|
688
843
|
this.attemptsById.set(a.id, a);
|
|
689
844
|
this.notify();
|
|
690
845
|
}
|
|
846
|
+
/**
|
|
847
|
+
* Claims for a worker: the attempts of handler nodes of full runs first,
|
|
848
|
+
* in creation order, as they always were; then jobs, through their
|
|
849
|
+
* queues' limits and fairness rule (see ./queues.ts).
|
|
850
|
+
*/
|
|
691
851
|
claim(pool, handlers, workerId, max) {
|
|
692
852
|
const now = Date.now();
|
|
693
853
|
const out = [];
|
|
854
|
+
const queued = [];
|
|
694
855
|
for (const run of this.runs.values()) {
|
|
695
856
|
if (run.tenant !== pool.owner?.tenant || run.cancelRequestedAt !== undefined || run.notBefore > now)
|
|
696
857
|
continue;
|
|
@@ -698,21 +859,48 @@ export class FakePipelines {
|
|
|
698
859
|
if (!handlers.includes(n.handler))
|
|
699
860
|
continue;
|
|
700
861
|
for (const a of n.attempts) {
|
|
701
|
-
if (out.length >= max)
|
|
702
|
-
return out;
|
|
703
862
|
if (a.state !== AttemptState.OFFERED || a.notBefore > now)
|
|
704
863
|
continue;
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
a.leaseExpiresAt = now + this.opts.leaseMs;
|
|
710
|
-
out.push(a);
|
|
864
|
+
if (run.queueId)
|
|
865
|
+
queued.push({ item: a, queueId: run.queueId, tenant: run.tenant, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
|
|
866
|
+
else if (out.length < max)
|
|
867
|
+
out.push(a);
|
|
711
868
|
}
|
|
712
869
|
}
|
|
713
870
|
}
|
|
871
|
+
if (out.length < max && queued.length > 0) {
|
|
872
|
+
const limits = new Map();
|
|
873
|
+
for (const q of this.queues.values())
|
|
874
|
+
limits.set(q.id, queueLimits(q));
|
|
875
|
+
const picks = pickQueued(queued, limits, { running: (q, t) => this.queueRunning(q, t), claimedSince: (q, since) => (this.queueClaims.get(q) ?? []).filter((x) => x >= since).length }, now, max - out.length);
|
|
876
|
+
for (const p of picks) {
|
|
877
|
+
out.push(p.item);
|
|
878
|
+
this.queueClaims.set(p.queueId, [...(this.queueClaims.get(p.queueId) ?? []).filter((x) => x > now - 86_400_000), now]);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
for (const a of out) {
|
|
882
|
+
a.state = AttemptState.CLAIMED;
|
|
883
|
+
a.epoch += 1n;
|
|
884
|
+
a.workerId = workerId;
|
|
885
|
+
a.poolId = pool.id;
|
|
886
|
+
a.leaseExpiresAt = now + this.opts.leaseMs;
|
|
887
|
+
this.initProgress(a);
|
|
888
|
+
}
|
|
714
889
|
return out;
|
|
715
890
|
}
|
|
891
|
+
/** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
|
|
892
|
+
queueRunning(queueId, tenant) {
|
|
893
|
+
let n = 0;
|
|
894
|
+
for (const run of this.runs.values()) {
|
|
895
|
+
if (run.queueId !== queueId || (tenant !== undefined && run.tenant !== tenant))
|
|
896
|
+
continue;
|
|
897
|
+
for (const node of run.nodes.values())
|
|
898
|
+
for (const a of node.attempts)
|
|
899
|
+
if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING)
|
|
900
|
+
n++;
|
|
901
|
+
}
|
|
902
|
+
return n;
|
|
903
|
+
}
|
|
716
904
|
describe(a) {
|
|
717
905
|
const run = this.run(a.runId);
|
|
718
906
|
const n = run.nodes.get(a.nodeId);
|
|
@@ -770,11 +958,16 @@ export class FakePipelines {
|
|
|
770
958
|
const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
|
|
771
959
|
const freeUsed = n.attempts.filter(free).length;
|
|
772
960
|
if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
|
|
961
|
+
// Up to the bound, an interruption is offered again at once.
|
|
773
962
|
this.offer(run, n, a.inputs, 0);
|
|
774
963
|
return;
|
|
775
964
|
}
|
|
776
965
|
const retries = n.attempts.slice(0, -1).filter((x) => !free(x)).length;
|
|
777
|
-
|
|
966
|
+
// As the api: a worker that vanished (ABANDONED) may have had outside
|
|
967
|
+
// effects, so its node runs again automatically only when it declares
|
|
968
|
+
// itself interruptible.
|
|
969
|
+
const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
|
|
970
|
+
if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
|
|
778
971
|
this.offer(run, n, a.inputs, this.opts.retryBackoffMs);
|
|
779
972
|
return;
|
|
780
973
|
}
|
|
@@ -836,6 +1029,7 @@ export class FakePipelines {
|
|
|
836
1029
|
}
|
|
837
1030
|
}
|
|
838
1031
|
}
|
|
1032
|
+
this.schedules.evaluate();
|
|
839
1033
|
this.notify();
|
|
840
1034
|
}
|
|
841
1035
|
notify() {
|
|
@@ -843,6 +1037,430 @@ export class FakePipelines {
|
|
|
843
1037
|
this.changed = signal();
|
|
844
1038
|
prev.resolve();
|
|
845
1039
|
}
|
|
1040
|
+
/** QueuesService, as the contract states it. */
|
|
1041
|
+
queuesService() {
|
|
1042
|
+
const readable = (who, q) => {
|
|
1043
|
+
if (!q || q.owner?.product !== productOf(who) || (q.owner.tenant && q.owner.tenant !== who.tenant))
|
|
1044
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1045
|
+
return q;
|
|
1046
|
+
};
|
|
1047
|
+
const byId = (who, id) => readable(who, this.queues.get(id));
|
|
1048
|
+
return {
|
|
1049
|
+
setQueue: (req, ctx) => {
|
|
1050
|
+
const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
|
|
1051
|
+
const q = req.queue;
|
|
1052
|
+
if (!q?.owner)
|
|
1053
|
+
throw new ConnectError("queue.owner: required", Code.InvalidArgument);
|
|
1054
|
+
if (q.owner.product !== productOf(who))
|
|
1055
|
+
throw new ConnectError("queue.owner.product: a product sets only its own queues", Code.PermissionDenied);
|
|
1056
|
+
if (q.owner.tenant && q.owner.tenant !== who.tenant)
|
|
1057
|
+
throw new ConnectError("queue.owner.tenant: the caller does not act for it", Code.PermissionDenied);
|
|
1058
|
+
if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
|
|
1059
|
+
throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
|
|
1060
|
+
if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
|
|
1061
|
+
throw new ConnectError("max_concurrency_per_tenant must not exceed a non-zero max_concurrency", Code.InvalidArgument);
|
|
1062
|
+
}
|
|
1063
|
+
if (req.updateMask)
|
|
1064
|
+
throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
|
|
1065
|
+
const existing = [...this.queues.values()].find((x) => x.owner?.product === q.owner.product && x.owner.tenant === q.owner.tenant && x.name === q.name);
|
|
1066
|
+
const now = Date.now();
|
|
1067
|
+
const next = create(QueueSchema, {
|
|
1068
|
+
...q,
|
|
1069
|
+
id: existing?.id ?? randomUUID(),
|
|
1070
|
+
paused: existing?.paused ?? false,
|
|
1071
|
+
pausedReason: existing?.pausedReason ?? "",
|
|
1072
|
+
pausedBy: existing?.pausedBy ?? "",
|
|
1073
|
+
...(existing?.pausedAt ? { pausedAt: existing.pausedAt } : {}),
|
|
1074
|
+
createdAt: existing?.createdAt ?? timestampFromMs(now),
|
|
1075
|
+
updatedAt: timestampFromMs(now),
|
|
1076
|
+
});
|
|
1077
|
+
this.queues.set(next.id, next);
|
|
1078
|
+
this.notify();
|
|
1079
|
+
return { queue: next, created: !existing };
|
|
1080
|
+
},
|
|
1081
|
+
getQueue: (req, ctx) => {
|
|
1082
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1083
|
+
if (req.queue.case === "queueId")
|
|
1084
|
+
return { queue: byId(who, req.queue.value) };
|
|
1085
|
+
if (req.queue.case !== "byName")
|
|
1086
|
+
throw new ConnectError("queue: required", Code.InvalidArgument);
|
|
1087
|
+
const { owner, name } = req.queue.value;
|
|
1088
|
+
if (owner?.product !== productOf(who))
|
|
1089
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1090
|
+
const found = [...this.queues.values()].find((x) => x.owner?.product === owner.product && x.owner.tenant === owner.tenant && x.name === name);
|
|
1091
|
+
if (found)
|
|
1092
|
+
return { queue: readable(who, found) };
|
|
1093
|
+
if (name === "default" && !owner.tenant)
|
|
1094
|
+
return { queue: this.defaultQueue(owner.product) };
|
|
1095
|
+
throw new ConnectError("no such queue", Code.NotFound);
|
|
1096
|
+
},
|
|
1097
|
+
listQueues: (req, ctx) => {
|
|
1098
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1099
|
+
const product = req.product || productOf(who);
|
|
1100
|
+
if (product !== productOf(who))
|
|
1101
|
+
return { queues: [], nextCursor: "" };
|
|
1102
|
+
const queues = [...this.queues.values()]
|
|
1103
|
+
.filter((q) => q.owner?.product === product && (!req.tenant || q.owner.tenant === req.tenant) && (!q.owner.tenant || q.owner.tenant === who.tenant))
|
|
1104
|
+
.sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
|
|
1105
|
+
return { queues, nextCursor: "" };
|
|
1106
|
+
},
|
|
1107
|
+
markQueuePaused: (req, ctx) => {
|
|
1108
|
+
const who = this.enter("MarkQueuePaused", ctx, "pipelines:queue:edit");
|
|
1109
|
+
const q = byId(who, req.queueId);
|
|
1110
|
+
if (!q.paused) {
|
|
1111
|
+
q.paused = true;
|
|
1112
|
+
q.pausedAt = timestampFromMs(Date.now());
|
|
1113
|
+
q.pausedBy = `urn:fake:sa:${who.tenant}`;
|
|
1114
|
+
q.pausedReason = req.reason;
|
|
1115
|
+
}
|
|
1116
|
+
return { queue: q };
|
|
1117
|
+
},
|
|
1118
|
+
markQueueResumed: (req, ctx) => {
|
|
1119
|
+
const who = this.enter("MarkQueueResumed", ctx, "pipelines:queue:edit");
|
|
1120
|
+
const q = byId(who, req.queueId);
|
|
1121
|
+
q.paused = false;
|
|
1122
|
+
q.pausedReason = "";
|
|
1123
|
+
q.pausedBy = "";
|
|
1124
|
+
delete q.pausedAt;
|
|
1125
|
+
this.notify();
|
|
1126
|
+
return { queue: q };
|
|
1127
|
+
},
|
|
1128
|
+
deleteQueue: (req, ctx) => {
|
|
1129
|
+
const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
|
|
1130
|
+
const q = byId(who, req.queueId);
|
|
1131
|
+
const waiting = [...this.runs.values()].filter((r) => r.queueId === q.id && (r.state === RunState.QUEUED || (r.state === RunState.RUNNING && this.attemptsOf(r.id).some((a) => a.state === AttemptState.OFFERED))));
|
|
1132
|
+
if (waiting.length > 0 && !req.cancelWaiting)
|
|
1133
|
+
throw new ConnectError(`queue ${q.name} has ${waiting.length} waiting jobs`, Code.FailedPrecondition);
|
|
1134
|
+
for (const r of waiting)
|
|
1135
|
+
this.cancel(r, "queue deleted");
|
|
1136
|
+
if (q.name === "default" && !q.owner?.tenant) {
|
|
1137
|
+
this.queues.set(q.id, create(QueueSchema, { id: q.id, owner: q.owner, name: "default", createdAt: q.createdAt, updatedAt: timestampFromMs(Date.now()) }));
|
|
1138
|
+
}
|
|
1139
|
+
else {
|
|
1140
|
+
this.queues.delete(q.id);
|
|
1141
|
+
}
|
|
1142
|
+
return { cancelledJobs: waiting.length };
|
|
1143
|
+
},
|
|
1144
|
+
getQueueStats: (req, ctx) => {
|
|
1145
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1146
|
+
const q = byId(who, req.queueId);
|
|
1147
|
+
const now = Date.now();
|
|
1148
|
+
const windowMs = req.window ? durationMs(req.window) : 3_600_000;
|
|
1149
|
+
const runs = [...this.runs.values()].filter((r) => r.queueId === q.id);
|
|
1150
|
+
let waiting = 0;
|
|
1151
|
+
let delayed = 0;
|
|
1152
|
+
let running = 0;
|
|
1153
|
+
let started = 0n;
|
|
1154
|
+
let oldest;
|
|
1155
|
+
const tenants = new Map();
|
|
1156
|
+
for (const r of runs) {
|
|
1157
|
+
const t = tenants.get(r.tenant) ?? { waiting: 0, running: 0 };
|
|
1158
|
+
for (const a of this.attemptsOf(r.id)) {
|
|
1159
|
+
if (a.state === AttemptState.OFFERED) {
|
|
1160
|
+
if (a.notBefore > now || r.notBefore > now)
|
|
1161
|
+
delayed++;
|
|
1162
|
+
else {
|
|
1163
|
+
waiting++;
|
|
1164
|
+
t.waiting++;
|
|
1165
|
+
const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
|
|
1166
|
+
oldest = oldest === undefined ? since : Math.min(oldest, since);
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
|
|
1170
|
+
running++;
|
|
1171
|
+
t.running++;
|
|
1172
|
+
}
|
|
1173
|
+
if (a.startedAt && a.startedAt >= now - windowMs)
|
|
1174
|
+
started++;
|
|
1175
|
+
}
|
|
1176
|
+
if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
|
|
1177
|
+
delayed++;
|
|
1178
|
+
tenants.set(r.tenant, t);
|
|
1179
|
+
}
|
|
1180
|
+
const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
|
|
1181
|
+
return {
|
|
1182
|
+
waiting,
|
|
1183
|
+
delayed,
|
|
1184
|
+
running,
|
|
1185
|
+
...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
|
|
1186
|
+
tenants: [...tenants.entries()]
|
|
1187
|
+
.map(([tenant, v]) => ({ tenant, ...v }))
|
|
1188
|
+
.sort((a, b) => b.waiting - a.waiting || b.running - a.running)
|
|
1189
|
+
.slice(0, req.topTenants || 10),
|
|
1190
|
+
started,
|
|
1191
|
+
failed,
|
|
1192
|
+
paused: q.paused,
|
|
1193
|
+
at: timestampFromMs(now),
|
|
1194
|
+
};
|
|
1195
|
+
},
|
|
1196
|
+
listQueueJobs: (req, ctx) => {
|
|
1197
|
+
const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
|
|
1198
|
+
const q = byId(who, req.queueId);
|
|
1199
|
+
const runs = [...this.runs.values()]
|
|
1200
|
+
.filter((r) => r.queueId === q.id && (!req.tenant || r.tenant === req.tenant) && (req.states.length === 0 || req.states.includes(r.state)))
|
|
1201
|
+
.reverse();
|
|
1202
|
+
return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
|
|
1203
|
+
},
|
|
1204
|
+
};
|
|
1205
|
+
}
|
|
1206
|
+
// ---- jobs and queues --------------------------------------------------
|
|
1207
|
+
/** CreateJob's work, shared with schedules: resolve the queue, apply its limits, make the one-node run. */
|
|
1208
|
+
createJobRun(who, spec) {
|
|
1209
|
+
this.requireHandler(spec.handler);
|
|
1210
|
+
if (spec.retry && spec.retry.max > 10)
|
|
1211
|
+
throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
|
|
1212
|
+
if (spec.retry?.when.includes(FailureClass.CANCELLED))
|
|
1213
|
+
throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
|
|
1214
|
+
const q = this.resolveQueue(productOf(who), spec.tenant, spec.queue);
|
|
1215
|
+
if (!q)
|
|
1216
|
+
throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among this product's queues`, Code.NotFound);
|
|
1217
|
+
const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
|
|
1218
|
+
const payload = this.checkInputs("job", spec.handler, spec.payload);
|
|
1219
|
+
const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
|
|
1220
|
+
run.queueId = q.id;
|
|
1221
|
+
run.queue = q.name;
|
|
1222
|
+
run.trigger = spec.trigger ?? RunTrigger.API;
|
|
1223
|
+
if (spec.schedule) {
|
|
1224
|
+
run.scheduleId = spec.schedule.id;
|
|
1225
|
+
run.scheduledFor = spec.schedule.scheduledFor;
|
|
1226
|
+
run.scheduleTarget = spec.schedule.target;
|
|
1227
|
+
}
|
|
1228
|
+
this.addNode(run, {
|
|
1229
|
+
id: "job",
|
|
1230
|
+
handler: spec.handler,
|
|
1231
|
+
uses: "",
|
|
1232
|
+
with: payload,
|
|
1233
|
+
needs: [],
|
|
1234
|
+
retry,
|
|
1235
|
+
timeoutMs,
|
|
1236
|
+
interruptible: spec.interruptible,
|
|
1237
|
+
});
|
|
1238
|
+
this.advance(run);
|
|
1239
|
+
return run;
|
|
1240
|
+
}
|
|
1241
|
+
/** A job's queue: the tenant's own of that name, else the product-wide one; empty is `default`. */
|
|
1242
|
+
resolveQueue(product, tenant, name) {
|
|
1243
|
+
const want = name || "default";
|
|
1244
|
+
const all = [...this.queues.values()].filter((q) => q.owner?.product === product && q.name === want);
|
|
1245
|
+
const found = all.find((q) => q.owner?.tenant === tenant) ?? all.find((q) => !q.owner?.tenant);
|
|
1246
|
+
if (found)
|
|
1247
|
+
return found;
|
|
1248
|
+
return want === "default" ? this.defaultQueue(product) : undefined;
|
|
1249
|
+
}
|
|
1250
|
+
/** The product's `default` queue, made on first use with no limits. */
|
|
1251
|
+
defaultQueue(product) {
|
|
1252
|
+
const existing = [...this.queues.values()].find((q) => q.owner?.product === product && !q.owner.tenant && q.name === "default");
|
|
1253
|
+
if (existing)
|
|
1254
|
+
return existing;
|
|
1255
|
+
const now = Date.now();
|
|
1256
|
+
const q = create(QueueSchema, { id: randomUUID(), owner: { product, tenant: "" }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
|
|
1257
|
+
this.queues.set(q.id, q);
|
|
1258
|
+
return q;
|
|
1259
|
+
}
|
|
1260
|
+
/** RetryNode: reopens a finished run and offers the node again; on a job, a requeue. */
|
|
1261
|
+
retryNode(run, nodeId) {
|
|
1262
|
+
const n = run.nodes.get(nodeId);
|
|
1263
|
+
if (!n)
|
|
1264
|
+
throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
|
|
1265
|
+
if (!terminal(run.state))
|
|
1266
|
+
throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
|
|
1267
|
+
if (!n.handler || n.attempts.length === 0 || !terminalNode(n.state))
|
|
1268
|
+
throw new ConnectError("the node never settled", Code.FailedPrecondition);
|
|
1269
|
+
run.state = RunState.RUNNING;
|
|
1270
|
+
run.finishedAt = undefined;
|
|
1271
|
+
run.failureClass = FailureClass.UNSPECIFIED;
|
|
1272
|
+
run.failedNodeId = "";
|
|
1273
|
+
run.cancelRequestedAt = undefined;
|
|
1274
|
+
// The node and everything downstream of it run again.
|
|
1275
|
+
const reset = new Set([nodeId]);
|
|
1276
|
+
for (let grew = true; grew;) {
|
|
1277
|
+
grew = false;
|
|
1278
|
+
for (const x of run.nodes.values()) {
|
|
1279
|
+
if (!reset.has(x.id) && x.needs.some((d) => reset.has(d))) {
|
|
1280
|
+
reset.add(x.id);
|
|
1281
|
+
grew = true;
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
for (const id of reset) {
|
|
1286
|
+
const x = run.nodes.get(id);
|
|
1287
|
+
x.state = id === nodeId ? NodeState.READY : NodeState.PENDING;
|
|
1288
|
+
x.failureClass = FailureClass.UNSPECIFIED;
|
|
1289
|
+
x.kv = {};
|
|
1290
|
+
}
|
|
1291
|
+
const inputs = this.resolve(run, n);
|
|
1292
|
+
this.offer(run, n, typeof inputs === "string" ? {} : inputs, 0);
|
|
1293
|
+
}
|
|
1294
|
+
// ---- issues, progress and spans ---------------------------------------
|
|
1295
|
+
/** Every declared stage PENDING from the claim, as the platform does. */
|
|
1296
|
+
initProgress(a) {
|
|
1297
|
+
if (a.progress)
|
|
1298
|
+
return;
|
|
1299
|
+
const n = this.run(a.runId).nodes.get(a.nodeId);
|
|
1300
|
+
const pkg = this.packages.get(n.handler);
|
|
1301
|
+
const manifest = pkg ? [...pkg.versions.values()].at(-1) : undefined;
|
|
1302
|
+
const stages = manifest?.stages ?? [];
|
|
1303
|
+
if (stages.length === 0)
|
|
1304
|
+
return;
|
|
1305
|
+
a.progress = {
|
|
1306
|
+
fraction: 0,
|
|
1307
|
+
message: "",
|
|
1308
|
+
processed: 0n,
|
|
1309
|
+
total: 0n,
|
|
1310
|
+
failed: 0n,
|
|
1311
|
+
updatedAt: Date.now(),
|
|
1312
|
+
reports: 0,
|
|
1313
|
+
stages: stages.map((st) => ({ key: st.key, name: st.name, weight: st.weight || 1, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n })),
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
setProgress(a, req) {
|
|
1317
|
+
const now = Date.now();
|
|
1318
|
+
const p = (a.progress ??= { fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n, stages: [], updatedAt: now, reports: 0 });
|
|
1319
|
+
p.reports++;
|
|
1320
|
+
p.updatedAt = now;
|
|
1321
|
+
p.message = req.message;
|
|
1322
|
+
p.processed = req.processed;
|
|
1323
|
+
p.total = req.total;
|
|
1324
|
+
p.failed = req.failed;
|
|
1325
|
+
if (!req.stage) {
|
|
1326
|
+
p.fraction = Math.max(p.fraction, req.fraction);
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
let st = p.stages.find((x) => x.key === req.stage);
|
|
1330
|
+
if (!st) {
|
|
1331
|
+
st = { key: req.stage, name: req.stageName || req.stage, weight: 0, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n };
|
|
1332
|
+
p.stages.push(st);
|
|
1333
|
+
}
|
|
1334
|
+
// Reporting a stage finishes the one before it.
|
|
1335
|
+
for (const other of p.stages) {
|
|
1336
|
+
if (other !== st && other.state === StageState.RUNNING) {
|
|
1337
|
+
other.state = StageState.SUCCEEDED;
|
|
1338
|
+
other.fraction = 1;
|
|
1339
|
+
other.finishedAt = now;
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
const state = req.stageState === StageState.UNSPECIFIED ? StageState.RUNNING : req.stageState;
|
|
1343
|
+
if (st.state === StageState.PENDING || st.state === StageState.RUNNING) {
|
|
1344
|
+
st.state = state;
|
|
1345
|
+
st.startedAt ??= now;
|
|
1346
|
+
if (state !== StageState.RUNNING)
|
|
1347
|
+
st.finishedAt = now;
|
|
1348
|
+
}
|
|
1349
|
+
st.fraction = state === StageState.SUCCEEDED ? 1 : req.fraction;
|
|
1350
|
+
st.message = req.message;
|
|
1351
|
+
st.processed = req.processed;
|
|
1352
|
+
st.total = req.total;
|
|
1353
|
+
st.failed = req.failed;
|
|
1354
|
+
const total = p.stages.reduce((s, x) => s + x.weight, 0);
|
|
1355
|
+
if (total > 0) {
|
|
1356
|
+
const done = p.stages.reduce((s, x) => s + (x.state === StageState.SUCCEEDED || x.state === StageState.FAILED || x.state === StageState.SKIPPED ? x.weight : x.state === StageState.RUNNING ? x.weight * x.fraction : 0), 0);
|
|
1357
|
+
p.fraction = Math.max(p.fraction, Math.min(1, done / total));
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
appendIssues(a, issues) {
|
|
1361
|
+
const run = this.run(a.runId);
|
|
1362
|
+
let accepted = 0;
|
|
1363
|
+
for (const i of issues) {
|
|
1364
|
+
if (!i.message)
|
|
1365
|
+
throw new ConnectError("issues.message: required", Code.InvalidArgument);
|
|
1366
|
+
if (i.severity === 0)
|
|
1367
|
+
throw new ConnectError("issues.severity: required", Code.InvalidArgument);
|
|
1368
|
+
}
|
|
1369
|
+
for (const i of issues) {
|
|
1370
|
+
if (run.issues.length >= this.opts.issueCap) {
|
|
1371
|
+
run.issuesTruncated = true;
|
|
1372
|
+
continue;
|
|
1373
|
+
}
|
|
1374
|
+
run.issues.push({
|
|
1375
|
+
id: randomUUID(),
|
|
1376
|
+
runId: run.id,
|
|
1377
|
+
nodeId: a.nodeId,
|
|
1378
|
+
attemptId: a.id,
|
|
1379
|
+
stage: i.stage,
|
|
1380
|
+
severity: i.severity,
|
|
1381
|
+
subject: i.subject || i.subjectId,
|
|
1382
|
+
subjectKind: i.subjectKind,
|
|
1383
|
+
subjectId: i.subjectId,
|
|
1384
|
+
message: i.message,
|
|
1385
|
+
detail: i.detail,
|
|
1386
|
+
createdAt: Date.now(),
|
|
1387
|
+
});
|
|
1388
|
+
accepted++;
|
|
1389
|
+
}
|
|
1390
|
+
return accepted;
|
|
1391
|
+
}
|
|
1392
|
+
/** Issues of a run, filtered as ListIssues filters them. */
|
|
1393
|
+
issuesOf(runId, filter = {}) {
|
|
1394
|
+
return this.run(runId).issues.filter((i) => (!filter.nodeId || i.nodeId === filter.nodeId) && (!filter.attemptId || i.attemptId === filter.attemptId) && (!filter.minSeverity || i.severity >= filter.minSeverity));
|
|
1395
|
+
}
|
|
1396
|
+
/** The run's trace: the platform's spans for the run, its nodes and attempts, and the step spans reported. */
|
|
1397
|
+
traceOf(runId, filter = {}) {
|
|
1398
|
+
const run = this.run(runId);
|
|
1399
|
+
const traceId = run.id.replaceAll("-", "");
|
|
1400
|
+
const sid = (s) => s.replaceAll("-", "").slice(0, 16);
|
|
1401
|
+
const out = [
|
|
1402
|
+
create(TraceSpanSchema, {
|
|
1403
|
+
traceId,
|
|
1404
|
+
spanId: sid(run.id),
|
|
1405
|
+
name: `run ${run.id}`,
|
|
1406
|
+
source: TraceSpanSource.PLATFORM,
|
|
1407
|
+
startedAt: timestampFromMs(run.createdAt),
|
|
1408
|
+
...(run.finishedAt ? { endedAt: timestampFromMs(run.finishedAt) } : {}),
|
|
1409
|
+
}),
|
|
1410
|
+
];
|
|
1411
|
+
for (const n of run.nodes.values()) {
|
|
1412
|
+
if (filter.nodeId && n.id !== filter.nodeId)
|
|
1413
|
+
continue;
|
|
1414
|
+
const nodeSpan = sid(randomUUIDFrom(run.id + n.id));
|
|
1415
|
+
const first = n.attempts[0];
|
|
1416
|
+
out.push(create(TraceSpanSchema, {
|
|
1417
|
+
traceId,
|
|
1418
|
+
spanId: nodeSpan,
|
|
1419
|
+
parentSpanId: sid(run.id),
|
|
1420
|
+
name: `node ${n.id}`,
|
|
1421
|
+
source: TraceSpanSource.PLATFORM,
|
|
1422
|
+
nodeId: n.id,
|
|
1423
|
+
startedAt: timestampFromMs(first?.offeredAt ?? run.createdAt),
|
|
1424
|
+
}));
|
|
1425
|
+
for (const a of n.attempts) {
|
|
1426
|
+
if (filter.attemptId && a.id !== filter.attemptId)
|
|
1427
|
+
continue;
|
|
1428
|
+
out.push(create(TraceSpanSchema, {
|
|
1429
|
+
traceId,
|
|
1430
|
+
spanId: sid(a.id),
|
|
1431
|
+
parentSpanId: nodeSpan,
|
|
1432
|
+
name: `attempt ${a.number}`,
|
|
1433
|
+
source: TraceSpanSource.PLATFORM,
|
|
1434
|
+
nodeId: n.id,
|
|
1435
|
+
attemptId: a.id,
|
|
1436
|
+
startedAt: timestampFromMs(a.offeredAt),
|
|
1437
|
+
...(a.completedAt ? { endedAt: timestampFromMs(a.completedAt) } : {}),
|
|
1438
|
+
}));
|
|
1439
|
+
for (const sp of a.spans) {
|
|
1440
|
+
out.push(create(TraceSpanSchema, {
|
|
1441
|
+
traceId: sp.traceId,
|
|
1442
|
+
spanId: sp.spanId,
|
|
1443
|
+
parentSpanId: sp.parentSpanId || sid(a.id),
|
|
1444
|
+
name: sp.name,
|
|
1445
|
+
source: TraceSpanSource.STEP,
|
|
1446
|
+
nodeId: n.id,
|
|
1447
|
+
attemptId: a.id,
|
|
1448
|
+
...(sp.startedAt ? { startedAt: sp.startedAt } : {}),
|
|
1449
|
+
...(sp.endedAt ? { endedAt: sp.endedAt } : {}),
|
|
1450
|
+
status: sp.status,
|
|
1451
|
+
statusMessage: sp.statusMessage,
|
|
1452
|
+
...(sp.attributes ? { attributes: sp.attributes } : {}),
|
|
1453
|
+
}));
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
return out.sort((x, y) => Number(x.startedAt?.seconds ?? 0n) - Number(y.startedAt?.seconds ?? 0n));
|
|
1458
|
+
}
|
|
1459
|
+
/** Runs whenever a run becomes terminal. */
|
|
1460
|
+
runFinished(run) {
|
|
1461
|
+
// The schedules are made after the router; a run cannot finish before.
|
|
1462
|
+
this.schedules?.runFinished(run);
|
|
1463
|
+
}
|
|
846
1464
|
// ---- messages ---------------------------------------------------------
|
|
847
1465
|
runMessage(r) {
|
|
848
1466
|
return create(RunSchema, {
|
|
@@ -862,6 +1480,16 @@ export class FakePipelines {
|
|
|
862
1480
|
...(r.finishedAt ? { finishedAt: timestampFromMs(r.finishedAt) } : {}),
|
|
863
1481
|
...(r.cancelRequestedAt ? { cancelRequestedAt: timestampFromMs(r.cancelRequestedAt) } : {}),
|
|
864
1482
|
cancelReason: r.cancelReason,
|
|
1483
|
+
queueId: r.queueId,
|
|
1484
|
+
queue: r.queue,
|
|
1485
|
+
trigger: r.trigger,
|
|
1486
|
+
scheduleId: r.scheduleId,
|
|
1487
|
+
...(r.scheduledFor !== undefined ? { scheduledFor: timestampFromMs(r.scheduledFor) } : {}),
|
|
1488
|
+
scheduleTarget: r.scheduleTarget,
|
|
1489
|
+
issueInfoCount: r.issues.filter((i) => i.severity === IssueSeverity.INFO).length,
|
|
1490
|
+
issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
|
|
1491
|
+
issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
|
|
1492
|
+
issuesTruncated: r.issuesTruncated,
|
|
865
1493
|
});
|
|
866
1494
|
}
|
|
867
1495
|
nodeMessage(n) {
|
|
@@ -886,6 +1514,31 @@ export class FakePipelines {
|
|
|
886
1514
|
message: a.message,
|
|
887
1515
|
logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
|
|
888
1516
|
logTruncated: a.logTruncated,
|
|
1517
|
+
...(a.progress
|
|
1518
|
+
? {
|
|
1519
|
+
progress: {
|
|
1520
|
+
fraction: a.progress.fraction,
|
|
1521
|
+
message: a.progress.message,
|
|
1522
|
+
processed: a.progress.processed,
|
|
1523
|
+
total: a.progress.total,
|
|
1524
|
+
failed: a.progress.failed,
|
|
1525
|
+
updatedAt: timestampFromMs(a.progress.updatedAt),
|
|
1526
|
+
stages: a.progress.stages.map((st) => ({
|
|
1527
|
+
key: st.key,
|
|
1528
|
+
name: st.name,
|
|
1529
|
+
weight: st.weight,
|
|
1530
|
+
state: st.state,
|
|
1531
|
+
fraction: st.fraction,
|
|
1532
|
+
message: st.message,
|
|
1533
|
+
processed: st.processed,
|
|
1534
|
+
total: st.total,
|
|
1535
|
+
failed: st.failed,
|
|
1536
|
+
...(st.startedAt ? { startedAt: timestampFromMs(st.startedAt) } : {}),
|
|
1537
|
+
...(st.finishedAt ? { finishedAt: timestampFromMs(st.finishedAt) } : {}),
|
|
1538
|
+
})),
|
|
1539
|
+
},
|
|
1540
|
+
}
|
|
1541
|
+
: {}),
|
|
889
1542
|
})),
|
|
890
1543
|
});
|
|
891
1544
|
}
|
|
@@ -1012,4 +1665,44 @@ function aborted(s) {
|
|
|
1012
1665
|
s.addEventListener("abort", () => r(), { once: true });
|
|
1013
1666
|
});
|
|
1014
1667
|
}
|
|
1668
|
+
/** The product an account belongs to. */
|
|
1669
|
+
function productOf(who) {
|
|
1670
|
+
return who.product ?? FAKE_PRODUCT;
|
|
1671
|
+
}
|
|
1672
|
+
/** A queue's limits as the selection reads them. */
|
|
1673
|
+
function queueLimits(q) {
|
|
1674
|
+
return {
|
|
1675
|
+
id: q.id,
|
|
1676
|
+
priority: q.priority || 1,
|
|
1677
|
+
paused: q.paused,
|
|
1678
|
+
maxConcurrency: q.maxConcurrency,
|
|
1679
|
+
maxConcurrencyPerTenant: q.maxConcurrencyPerTenant,
|
|
1680
|
+
...(q.rateLimit?.window ? { rateLimit: { limit: q.rateLimit.limit, windowMs: durationMs(q.rateLimit.window) } } : {}),
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
/**
|
|
1684
|
+
* A job's retry and timeout under its queue: the queue's defaults when the
|
|
1685
|
+
* job gives none, and its ceilings, past which the job is refused.
|
|
1686
|
+
*/
|
|
1687
|
+
function jobLimits(q, retry, timeoutMs) {
|
|
1688
|
+
const ceiling = q.defaultRetry;
|
|
1689
|
+
if (retry && ceiling) {
|
|
1690
|
+
if (retry.max > ceiling.max)
|
|
1691
|
+
throw new ConnectError(`retry.max ${retry.max} is above queue ${q.name}'s ${ceiling.max}`, Code.InvalidArgument);
|
|
1692
|
+
const allowed = ceiling.when.length > 0 ? ceiling.when : TRANSIENT;
|
|
1693
|
+
const outside = retry.when.filter((c) => !allowed.includes(c));
|
|
1694
|
+
if (outside.length > 0)
|
|
1695
|
+
throw new ConnectError(`retry.when lists ${outside.map((c) => FailureClass[c]).join(", ")}, which queue ${q.name} does not retry`, Code.InvalidArgument);
|
|
1696
|
+
}
|
|
1697
|
+
const maxTimeout = q.defaultTimeout ? durationMs(q.defaultTimeout) : 0;
|
|
1698
|
+
if (timeoutMs > 0 && maxTimeout > 0 && timeoutMs > maxTimeout) {
|
|
1699
|
+
throw new ConnectError(`timeout ${timeoutMs} ms is above queue ${q.name}'s ${maxTimeout} ms`, Code.InvalidArgument);
|
|
1700
|
+
}
|
|
1701
|
+
const r = retry ?? ceiling;
|
|
1702
|
+
return { retry: r ? { max: r.max, when: r.when } : undefined, timeoutMs: timeoutMs || maxTimeout };
|
|
1703
|
+
}
|
|
1704
|
+
/** A stable UUID-shaped id from a string, for the fake's platform spans. */
|
|
1705
|
+
function randomUUIDFrom(s) {
|
|
1706
|
+
return createHash("sha256").update(s).digest("hex").slice(0, 32);
|
|
1707
|
+
}
|
|
1015
1708
|
//# sourceMappingURL=index.js.map
|