@xeonr/pipelines-sdk 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +247 -24
  2. package/dist/client.d.ts +73 -3
  3. package/dist/client.d.ts.map +1 -1
  4. package/dist/client.js +48 -1
  5. package/dist/client.js.map +1 -1
  6. package/dist/errors.d.ts +42 -6
  7. package/dist/errors.d.ts.map +1 -1
  8. package/dist/errors.js +45 -2
  9. package/dist/errors.js.map +1 -1
  10. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +173 -12
  11. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -1
  12. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +45 -28
  13. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -1
  14. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +136 -17
  15. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -1
  16. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +42 -35
  17. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -1
  18. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +346 -15
  19. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -1
  20. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +105 -10
  21. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -1
  22. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +38 -6
  23. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -1
  24. package/dist/gen/xeonr/pipelines/v1/common_pb.js +37 -5
  25. package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -1
  26. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +280 -5
  27. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -1
  28. package/dist/gen/xeonr/pipelines/v1/definition_pb.js +100 -7
  29. package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -1
  30. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts +1100 -0
  31. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.d.ts.map +1 -0
  32. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js +298 -0
  33. package/dist/gen/xeonr/pipelines/v1/pipelines_pb.js.map +1 -0
  34. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +120 -1
  35. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -1
  36. package/dist/gen/xeonr/pipelines/v1/pools_pb.js +36 -11
  37. package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -1
  38. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts +769 -0
  39. package/dist/gen/xeonr/pipelines/v1/queues_pb.d.ts.map +1 -0
  40. package/dist/gen/xeonr/pipelines/v1/queues_pb.js +190 -0
  41. package/dist/gen/xeonr/pipelines/v1/queues_pb.js.map +1 -0
  42. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +60 -0
  43. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -1
  44. package/dist/gen/xeonr/pipelines/v1/registry_pb.js +21 -16
  45. package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -1
  46. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +867 -4
  47. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -1
  48. package/dist/gen/xeonr/pipelines/v1/runs_pb.js +243 -27
  49. package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -1
  50. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts +1110 -0
  51. package/dist/gen/xeonr/pipelines/v1/schedules_pb.d.ts.map +1 -0
  52. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js +295 -0
  53. package/dist/gen/xeonr/pipelines/v1/schedules_pb.js.map +1 -0
  54. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +8 -3
  55. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -1
  56. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +1 -1
  57. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -1
  58. package/dist/gen/xeonr/platform/v1/options_pb.d.ts +123 -1
  59. package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -1
  60. package/dist/gen/xeonr/platform/v1/options_pb.js +90 -1
  61. package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -1
  62. package/dist/handler.d.ts +50 -0
  63. package/dist/handler.d.ts.map +1 -1
  64. package/dist/handler.js +27 -0
  65. package/dist/handler.js.map +1 -1
  66. package/dist/index.d.ts +13 -1
  67. package/dist/index.d.ts.map +1 -1
  68. package/dist/index.js +7 -1
  69. package/dist/index.js.map +1 -1
  70. package/dist/logship.d.ts +60 -10
  71. package/dist/logship.d.ts.map +1 -1
  72. package/dist/logship.js +171 -28
  73. package/dist/logship.js.map +1 -1
  74. package/dist/queues.d.ts +171 -0
  75. package/dist/queues.d.ts.map +1 -0
  76. package/dist/queues.js +135 -0
  77. package/dist/queues.js.map +1 -0
  78. package/dist/reporter.d.ts +120 -0
  79. package/dist/reporter.d.ts.map +1 -0
  80. package/dist/reporter.js +323 -0
  81. package/dist/reporter.js.map +1 -0
  82. package/dist/schedules.d.ts +201 -0
  83. package/dist/schedules.d.ts.map +1 -0
  84. package/dist/schedules.js +127 -0
  85. package/dist/schedules.js.map +1 -0
  86. package/dist/testing/cron.d.ts +33 -0
  87. package/dist/testing/cron.d.ts.map +1 -0
  88. package/dist/testing/cron.js +177 -0
  89. package/dist/testing/cron.js.map +1 -0
  90. package/dist/testing/index.d.ts +142 -4
  91. package/dist/testing/index.d.ts.map +1 -1
  92. package/dist/testing/index.js +843 -44
  93. package/dist/testing/index.js.map +1 -1
  94. package/dist/testing/queues.d.ts +46 -0
  95. package/dist/testing/queues.d.ts.map +1 -0
  96. package/dist/testing/queues.js +70 -0
  97. package/dist/testing/queues.js.map +1 -0
  98. package/dist/testing/schedules.d.ts +113 -0
  99. package/dist/testing/schedules.d.ts.map +1 -0
  100. package/dist/testing/schedules.js +419 -0
  101. package/dist/testing/schedules.js.map +1 -0
  102. package/dist/worker.d.ts +78 -8
  103. package/dist/worker.d.ts.map +1 -1
  104. package/dist/worker.js +163 -33
  105. package/dist/worker.js.map +1 -1
  106. package/package.json +1 -1
@@ -19,17 +19,22 @@
19
19
  *
20
20
  * What it models: auth by bearer token (accounts with a tenant and scopes),
21
21
  * namespace and pool ownership, `PublishVersion`, `SetPool`, `CreateJob`
22
- * (with delay and idempotency), `CreateRun` with handler nodes and external
23
- * (image) nodes that a test settles with {@link FakePipelines.completeNode},
24
- * `${{ steps.<node>.outputs.kv.<slot> }}` references, the claim long-poll,
25
- * leases and their lapse, heartbeats with cancel, logs (idempotent on seq,
26
- * `out_of_range` for a gap), completion with the server's termination to
27
- * failure-class mapping, retries by class, cancel, and error injection.
22
+ * (with delay, idempotency, retry and timeout), `CreateRun` with handler
23
+ * nodes and external (image) nodes that a test settles with
24
+ * {@link FakePipelines.completeNode}, `${{ steps.<node>.outputs.kv.<slot> }}`
25
+ * references (only to a node in the referrer's needs closure), the claim
26
+ * long-poll, the worker id on every ref, leases and their lapse,
27
+ * `ReleaseAttempt`, heartbeats with cancel, logs (idempotent on seq,
28
+ * `out_of_range` naming the expected seq for a gap, and
29
+ * {@link FakePipelines.loseLogChunks} to play a server that lost chunks),
30
+ * completion with the server's termination to failure-class mapping,
31
+ * `handler_failure_kind`, retries by class, free interruptions on an
32
+ * `interruptible` node, cancel, and error injection.
28
33
  *
29
34
  * What it does not: secrets, blobs, tiers, routing between pools, the
30
35
  * outbox, and the platform's own deadline enforcement.
31
36
  */
32
- import { randomUUID } from "node:crypto";
37
+ import { createHash, randomUUID } from "node:crypto";
33
38
  import { create, fromJson, toJson } from "@bufbuild/protobuf";
34
39
  import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
35
40
  import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
@@ -38,8 +43,15 @@ import { authInterceptor } from "../transport.js";
38
43
  import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
39
44
  import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
40
45
  import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
41
- import { AttemptSchema, NodeRunSchema, RunSchema, RunsService } from "../gen/xeonr/pipelines/v1/runs_pb.js";
42
- import { ClaimedAttemptSchema, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
46
+ import { AttemptSchema, IssueSeverity, NodeRunSchema, RunIssueSchema, RunSchema, RunsService, RunTrigger, StageState, TraceSpanSchema, TraceSpanSource, } from "../gen/xeonr/pipelines/v1/runs_pb.js";
47
+ import { QueueSchema, QueuesService } from "../gen/xeonr/pipelines/v1/queues_pb.js";
48
+ import { pickQueued } from "./queues.js";
49
+ import { FakeClock, FakeSchedules } from "./schedules.js";
50
+ import { SchedulesService } from "../gen/xeonr/pipelines/v1/schedules_pb.js";
51
+ export { FakeClock, FakeSchedules } from "./schedules.js";
52
+ export { parseCron, nextOccurrence } from "./cron.js";
53
+ export { pickQueued } from "./queues.js";
54
+ import { ClaimedAttemptSchema, HandlerFailureKind, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
43
55
  /** The tenant the default account acts for. */
44
56
  export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
45
57
  /** The default account's token. */
@@ -53,15 +65,24 @@ export const ALL_SCOPES = [
53
65
  "pipelines:pool:read",
54
66
  "pipelines:pool:edit",
55
67
  "pipelines:worker:edit",
68
+ "pipelines:queue:read",
69
+ "pipelines:queue:edit",
70
+ "pipelines:schedule:read",
71
+ "pipelines:schedule:edit",
56
72
  ];
73
+ /** The product the default account belongs to. */
74
+ export const FAKE_PRODUCT = "00000000-0000-4000-8000-0000000000a1";
57
75
  const TRANSIENT = [
58
76
  FailureClass.ABANDONED,
59
77
  FailureClass.RUNNER_CRASHED,
60
78
  FailureClass.IMAGE_PULL,
61
79
  FailureClass.STAGING_FAILED,
80
+ FailureClass.INTERRUPTED,
62
81
  FailureClass.INTERNAL,
63
82
  ];
64
83
  const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
84
+ /** How many interruptions an interruptible node gets without spending a retry (the api's default). */
85
+ const FREE_INTERRUPTIONS = 3;
65
86
  const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
66
87
  /**
67
88
  * An in-memory pipelines-api for tests. Call {@link close} when done (it
@@ -80,6 +101,16 @@ export class FakePipelines {
80
101
  attemptsById = new Map();
81
102
  idempotency = new Map();
82
103
  injected = new Map();
104
+ /** Queues by id. */
105
+ queues = new Map();
106
+ /**
107
+ * The clock schedules run on: real time until a test calls
108
+ * `clock.set()` or `clock.advance()`, each of which fires what is due.
109
+ */
110
+ clock = new FakeClock();
111
+ /** The schedules, with their recorded occurrences. */
112
+ schedules;
113
+ queueClaims = new Map();
83
114
  timer;
84
115
  changed;
85
116
  constructor(opts = {}) {
@@ -94,8 +125,29 @@ export class FakePipelines {
94
125
  retryBackoffMs: opts.retryBackoffMs ?? 0,
95
126
  logMaxBytes: opts.logMaxBytes ?? 0,
96
127
  tickMs: opts.tickMs ?? 10,
128
+ issueCap: opts.issueCap ?? 1000,
129
+ spanCap: opts.spanCap ?? 10_000,
97
130
  };
98
131
  this.changed = signal();
132
+ this.schedules = new FakeSchedules({
133
+ createJob: (owner, spec) => this.createJobRun({ tenant: owner.tenant, scopes: ALL_SCOPES, product: owner.product }, {
134
+ tenant: owner.tenant,
135
+ handler: spec.handler,
136
+ key: `schedule:${spec.schedule.id}:${spec.schedule.target}:${spec.schedule.scheduledFor ?? randomUUID()}`,
137
+ payload: spec.payload,
138
+ delayMs: 0,
139
+ resource: spec.resource,
140
+ retry: spec.retry,
141
+ timeoutMs: spec.timeoutMs,
142
+ queue: spec.queue,
143
+ interruptible: spec.interruptible,
144
+ trigger: spec.trigger,
145
+ schedule: spec.schedule,
146
+ }),
147
+ runs: () => this.runs.values(),
148
+ runMessage: (id) => this.runMessage(this.run(id)),
149
+ checkHandler: (_product, handler) => this.requireHandler(handler),
150
+ }, this.clock, FAKE_PRODUCT);
99
151
  this.timer = setInterval(() => this.tick(), this.opts.tickMs);
100
152
  this.timer.unref();
101
153
  }
@@ -180,6 +232,15 @@ export class FakePipelines {
180
232
  }
181
233
  }
182
234
  /** Lapses an attempt's lease now, as the reaper would at its expiry. */
235
+ /**
236
+ * Forgets an attempt's log chunks from `fromSeq` on, as a server that lost
237
+ * chunks it had acknowledged would: the worker's next chunk is refused as
238
+ * ahead (`out_of_range` naming `fromSeq`).
239
+ */
240
+ loseLogChunks(attemptId, fromSeq) {
241
+ const a = this.attempt(attemptId);
242
+ a.chunks.length = Math.min(a.chunks.length, fromSeq);
243
+ }
183
244
  expireLease(attemptId) {
184
245
  this.attempt(attemptId).leaseExpiresAt = 0;
185
246
  this.tick();
@@ -279,7 +340,7 @@ export class FakePipelines {
279
340
  throw stale();
280
341
  const next = BigInt(a.chunks.length);
281
342
  if (req.seq > next)
282
- throw new ConnectError(`the next expected seq is ${next}`, Code.OutOfRange);
343
+ throw new ConnectError(`seq: the next expected seq is ${next}`, Code.OutOfRange);
283
344
  if (req.data.length < 1 || req.data.length > 1 << 20)
284
345
  throw new ConnectError("data: 1 byte to 1 MiB", Code.InvalidArgument);
285
346
  if (req.seq === next && !a.logTruncated) {
@@ -302,6 +363,9 @@ export class FakePipelines {
302
363
  throw new ConnectError("termination: required", Code.InvalidArgument);
303
364
  if (req.message.length > 2048)
304
365
  throw new ConnectError("message: at most 2048", Code.InvalidArgument);
366
+ if (req.handlerFailureKind !== HandlerFailureKind.UNSPECIFIED && req.termination !== Termination.HANDLER_ERROR) {
367
+ throw new ConnectError("handler_failure_kind is only for termination TERMINATION_HANDLER_ERROR", Code.InvalidArgument);
368
+ }
305
369
  const kv = {};
306
370
  const entries = Object.entries(req.kv);
307
371
  if (entries.length > 100)
@@ -317,29 +381,141 @@ export class FakePipelines {
317
381
  a.termination = req.termination;
318
382
  a.exitCode = req.exitCode;
319
383
  a.message = req.message;
384
+ a.handlerFailureKind = req.handlerFailureKind;
320
385
  a.kv = state === AttemptState.SUCCEEDED ? kv : {};
321
386
  a.completedAt = Date.now();
322
387
  this.settle(a);
323
388
  return { state: a.state };
324
389
  },
390
+ releaseAttempt: (req, ctx) => {
391
+ const who = this.enter("ReleaseAttempt", ctx, "pipelines:worker:edit");
392
+ if (req.reason.length > 256)
393
+ throw new ConnectError("reason: at most 256", Code.InvalidArgument);
394
+ const a = this.held(who, req.ref);
395
+ if (a.state === AttemptState.RUNNING)
396
+ return { state: a.state };
397
+ if (a.state !== AttemptState.CLAIMED)
398
+ throw stale();
399
+ // CLAIMED -> OFFERED at once: the epoch is bumped, nothing spent.
400
+ a.epoch += 1n;
401
+ a.workerId = "";
402
+ a.poolId = "";
403
+ a.leaseExpiresAt = 0;
404
+ a.releases++;
405
+ if (this.run(a.runId).cancelRequestedAt !== undefined) {
406
+ a.state = AttemptState.CANCELLED;
407
+ a.failureClass = FailureClass.CANCELLED;
408
+ a.completedAt = Date.now();
409
+ this.settle(a);
410
+ }
411
+ else {
412
+ a.state = AttemptState.OFFERED;
413
+ a.notBefore = 0;
414
+ }
415
+ this.notify();
416
+ return { state: a.state };
417
+ },
418
+ appendIssues: (req, ctx) => {
419
+ const who = this.enter("AppendIssues", ctx, "pipelines:worker:edit");
420
+ const a = this.held(who, req.ref);
421
+ if (a.state !== AttemptState.RUNNING)
422
+ throw stale();
423
+ if (req.issues.length < 1 || req.issues.length > 100)
424
+ throw new ConnectError("issues: 1 to 100", Code.InvalidArgument);
425
+ return { accepted: this.appendIssues(a, req.issues) };
426
+ },
427
+ setProgress: (req, ctx) => {
428
+ const who = this.enter("SetProgress", ctx, "pipelines:worker:edit");
429
+ const a = this.held(who, req.ref);
430
+ if (a.state !== AttemptState.RUNNING)
431
+ throw stale();
432
+ if (!(req.fraction >= 0 && req.fraction <= 1))
433
+ throw new ConnectError("fraction: 0 to 1", Code.InvalidArgument);
434
+ if (req.stageState === StageState.PENDING)
435
+ throw new ConnectError("stage_state: a stage never moves back to PENDING", Code.InvalidArgument);
436
+ if (req.stage && !/^[a-z][a-z0-9_.-]{0,63}$/.test(req.stage))
437
+ throw new ConnectError("stage: invalid key", Code.InvalidArgument);
438
+ if (req.message.length > 255)
439
+ throw new ConnectError("message: at most 255", Code.InvalidArgument);
440
+ this.setProgress(a, req);
441
+ return {};
442
+ },
443
+ reportSpans: (req, ctx) => {
444
+ const who = this.enter("ReportSpans", ctx, "pipelines:worker:edit");
445
+ const a = this.held(who, req.ref);
446
+ if (a.state !== AttemptState.RUNNING)
447
+ throw stale();
448
+ if (req.spans.length < 1 || req.spans.length > 1000)
449
+ throw new ConnectError("spans: 1 to 1000", Code.InvalidArgument);
450
+ for (const sp of req.spans) {
451
+ if (!/^[0-9a-f]{32}$/.test(sp.traceId) || !/^[0-9a-f]{16}$/.test(sp.spanId) || !sp.startedAt || !sp.endedAt || !sp.name) {
452
+ throw new ConnectError("spans: trace_id, span_id, name, started_at and ended_at are required", Code.InvalidArgument);
453
+ }
454
+ }
455
+ const room = Math.max(0, this.opts.spanCap - a.spans.length);
456
+ a.spans.push(...req.spans.slice(0, room));
457
+ return { accepted: Math.min(room, req.spans.length) };
458
+ },
325
459
  });
460
+ router.service(QueuesService, this.queuesService());
461
+ router.service(SchedulesService, this.schedules.service((method, ctx, scope) => this.enter(method, ctx, scope)));
326
462
  router.service(RunsService, {
463
+ listIssues: (req, ctx) => {
464
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
465
+ this.visible(who, req.runId);
466
+ const issues = this.issuesOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId, minSeverity: req.minSeverity });
467
+ return {
468
+ issues: issues.map((i) => create(RunIssueSchema, { ...i, detail: i.detail, createdAt: timestampFromMs(i.createdAt) })),
469
+ nextCursor: "",
470
+ };
471
+ },
472
+ getLogs: (req, ctx) => {
473
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
474
+ const a = this.attemptsById.get(req.attemptId);
475
+ if (!a)
476
+ throw new ConnectError("no such attempt", Code.NotFound);
477
+ this.visible(who, a.runId);
478
+ const all = Buffer.concat(a.chunks);
479
+ const from = Math.min(Number(req.offset), all.length);
480
+ const to = req.limitBytes ? Math.min(all.length, from + req.limitBytes) : all.length;
481
+ const complete = to === all.length && (a.logFinal || a.completedAt !== undefined);
482
+ return { data: new Uint8Array(all.subarray(from, to)), nextOffset: BigInt(to), complete, truncated: a.logTruncated };
483
+ },
484
+ getRunTrace: (req, ctx) => {
485
+ const who = this.enter("GetRun", ctx, "pipelines:run:read");
486
+ this.visible(who, req.runId);
487
+ return { spans: this.traceOf(req.runId, { nodeId: req.nodeId, attemptId: req.attemptId }), truncated: false };
488
+ },
327
489
  createJob: (req, ctx) => {
328
490
  const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
329
491
  this.actsFor(who, req.tenant);
330
- const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
492
+ const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
331
493
  if (existing)
332
494
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
333
- this.requireHandler(req.handler);
334
- const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, {}, req.delay ? durationMs(req.delay) : 0);
335
- this.addNode(run, { id: "job", handler: req.handler, uses: "", with: req.payload ?? {}, needs: [], retry: undefined, timeoutMs: 0 });
336
- this.advance(run);
495
+ const run = this.createJobRun(who, {
496
+ tenant: req.tenant,
497
+ handler: req.handler,
498
+ key: req.idempotencyKey,
499
+ payload: req.payload ?? {},
500
+ delayMs: req.delay ? durationMs(req.delay) : 0,
501
+ resource: req.resource,
502
+ retry: req.retry,
503
+ timeoutMs: req.timeout ? durationMs(req.timeout) : 0,
504
+ queue: req.queue,
505
+ interruptible: req.interruptible,
506
+ });
337
507
  return { run: this.runMessage(run), deduplicated: false };
338
508
  },
509
+ retryNode: (req, ctx) => {
510
+ const who = this.enter("RetryNode", ctx, "pipelines:run:edit");
511
+ const run = this.visible(who, req.runId);
512
+ this.retryNode(run, req.nodeId);
513
+ return { run: this.runMessage(run) };
514
+ },
339
515
  createRun: (req, ctx) => {
340
516
  const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
341
517
  this.actsFor(who, req.tenant);
342
- const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
518
+ const existing = this.idempotency.get(`${productOf(who)}\u0000${req.idempotencyKey}`);
343
519
  if (existing)
344
520
  return { run: this.runMessage(this.run(existing)), deduplicated: true };
345
521
  if (req.source.case !== "inlineDefinition")
@@ -347,14 +523,14 @@ export class FakePipelines {
347
523
  const def = req.source.value;
348
524
  if (def.nodes.length === 0)
349
525
  throw new ConnectError("definition: no nodes", Code.InvalidArgument);
526
+ const specs = def.nodes.map((n) => nodeSpec(n, (h) => this.requireHandler(h)));
527
+ for (const sp of specs)
528
+ if (sp.handler)
529
+ sp.with = this.checkInputs(sp.id, sp.handler, sp.with);
530
+ checkGraph(specs);
350
531
  const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, req.variables ?? {}, 0);
351
- for (const n of def.nodes)
352
- this.addNode(run, nodeSpec(n, (h) => this.requireHandler(h)));
353
- for (const n of run.nodes.values()) {
354
- for (const need of n.needs)
355
- if (!run.nodes.has(need))
356
- throw new ConnectError(`node ${n.id}: unknown need ${need}`, Code.InvalidArgument);
357
- }
532
+ for (const spec of specs)
533
+ this.addNode(run, spec);
358
534
  this.advance(run);
359
535
  return { run: this.runMessage(run), deduplicated: false };
360
536
  },
@@ -465,6 +641,9 @@ export class FakePipelines {
465
641
  const a = this.attemptsById.get(ref.attemptId);
466
642
  if (!a || a.epoch !== ref.claimEpoch || !a.poolId)
467
643
  throw stale();
644
+ // Claimed by handler: the ref must name the worker recorded at the claim.
645
+ if (!ref.workerId || ref.workerId !== a.workerId)
646
+ throw stale();
468
647
  if (this.pools.get(a.poolId)?.owner?.tenant !== who.tenant)
469
648
  throw stale();
470
649
  return a;
@@ -475,13 +654,39 @@ export class FakePipelines {
475
654
  throw new ConnectError(`handler ${handler} is not a published worker package`, Code.InvalidArgument);
476
655
  }
477
656
  }
657
+ /**
658
+ * Holds a handler node's `with` to its manifest, as the api does when a run
659
+ * is created: every key must be a declared input, every required input
660
+ * given; defaults fill the rest. Returns the inputs with defaults.
661
+ */
662
+ checkInputs(nodeId, handler, given) {
663
+ const pkg = this.packages.get(handler);
664
+ const version = pkg ? [...pkg.versions.keys()].at(-1) : "";
665
+ const manifest = pkg?.versions.get(version);
666
+ const inputs = manifest?.inputs ?? [];
667
+ for (const k of Object.keys(given)) {
668
+ if (!inputs.some((i) => i.name === k)) {
669
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": input "${k}" is not an input of ${handler}@${version}`, Code.InvalidArgument);
670
+ }
671
+ }
672
+ const out = { ...given };
673
+ for (const i of inputs) {
674
+ if (out[i.name] !== undefined)
675
+ continue;
676
+ if (i.default)
677
+ out[i.name] = toJson(ValueSchema, i.default);
678
+ else if (i.required)
679
+ throw new ConnectError(`the definition is invalid: node "${nodeId}": required input "${i.name}" is not given`, Code.InvalidArgument);
680
+ }
681
+ return out;
682
+ }
478
683
  // ---- the scheduler ----------------------------------------------------
479
684
  newRun(who, tenant, key, resource, variables, delayMs) {
480
685
  const now = Date.now();
481
686
  const run = {
482
687
  id: randomUUID(),
483
688
  tenant,
484
- product: who.product ?? who.tenant,
689
+ product: productOf(who),
485
690
  idempotencyKey: key,
486
691
  createdBy: `urn:fake:sa:${who.tenant}`,
487
692
  resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
@@ -496,19 +701,20 @@ export class FakePipelines {
496
701
  cancelRequestedAt: undefined,
497
702
  cancelReason: "",
498
703
  nodes: new Map(),
704
+ queueId: "",
705
+ queue: "",
706
+ trigger: RunTrigger.API,
707
+ scheduleId: "",
708
+ scheduledFor: undefined,
709
+ scheduleTarget: "",
710
+ issues: [],
711
+ issuesTruncated: false,
499
712
  };
500
713
  this.runs.set(run.id, run);
501
714
  this.idempotency.set(`${run.product}\u0000${key}`, run.id);
502
715
  return run;
503
716
  }
504
717
  addNode(run, spec) {
505
- const refs = new Set(spec.needs);
506
- for (const v of Object.values(spec.with)) {
507
- const m = typeof v === "string" ? REF.exec(v) : null;
508
- const node = m?.[1]?.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
509
- if (node)
510
- refs.add(node);
511
- }
512
718
  run.nodes.set(spec.id, {
513
719
  id: spec.id,
514
720
  handler: spec.handler,
@@ -517,10 +723,11 @@ export class FakePipelines {
517
723
  failureClass: FailureClass.UNSPECIFIED,
518
724
  kv: {},
519
725
  attempts: [],
520
- needs: [...refs],
726
+ needs: spec.needs,
521
727
  with: spec.with,
522
728
  retry: spec.retry ?? { max: this.opts.defaultRetries, when: [] },
523
729
  timeoutMs: spec.timeoutMs || this.opts.defaultTimeoutMs,
730
+ interruptible: spec.interruptible,
524
731
  });
525
732
  }
526
733
  /** Makes ready what can be, skips what cannot run, and settles the run. */
@@ -577,6 +784,7 @@ export class FakePipelines {
577
784
  else {
578
785
  run.state = RunState.SUCCEEDED;
579
786
  }
787
+ this.runFinished(run);
580
788
  }
581
789
  }
582
790
  resolve(run, n) {
@@ -625,14 +833,25 @@ export class FakePipelines {
625
833
  logFinal: false,
626
834
  logTruncated: false,
627
835
  completedAt: undefined,
836
+ handlerFailureKind: HandlerFailureKind.UNSPECIFIED,
837
+ releases: 0,
838
+ offeredAt: Date.now(),
839
+ progress: undefined,
840
+ spans: [],
628
841
  };
629
842
  n.attempts.push(a);
630
843
  this.attemptsById.set(a.id, a);
631
844
  this.notify();
632
845
  }
846
+ /**
847
+ * Claims for a worker: the attempts of handler nodes of full runs first,
848
+ * in creation order, as they always were; then jobs, through their
849
+ * queues' limits and fairness rule (see ./queues.ts).
850
+ */
633
851
  claim(pool, handlers, workerId, max) {
634
852
  const now = Date.now();
635
853
  const out = [];
854
+ const queued = [];
636
855
  for (const run of this.runs.values()) {
637
856
  if (run.tenant !== pool.owner?.tenant || run.cancelRequestedAt !== undefined || run.notBefore > now)
638
857
  continue;
@@ -640,28 +859,55 @@ export class FakePipelines {
640
859
  if (!handlers.includes(n.handler))
641
860
  continue;
642
861
  for (const a of n.attempts) {
643
- if (out.length >= max)
644
- return out;
645
862
  if (a.state !== AttemptState.OFFERED || a.notBefore > now)
646
863
  continue;
647
- a.state = AttemptState.CLAIMED;
648
- a.epoch += 1n;
649
- a.workerId = workerId;
650
- a.poolId = pool.id;
651
- a.leaseExpiresAt = now + this.opts.leaseMs;
652
- out.push(a);
864
+ if (run.queueId)
865
+ queued.push({ item: a, queueId: run.queueId, tenant: run.tenant, waitingSince: Math.max(a.offeredAt, a.notBefore, run.notBefore) });
866
+ else if (out.length < max)
867
+ out.push(a);
653
868
  }
654
869
  }
655
870
  }
871
+ if (out.length < max && queued.length > 0) {
872
+ const limits = new Map();
873
+ for (const q of this.queues.values())
874
+ limits.set(q.id, queueLimits(q));
875
+ const picks = pickQueued(queued, limits, { running: (q, t) => this.queueRunning(q, t), claimedSince: (q, since) => (this.queueClaims.get(q) ?? []).filter((x) => x >= since).length }, now, max - out.length);
876
+ for (const p of picks) {
877
+ out.push(p.item);
878
+ this.queueClaims.set(p.queueId, [...(this.queueClaims.get(p.queueId) ?? []).filter((x) => x > now - 86_400_000), now]);
879
+ }
880
+ }
881
+ for (const a of out) {
882
+ a.state = AttemptState.CLAIMED;
883
+ a.epoch += 1n;
884
+ a.workerId = workerId;
885
+ a.poolId = pool.id;
886
+ a.leaseExpiresAt = now + this.opts.leaseMs;
887
+ this.initProgress(a);
888
+ }
656
889
  return out;
657
890
  }
891
+ /** CLAIMED and RUNNING attempts of a queue's jobs, optionally of one tenant. */
892
+ queueRunning(queueId, tenant) {
893
+ let n = 0;
894
+ for (const run of this.runs.values()) {
895
+ if (run.queueId !== queueId || (tenant !== undefined && run.tenant !== tenant))
896
+ continue;
897
+ for (const node of run.nodes.values())
898
+ for (const a of node.attempts)
899
+ if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING)
900
+ n++;
901
+ }
902
+ return n;
903
+ }
658
904
  describe(a) {
659
905
  const run = this.run(a.runId);
660
906
  const n = run.nodes.get(a.nodeId);
661
907
  const pkg = this.packages.get(n.handler);
662
908
  const version = pkg ? [...pkg.versions.keys()].at(-1) : "0.0.0";
663
909
  return create(ClaimedAttemptSchema, {
664
- ref: { attemptId: a.id, claimEpoch: a.epoch },
910
+ ref: { attemptId: a.id, claimEpoch: a.epoch, workerId: a.workerId },
665
911
  runId: run.id,
666
912
  nodeId: n.id,
667
913
  number: a.number,
@@ -705,9 +951,23 @@ export class FakePipelines {
705
951
  n.failureClass = FailureClass.CANCELLED;
706
952
  }
707
953
  else {
708
- const when = n.retry.when.length > 0 ? n.retry.when : TRANSIENT;
709
- const retries = n.attempts.length - 1;
710
- if (when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
954
+ const permanent = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.PERMANENT;
955
+ const transientHandler = a.failureClass === FailureClass.HANDLER_ERROR && a.handlerFailureKind === HandlerFailureKind.TRANSIENT;
956
+ const when = n.retry.when.length > 0 ? n.retry.when : transientHandler ? [...TRANSIENT, FailureClass.HANDLER_ERROR] : TRANSIENT;
957
+ // On an interruptible node an interruption is free, up to a bound.
958
+ const free = (x) => n.interruptible && x.failureClass === FailureClass.INTERRUPTED;
959
+ const freeUsed = n.attempts.filter(free).length;
960
+ if (free(a) && freeUsed <= FREE_INTERRUPTIONS) {
961
+ // Up to the bound, an interruption is offered again at once.
962
+ this.offer(run, n, a.inputs, 0);
963
+ return;
964
+ }
965
+ const retries = n.attempts.slice(0, -1).filter((x) => !free(x)).length;
966
+ // As the api: a worker that vanished (ABANDONED) may have had outside
967
+ // effects, so its node runs again automatically only when it declares
968
+ // itself interruptible.
969
+ const abandonedOk = a.failureClass !== FailureClass.ABANDONED || n.interruptible;
970
+ if (!permanent && abandonedOk && when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
711
971
  this.offer(run, n, a.inputs, this.opts.retryBackoffMs);
712
972
  return;
713
973
  }
@@ -769,6 +1029,7 @@ export class FakePipelines {
769
1029
  }
770
1030
  }
771
1031
  }
1032
+ this.schedules.evaluate();
772
1033
  this.notify();
773
1034
  }
774
1035
  notify() {
@@ -776,6 +1037,430 @@ export class FakePipelines {
776
1037
  this.changed = signal();
777
1038
  prev.resolve();
778
1039
  }
1040
+ /** QueuesService, as the contract states it. */
1041
+ queuesService() {
1042
+ const readable = (who, q) => {
1043
+ if (!q || q.owner?.product !== productOf(who) || (q.owner.tenant && q.owner.tenant !== who.tenant))
1044
+ throw new ConnectError("no such queue", Code.NotFound);
1045
+ return q;
1046
+ };
1047
+ const byId = (who, id) => readable(who, this.queues.get(id));
1048
+ return {
1049
+ setQueue: (req, ctx) => {
1050
+ const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
1051
+ const q = req.queue;
1052
+ if (!q?.owner)
1053
+ throw new ConnectError("queue.owner: required", Code.InvalidArgument);
1054
+ if (q.owner.product !== productOf(who))
1055
+ throw new ConnectError("queue.owner.product: a product sets only its own queues", Code.PermissionDenied);
1056
+ if (q.owner.tenant && q.owner.tenant !== who.tenant)
1057
+ throw new ConnectError("queue.owner.tenant: the caller does not act for it", Code.PermissionDenied);
1058
+ if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(q.name))
1059
+ throw new ConnectError("queue.name: invalid", Code.InvalidArgument);
1060
+ if (q.maxConcurrency > 0 && q.maxConcurrencyPerTenant > q.maxConcurrency) {
1061
+ throw new ConnectError("max_concurrency_per_tenant must not exceed a non-zero max_concurrency", Code.InvalidArgument);
1062
+ }
1063
+ if (req.updateMask)
1064
+ throw new ConnectError("update_mask: the fake replaces whole queues only", Code.Unimplemented);
1065
+ const existing = [...this.queues.values()].find((x) => x.owner?.product === q.owner.product && x.owner.tenant === q.owner.tenant && x.name === q.name);
1066
+ const now = Date.now();
1067
+ const next = create(QueueSchema, {
1068
+ ...q,
1069
+ id: existing?.id ?? randomUUID(),
1070
+ paused: existing?.paused ?? false,
1071
+ pausedReason: existing?.pausedReason ?? "",
1072
+ pausedBy: existing?.pausedBy ?? "",
1073
+ ...(existing?.pausedAt ? { pausedAt: existing.pausedAt } : {}),
1074
+ createdAt: existing?.createdAt ?? timestampFromMs(now),
1075
+ updatedAt: timestampFromMs(now),
1076
+ });
1077
+ this.queues.set(next.id, next);
1078
+ this.notify();
1079
+ return { queue: next, created: !existing };
1080
+ },
1081
+ getQueue: (req, ctx) => {
1082
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1083
+ if (req.queue.case === "queueId")
1084
+ return { queue: byId(who, req.queue.value) };
1085
+ if (req.queue.case !== "byName")
1086
+ throw new ConnectError("queue: required", Code.InvalidArgument);
1087
+ const { owner, name } = req.queue.value;
1088
+ if (owner?.product !== productOf(who))
1089
+ throw new ConnectError("no such queue", Code.NotFound);
1090
+ const found = [...this.queues.values()].find((x) => x.owner?.product === owner.product && x.owner.tenant === owner.tenant && x.name === name);
1091
+ if (found)
1092
+ return { queue: readable(who, found) };
1093
+ if (name === "default" && !owner.tenant)
1094
+ return { queue: this.defaultQueue(owner.product) };
1095
+ throw new ConnectError("no such queue", Code.NotFound);
1096
+ },
1097
+ listQueues: (req, ctx) => {
1098
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1099
+ const product = req.product || productOf(who);
1100
+ if (product !== productOf(who))
1101
+ return { queues: [], nextCursor: "" };
1102
+ const queues = [...this.queues.values()]
1103
+ .filter((q) => q.owner?.product === product && (!req.tenant || q.owner.tenant === req.tenant) && (!q.owner.tenant || q.owner.tenant === who.tenant))
1104
+ .sort((a, b) => (a.priority || 1) - (b.priority || 1) || a.name.localeCompare(b.name));
1105
+ return { queues, nextCursor: "" };
1106
+ },
1107
+ markQueuePaused: (req, ctx) => {
1108
+ const who = this.enter("MarkQueuePaused", ctx, "pipelines:queue:edit");
1109
+ const q = byId(who, req.queueId);
1110
+ if (!q.paused) {
1111
+ q.paused = true;
1112
+ q.pausedAt = timestampFromMs(Date.now());
1113
+ q.pausedBy = `urn:fake:sa:${who.tenant}`;
1114
+ q.pausedReason = req.reason;
1115
+ }
1116
+ return { queue: q };
1117
+ },
1118
+ markQueueResumed: (req, ctx) => {
1119
+ const who = this.enter("MarkQueueResumed", ctx, "pipelines:queue:edit");
1120
+ const q = byId(who, req.queueId);
1121
+ q.paused = false;
1122
+ q.pausedReason = "";
1123
+ q.pausedBy = "";
1124
+ delete q.pausedAt;
1125
+ this.notify();
1126
+ return { queue: q };
1127
+ },
1128
+ deleteQueue: (req, ctx) => {
1129
+ const who = this.enter("SetQueue", ctx, "pipelines:queue:edit");
1130
+ const q = byId(who, req.queueId);
1131
+ const waiting = [...this.runs.values()].filter((r) => r.queueId === q.id && (r.state === RunState.QUEUED || (r.state === RunState.RUNNING && this.attemptsOf(r.id).some((a) => a.state === AttemptState.OFFERED))));
1132
+ if (waiting.length > 0 && !req.cancelWaiting)
1133
+ throw new ConnectError(`queue ${q.name} has ${waiting.length} waiting jobs`, Code.FailedPrecondition);
1134
+ for (const r of waiting)
1135
+ this.cancel(r, "queue deleted");
1136
+ if (q.name === "default" && !q.owner?.tenant) {
1137
+ this.queues.set(q.id, create(QueueSchema, { id: q.id, owner: q.owner, name: "default", createdAt: q.createdAt, updatedAt: timestampFromMs(Date.now()) }));
1138
+ }
1139
+ else {
1140
+ this.queues.delete(q.id);
1141
+ }
1142
+ return { cancelledJobs: waiting.length };
1143
+ },
1144
+ getQueueStats: (req, ctx) => {
1145
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1146
+ const q = byId(who, req.queueId);
1147
+ const now = Date.now();
1148
+ const windowMs = req.window ? durationMs(req.window) : 3_600_000;
1149
+ const runs = [...this.runs.values()].filter((r) => r.queueId === q.id);
1150
+ let waiting = 0;
1151
+ let delayed = 0;
1152
+ let running = 0;
1153
+ let started = 0n;
1154
+ let oldest;
1155
+ const tenants = new Map();
1156
+ for (const r of runs) {
1157
+ const t = tenants.get(r.tenant) ?? { waiting: 0, running: 0 };
1158
+ for (const a of this.attemptsOf(r.id)) {
1159
+ if (a.state === AttemptState.OFFERED) {
1160
+ if (a.notBefore > now || r.notBefore > now)
1161
+ delayed++;
1162
+ else {
1163
+ waiting++;
1164
+ t.waiting++;
1165
+ const since = Math.max(a.offeredAt, a.notBefore, r.notBefore);
1166
+ oldest = oldest === undefined ? since : Math.min(oldest, since);
1167
+ }
1168
+ }
1169
+ if (a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING) {
1170
+ running++;
1171
+ t.running++;
1172
+ }
1173
+ if (a.startedAt && a.startedAt >= now - windowMs)
1174
+ started++;
1175
+ }
1176
+ if (r.state === RunState.QUEUED && r.notBefore > now && this.attemptsOf(r.id).length === 0)
1177
+ delayed++;
1178
+ tenants.set(r.tenant, t);
1179
+ }
1180
+ const failed = BigInt(runs.filter((r) => r.state === RunState.FAILED && (r.finishedAt ?? 0) >= now - windowMs).length);
1181
+ return {
1182
+ waiting,
1183
+ delayed,
1184
+ running,
1185
+ ...(oldest !== undefined ? { oldestWaitingAt: timestampFromMs(oldest), oldestWaitingAge: durationFromMs(now - oldest) } : {}),
1186
+ tenants: [...tenants.entries()]
1187
+ .map(([tenant, v]) => ({ tenant, ...v }))
1188
+ .sort((a, b) => b.waiting - a.waiting || b.running - a.running)
1189
+ .slice(0, req.topTenants || 10),
1190
+ started,
1191
+ failed,
1192
+ paused: q.paused,
1193
+ at: timestampFromMs(now),
1194
+ };
1195
+ },
1196
+ listQueueJobs: (req, ctx) => {
1197
+ const who = this.enter("GetQueue", ctx, "pipelines:queue:read");
1198
+ const q = byId(who, req.queueId);
1199
+ const runs = [...this.runs.values()]
1200
+ .filter((r) => r.queueId === q.id && (!req.tenant || r.tenant === req.tenant) && (req.states.length === 0 || req.states.includes(r.state)))
1201
+ .reverse();
1202
+ return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
1203
+ },
1204
+ };
1205
+ }
1206
+ // ---- jobs and queues --------------------------------------------------
1207
+ /** CreateJob's work, shared with schedules: resolve the queue, apply its limits, make the one-node run. */
1208
+ createJobRun(who, spec) {
1209
+ this.requireHandler(spec.handler);
1210
+ if (spec.retry && spec.retry.max > 10)
1211
+ throw new ConnectError("retry.max: at most 10", Code.InvalidArgument);
1212
+ if (spec.retry?.when.includes(FailureClass.CANCELLED))
1213
+ throw new ConnectError("retry.when: CANCELLED is never retried", Code.InvalidArgument);
1214
+ const q = this.resolveQueue(productOf(who), spec.tenant, spec.queue);
1215
+ if (!q)
1216
+ throw new ConnectError(`queue ${JSON.stringify(spec.queue)} not found among this product's queues`, Code.NotFound);
1217
+ const { retry, timeoutMs } = jobLimits(q, spec.retry, spec.timeoutMs);
1218
+ const payload = this.checkInputs("job", spec.handler, spec.payload);
1219
+ const run = this.newRun(who, spec.tenant, spec.key, spec.resource, {}, spec.delayMs);
1220
+ run.queueId = q.id;
1221
+ run.queue = q.name;
1222
+ run.trigger = spec.trigger ?? RunTrigger.API;
1223
+ if (spec.schedule) {
1224
+ run.scheduleId = spec.schedule.id;
1225
+ run.scheduledFor = spec.schedule.scheduledFor;
1226
+ run.scheduleTarget = spec.schedule.target;
1227
+ }
1228
+ this.addNode(run, {
1229
+ id: "job",
1230
+ handler: spec.handler,
1231
+ uses: "",
1232
+ with: payload,
1233
+ needs: [],
1234
+ retry,
1235
+ timeoutMs,
1236
+ interruptible: spec.interruptible,
1237
+ });
1238
+ this.advance(run);
1239
+ return run;
1240
+ }
1241
+ /** A job's queue: the tenant's own of that name, else the product-wide one; empty is `default`. */
1242
+ resolveQueue(product, tenant, name) {
1243
+ const want = name || "default";
1244
+ const all = [...this.queues.values()].filter((q) => q.owner?.product === product && q.name === want);
1245
+ const found = all.find((q) => q.owner?.tenant === tenant) ?? all.find((q) => !q.owner?.tenant);
1246
+ if (found)
1247
+ return found;
1248
+ return want === "default" ? this.defaultQueue(product) : undefined;
1249
+ }
1250
+ /** The product's `default` queue, made on first use with no limits. */
1251
+ defaultQueue(product) {
1252
+ const existing = [...this.queues.values()].find((q) => q.owner?.product === product && !q.owner.tenant && q.name === "default");
1253
+ if (existing)
1254
+ return existing;
1255
+ const now = Date.now();
1256
+ const q = create(QueueSchema, { id: randomUUID(), owner: { product, tenant: "" }, name: "default", createdAt: timestampFromMs(now), updatedAt: timestampFromMs(now) });
1257
+ this.queues.set(q.id, q);
1258
+ return q;
1259
+ }
1260
+ /** RetryNode: reopens a finished run and offers the node again; on a job, a requeue. */
1261
+ retryNode(run, nodeId) {
1262
+ const n = run.nodes.get(nodeId);
1263
+ if (!n)
1264
+ throw new ConnectError(`no node ${nodeId}`, Code.NotFound);
1265
+ if (!terminal(run.state))
1266
+ throw new ConnectError("the run is not terminal", Code.FailedPrecondition);
1267
+ if (!n.handler || n.attempts.length === 0 || !terminalNode(n.state))
1268
+ throw new ConnectError("the node never settled", Code.FailedPrecondition);
1269
+ run.state = RunState.RUNNING;
1270
+ run.finishedAt = undefined;
1271
+ run.failureClass = FailureClass.UNSPECIFIED;
1272
+ run.failedNodeId = "";
1273
+ run.cancelRequestedAt = undefined;
1274
+ // The node and everything downstream of it run again.
1275
+ const reset = new Set([nodeId]);
1276
+ for (let grew = true; grew;) {
1277
+ grew = false;
1278
+ for (const x of run.nodes.values()) {
1279
+ if (!reset.has(x.id) && x.needs.some((d) => reset.has(d))) {
1280
+ reset.add(x.id);
1281
+ grew = true;
1282
+ }
1283
+ }
1284
+ }
1285
+ for (const id of reset) {
1286
+ const x = run.nodes.get(id);
1287
+ x.state = id === nodeId ? NodeState.READY : NodeState.PENDING;
1288
+ x.failureClass = FailureClass.UNSPECIFIED;
1289
+ x.kv = {};
1290
+ }
1291
+ const inputs = this.resolve(run, n);
1292
+ this.offer(run, n, typeof inputs === "string" ? {} : inputs, 0);
1293
+ }
1294
+ // ---- issues, progress and spans ---------------------------------------
1295
+ /** Every declared stage PENDING from the claim, as the platform does. */
1296
+ initProgress(a) {
1297
+ if (a.progress)
1298
+ return;
1299
+ const n = this.run(a.runId).nodes.get(a.nodeId);
1300
+ const pkg = this.packages.get(n.handler);
1301
+ const manifest = pkg ? [...pkg.versions.values()].at(-1) : undefined;
1302
+ const stages = manifest?.stages ?? [];
1303
+ if (stages.length === 0)
1304
+ return;
1305
+ a.progress = {
1306
+ fraction: 0,
1307
+ message: "",
1308
+ processed: 0n,
1309
+ total: 0n,
1310
+ failed: 0n,
1311
+ updatedAt: Date.now(),
1312
+ reports: 0,
1313
+ stages: stages.map((st) => ({ key: st.key, name: st.name, weight: st.weight || 1, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n })),
1314
+ };
1315
+ }
1316
+ setProgress(a, req) {
1317
+ const now = Date.now();
1318
+ const p = (a.progress ??= { fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n, stages: [], updatedAt: now, reports: 0 });
1319
+ p.reports++;
1320
+ p.updatedAt = now;
1321
+ p.message = req.message;
1322
+ p.processed = req.processed;
1323
+ p.total = req.total;
1324
+ p.failed = req.failed;
1325
+ if (!req.stage) {
1326
+ p.fraction = Math.max(p.fraction, req.fraction);
1327
+ return;
1328
+ }
1329
+ let st = p.stages.find((x) => x.key === req.stage);
1330
+ if (!st) {
1331
+ st = { key: req.stage, name: req.stageName || req.stage, weight: 0, state: StageState.PENDING, fraction: 0, message: "", processed: 0n, total: 0n, failed: 0n };
1332
+ p.stages.push(st);
1333
+ }
1334
+ // Reporting a stage finishes the one before it.
1335
+ for (const other of p.stages) {
1336
+ if (other !== st && other.state === StageState.RUNNING) {
1337
+ other.state = StageState.SUCCEEDED;
1338
+ other.fraction = 1;
1339
+ other.finishedAt = now;
1340
+ }
1341
+ }
1342
+ const state = req.stageState === StageState.UNSPECIFIED ? StageState.RUNNING : req.stageState;
1343
+ if (st.state === StageState.PENDING || st.state === StageState.RUNNING) {
1344
+ st.state = state;
1345
+ st.startedAt ??= now;
1346
+ if (state !== StageState.RUNNING)
1347
+ st.finishedAt = now;
1348
+ }
1349
+ st.fraction = state === StageState.SUCCEEDED ? 1 : req.fraction;
1350
+ st.message = req.message;
1351
+ st.processed = req.processed;
1352
+ st.total = req.total;
1353
+ st.failed = req.failed;
1354
+ const total = p.stages.reduce((s, x) => s + x.weight, 0);
1355
+ if (total > 0) {
1356
+ const done = p.stages.reduce((s, x) => s + (x.state === StageState.SUCCEEDED || x.state === StageState.FAILED || x.state === StageState.SKIPPED ? x.weight : x.state === StageState.RUNNING ? x.weight * x.fraction : 0), 0);
1357
+ p.fraction = Math.max(p.fraction, Math.min(1, done / total));
1358
+ }
1359
+ }
1360
+ appendIssues(a, issues) {
1361
+ const run = this.run(a.runId);
1362
+ let accepted = 0;
1363
+ for (const i of issues) {
1364
+ if (!i.message)
1365
+ throw new ConnectError("issues.message: required", Code.InvalidArgument);
1366
+ if (i.severity === 0)
1367
+ throw new ConnectError("issues.severity: required", Code.InvalidArgument);
1368
+ }
1369
+ for (const i of issues) {
1370
+ if (run.issues.length >= this.opts.issueCap) {
1371
+ run.issuesTruncated = true;
1372
+ continue;
1373
+ }
1374
+ run.issues.push({
1375
+ id: randomUUID(),
1376
+ runId: run.id,
1377
+ nodeId: a.nodeId,
1378
+ attemptId: a.id,
1379
+ stage: i.stage,
1380
+ severity: i.severity,
1381
+ subject: i.subject || i.subjectId,
1382
+ subjectKind: i.subjectKind,
1383
+ subjectId: i.subjectId,
1384
+ message: i.message,
1385
+ detail: i.detail,
1386
+ createdAt: Date.now(),
1387
+ });
1388
+ accepted++;
1389
+ }
1390
+ return accepted;
1391
+ }
1392
+ /** Issues of a run, filtered as ListIssues filters them. */
1393
+ issuesOf(runId, filter = {}) {
1394
+ return this.run(runId).issues.filter((i) => (!filter.nodeId || i.nodeId === filter.nodeId) && (!filter.attemptId || i.attemptId === filter.attemptId) && (!filter.minSeverity || i.severity >= filter.minSeverity));
1395
+ }
1396
+ /** The run's trace: the platform's spans for the run, its nodes and attempts, and the step spans reported. */
1397
+ traceOf(runId, filter = {}) {
1398
+ const run = this.run(runId);
1399
+ const traceId = run.id.replaceAll("-", "");
1400
+ const sid = (s) => s.replaceAll("-", "").slice(0, 16);
1401
+ const out = [
1402
+ create(TraceSpanSchema, {
1403
+ traceId,
1404
+ spanId: sid(run.id),
1405
+ name: `run ${run.id}`,
1406
+ source: TraceSpanSource.PLATFORM,
1407
+ startedAt: timestampFromMs(run.createdAt),
1408
+ ...(run.finishedAt ? { endedAt: timestampFromMs(run.finishedAt) } : {}),
1409
+ }),
1410
+ ];
1411
+ for (const n of run.nodes.values()) {
1412
+ if (filter.nodeId && n.id !== filter.nodeId)
1413
+ continue;
1414
+ const nodeSpan = sid(randomUUIDFrom(run.id + n.id));
1415
+ const first = n.attempts[0];
1416
+ out.push(create(TraceSpanSchema, {
1417
+ traceId,
1418
+ spanId: nodeSpan,
1419
+ parentSpanId: sid(run.id),
1420
+ name: `node ${n.id}`,
1421
+ source: TraceSpanSource.PLATFORM,
1422
+ nodeId: n.id,
1423
+ startedAt: timestampFromMs(first?.offeredAt ?? run.createdAt),
1424
+ }));
1425
+ for (const a of n.attempts) {
1426
+ if (filter.attemptId && a.id !== filter.attemptId)
1427
+ continue;
1428
+ out.push(create(TraceSpanSchema, {
1429
+ traceId,
1430
+ spanId: sid(a.id),
1431
+ parentSpanId: nodeSpan,
1432
+ name: `attempt ${a.number}`,
1433
+ source: TraceSpanSource.PLATFORM,
1434
+ nodeId: n.id,
1435
+ attemptId: a.id,
1436
+ startedAt: timestampFromMs(a.offeredAt),
1437
+ ...(a.completedAt ? { endedAt: timestampFromMs(a.completedAt) } : {}),
1438
+ }));
1439
+ for (const sp of a.spans) {
1440
+ out.push(create(TraceSpanSchema, {
1441
+ traceId: sp.traceId,
1442
+ spanId: sp.spanId,
1443
+ parentSpanId: sp.parentSpanId || sid(a.id),
1444
+ name: sp.name,
1445
+ source: TraceSpanSource.STEP,
1446
+ nodeId: n.id,
1447
+ attemptId: a.id,
1448
+ ...(sp.startedAt ? { startedAt: sp.startedAt } : {}),
1449
+ ...(sp.endedAt ? { endedAt: sp.endedAt } : {}),
1450
+ status: sp.status,
1451
+ statusMessage: sp.statusMessage,
1452
+ ...(sp.attributes ? { attributes: sp.attributes } : {}),
1453
+ }));
1454
+ }
1455
+ }
1456
+ }
1457
+ return out.sort((x, y) => Number(x.startedAt?.seconds ?? 0n) - Number(y.startedAt?.seconds ?? 0n));
1458
+ }
1459
+ /** Runs whenever a run becomes terminal. */
1460
+ runFinished(run) {
1461
+ // The schedules are made after the router; a run cannot finish before.
1462
+ this.schedules?.runFinished(run);
1463
+ }
779
1464
  // ---- messages ---------------------------------------------------------
780
1465
  runMessage(r) {
781
1466
  return create(RunSchema, {
@@ -795,6 +1480,16 @@ export class FakePipelines {
795
1480
  ...(r.finishedAt ? { finishedAt: timestampFromMs(r.finishedAt) } : {}),
796
1481
  ...(r.cancelRequestedAt ? { cancelRequestedAt: timestampFromMs(r.cancelRequestedAt) } : {}),
797
1482
  cancelReason: r.cancelReason,
1483
+ queueId: r.queueId,
1484
+ queue: r.queue,
1485
+ trigger: r.trigger,
1486
+ scheduleId: r.scheduleId,
1487
+ ...(r.scheduledFor !== undefined ? { scheduledFor: timestampFromMs(r.scheduledFor) } : {}),
1488
+ scheduleTarget: r.scheduleTarget,
1489
+ issueInfoCount: r.issues.filter((i) => i.severity === IssueSeverity.INFO).length,
1490
+ issueWarningCount: r.issues.filter((i) => i.severity === IssueSeverity.WARNING).length,
1491
+ issueErrorCount: r.issues.filter((i) => i.severity === IssueSeverity.ERROR).length,
1492
+ issuesTruncated: r.issuesTruncated,
798
1493
  });
799
1494
  }
800
1495
  nodeMessage(n) {
@@ -819,10 +1514,71 @@ export class FakePipelines {
819
1514
  message: a.message,
820
1515
  logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
821
1516
  logTruncated: a.logTruncated,
1517
+ ...(a.progress
1518
+ ? {
1519
+ progress: {
1520
+ fraction: a.progress.fraction,
1521
+ message: a.progress.message,
1522
+ processed: a.progress.processed,
1523
+ total: a.progress.total,
1524
+ failed: a.progress.failed,
1525
+ updatedAt: timestampFromMs(a.progress.updatedAt),
1526
+ stages: a.progress.stages.map((st) => ({
1527
+ key: st.key,
1528
+ name: st.name,
1529
+ weight: st.weight,
1530
+ state: st.state,
1531
+ fraction: st.fraction,
1532
+ message: st.message,
1533
+ processed: st.processed,
1534
+ total: st.total,
1535
+ failed: st.failed,
1536
+ ...(st.startedAt ? { startedAt: timestampFromMs(st.startedAt) } : {}),
1537
+ ...(st.finishedAt ? { finishedAt: timestampFromMs(st.finishedAt) } : {}),
1538
+ })),
1539
+ },
1540
+ }
1541
+ : {}),
822
1542
  })),
823
1543
  });
824
1544
  }
825
1545
  }
1546
+ /**
1547
+ * Holds a definition to the server's rules the fake models: unique ids,
1548
+ * known needs, no cycle, and a reference names only a node in the referring
1549
+ * node's dependency closure (a reference adds no edge).
1550
+ */
1551
+ function checkGraph(specs) {
1552
+ const byId = new Map();
1553
+ for (const s of specs) {
1554
+ if (byId.has(s.id))
1555
+ throw new ConnectError(`node ${s.id}: duplicate id`, Code.InvalidArgument);
1556
+ byId.set(s.id, s);
1557
+ }
1558
+ const closure = (id, seen = new Set(), path = new Set()) => {
1559
+ for (const need of byId.get(id).needs) {
1560
+ if (!byId.has(need))
1561
+ throw new ConnectError(`node ${id}: unknown need ${need}`, Code.InvalidArgument);
1562
+ if (path.has(need) || need === id)
1563
+ throw new ConnectError(`node ${id}: the needs form a cycle`, Code.InvalidArgument);
1564
+ if (seen.has(need))
1565
+ continue;
1566
+ seen.add(need);
1567
+ closure(need, seen, new Set([...path, id]));
1568
+ }
1569
+ return seen;
1570
+ };
1571
+ for (const s of specs) {
1572
+ const reachable = closure(s.id);
1573
+ for (const v of Object.values(s.with)) {
1574
+ const m = typeof v === "string" ? REF.exec(v) : null;
1575
+ const node = m?.[1]?.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
1576
+ if (node && !reachable.has(node)) {
1577
+ throw new ConnectError(`node ${s.id}: ${m[1]} names a node outside its needs`, Code.InvalidArgument);
1578
+ }
1579
+ }
1580
+ }
1581
+ }
826
1582
  function nodeSpec(n, requireHandler) {
827
1583
  let handler = "";
828
1584
  let uses = "";
@@ -844,6 +1600,7 @@ function nodeSpec(n, requireHandler) {
844
1600
  needs: n.needs.map((x) => x.node),
845
1601
  retry: n.retry ? { max: n.retry.max, when: n.retry.when } : undefined,
846
1602
  timeoutMs: n.timeout ? durationMs(n.timeout) : 0,
1603
+ interruptible: n.interruptible,
847
1604
  };
848
1605
  }
849
1606
  /** The server's dispatch.Outcome. */
@@ -867,6 +1624,8 @@ function outcome(t, exitCode) {
867
1624
  return [AttemptState.FAILED, FailureClass.SECRETS_UNAVAILABLE];
868
1625
  case Termination.HANDLER_ERROR:
869
1626
  return [AttemptState.FAILED, FailureClass.HANDLER_ERROR];
1627
+ case Termination.INTERRUPTED:
1628
+ return [AttemptState.FAILED, FailureClass.INTERRUPTED];
870
1629
  default:
871
1630
  return [AttemptState.FAILED, FailureClass.RUNNER_CRASHED];
872
1631
  }
@@ -906,4 +1665,44 @@ function aborted(s) {
906
1665
  s.addEventListener("abort", () => r(), { once: true });
907
1666
  });
908
1667
  }
1668
+ /** The product an account belongs to. */
1669
+ function productOf(who) {
1670
+ return who.product ?? FAKE_PRODUCT;
1671
+ }
1672
+ /** A queue's limits as the selection reads them. */
1673
+ function queueLimits(q) {
1674
+ return {
1675
+ id: q.id,
1676
+ priority: q.priority || 1,
1677
+ paused: q.paused,
1678
+ maxConcurrency: q.maxConcurrency,
1679
+ maxConcurrencyPerTenant: q.maxConcurrencyPerTenant,
1680
+ ...(q.rateLimit?.window ? { rateLimit: { limit: q.rateLimit.limit, windowMs: durationMs(q.rateLimit.window) } } : {}),
1681
+ };
1682
+ }
1683
+ /**
1684
+ * A job's retry and timeout under its queue: the queue's defaults when the
1685
+ * job gives none, and its ceilings, past which the job is refused.
1686
+ */
1687
+ function jobLimits(q, retry, timeoutMs) {
1688
+ const ceiling = q.defaultRetry;
1689
+ if (retry && ceiling) {
1690
+ if (retry.max > ceiling.max)
1691
+ throw new ConnectError(`retry.max ${retry.max} is above queue ${q.name}'s ${ceiling.max}`, Code.InvalidArgument);
1692
+ const allowed = ceiling.when.length > 0 ? ceiling.when : TRANSIENT;
1693
+ const outside = retry.when.filter((c) => !allowed.includes(c));
1694
+ if (outside.length > 0)
1695
+ throw new ConnectError(`retry.when lists ${outside.map((c) => FailureClass[c]).join(", ")}, which queue ${q.name} does not retry`, Code.InvalidArgument);
1696
+ }
1697
+ const maxTimeout = q.defaultTimeout ? durationMs(q.defaultTimeout) : 0;
1698
+ if (timeoutMs > 0 && maxTimeout > 0 && timeoutMs > maxTimeout) {
1699
+ throw new ConnectError(`timeout ${timeoutMs} ms is above queue ${q.name}'s ${maxTimeout} ms`, Code.InvalidArgument);
1700
+ }
1701
+ const r = retry ?? ceiling;
1702
+ return { retry: r ? { max: r.max, when: r.when } : undefined, timeoutMs: timeoutMs || maxTimeout };
1703
+ }
1704
+ /** A stable UUID-shaped id from a string, for the fake's platform spans. */
1705
+ function randomUUIDFrom(s) {
1706
+ return createHash("sha256").update(s).digest("hex").slice(0, 32);
1707
+ }
909
1708
  //# sourceMappingURL=index.js.map