@xeonr/pipelines-sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +267 -2
- package/dist/auth.d.ts +72 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +139 -0
- package/dist/auth.js.map +1 -0
- package/dist/client.d.ts +118 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +101 -0
- package/dist/client.js.map +1 -0
- package/dist/errors.d.ts +77 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +72 -0
- package/dist/errors.js.map +1 -0
- package/dist/gen/buf/validate/validate_pb.d.ts +4828 -0
- package/dist/gen/buf/validate/validate_pb.d.ts.map +1 -0
- package/dist/gen/buf/validate/validate_pb.js +357 -0
- package/dist/gen/buf/validate/validate_pb.js.map +1 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.d.ts +24 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.d.ts.map +1 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.js +37 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.js.map +1 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.d.ts +2245 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.d.ts.map +1 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.js +410 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +1031 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +219 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.d.ts +190 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.js +53 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +1161 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +269 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +915 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +159 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +550 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.js +453 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +760 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js +190 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +800 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js +247 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +895 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js +277 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +1217 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js +204 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +642 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +181 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts +381 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js +145 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js.map +1 -0
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts +313 -0
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/platform/v1/options_pb.js +235 -0
- package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -0
- package/dist/handler.d.ts +94 -0
- package/dist/handler.d.ts.map +1 -0
- package/dist/handler.js +18 -0
- package/dist/handler.js.map +1 -0
- package/dist/index.d.ts +29 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/time.d.ts +8 -0
- package/dist/internal/time.d.ts.map +1 -0
- package/dist/internal/time.js +25 -0
- package/dist/internal/time.js.map +1 -0
- package/dist/logship.d.ts +67 -0
- package/dist/logship.d.ts.map +1 -0
- package/dist/logship.js +176 -0
- package/dist/logship.js.map +1 -0
- package/dist/testing/index.d.ts +202 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +909 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/transport.d.ts +43 -0
- package/dist/transport.d.ts.map +1 -0
- package/dist/transport.js +44 -0
- package/dist/transport.js.map +1 -0
- package/dist/worker.d.ts +213 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +664 -0
- package/dist/worker.js.map +1 -0
- package/package.json +70 -3
|
@@ -0,0 +1,909 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @xeonr/pipelines-sdk/testing: an in-memory fake of pipelines-api's worker
|
|
3
|
+
* and jobs surface, so a product can unit-test its handlers, and the code
|
|
4
|
+
* that enqueues them, with no server.
|
|
5
|
+
*
|
|
6
|
+
* It is a real Connect router behind an in-memory transport, so the SDK's
|
|
7
|
+
* `PipelinesClient` and `Worker` talk to it exactly as they talk to
|
|
8
|
+
* pipelines-api: the same generated clients, the same error codes.
|
|
9
|
+
*
|
|
10
|
+
* ```ts
|
|
11
|
+
* const fake = new FakePipelines();
|
|
12
|
+
* const worker = new Worker({ transport: fake.transport(), tenant: FAKE_TENANT, pool: { name: "test" } });
|
|
13
|
+
* worker.handle(sendEmail);
|
|
14
|
+
* await worker.start();
|
|
15
|
+
* const client = new PipelinesClient({ transport: fake.transport() });
|
|
16
|
+
* const { run } = await client.createJob({ tenant: FAKE_TENANT, handler: "acme/send-email", idempotencyKey: "k", payload: { to: "a@b" } });
|
|
17
|
+
* const done = await fake.waitForRun(run.id);
|
|
18
|
+
* ```
|
|
19
|
+
*
|
|
20
|
+
* What it models: auth by bearer token (accounts with a tenant and scopes),
|
|
21
|
+
* namespace and pool ownership, `PublishVersion`, `SetPool`, `CreateJob`
|
|
22
|
+
* (with delay and idempotency), `CreateRun` with handler nodes and external
|
|
23
|
+
* (image) nodes that a test settles with {@link FakePipelines.completeNode},
|
|
24
|
+
* `${{ steps.<node>.outputs.kv.<slot> }}` references, the claim long-poll,
|
|
25
|
+
* leases and their lapse, heartbeats with cancel, logs (idempotent on seq,
|
|
26
|
+
* `out_of_range` for a gap), completion with the server's termination to
|
|
27
|
+
* failure-class mapping, retries by class, cancel, and error injection.
|
|
28
|
+
*
|
|
29
|
+
* What it does not: secrets, blobs, tiers, routing between pools, the
|
|
30
|
+
* outbox, and the platform's own deadline enforcement.
|
|
31
|
+
*/
|
|
32
|
+
import { randomUUID } from "node:crypto";
|
|
33
|
+
import { create, fromJson, toJson } from "@bufbuild/protobuf";
|
|
34
|
+
import { durationFromMs, durationMs, timestampFromMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
35
|
+
import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect";
|
|
36
|
+
import { tokenSource } from "../auth.js";
|
|
37
|
+
import { authInterceptor } from "../transport.js";
|
|
38
|
+
import { AttemptState, FailureClass, NodeState, RunState, Termination } from "../gen/xeonr/pipelines/v1/common_pb.js";
|
|
39
|
+
import { PoolKind, PoolOwnerKind, PoolSchema, PoolsService } from "../gen/xeonr/pipelines/v1/pools_pb.js";
|
|
40
|
+
import { PackageKind, RegistryService, VersionState } from "../gen/xeonr/pipelines/v1/registry_pb.js";
|
|
41
|
+
import { AttemptSchema, NodeRunSchema, RunSchema, RunsService } from "../gen/xeonr/pipelines/v1/runs_pb.js";
|
|
42
|
+
import { ClaimedAttemptSchema, WorkerService } from "../gen/xeonr/pipelines/private/v1/worker_pb.js";
|
|
43
|
+
/** The tenant the default account acts for. */
|
|
44
|
+
export const FAKE_TENANT = "urn:xeonr:auth:team:00000000-0000-4000-8000-000000000001";
|
|
45
|
+
/** The default account's token. */
|
|
46
|
+
export const FAKE_TOKEN = "fake-token";
|
|
47
|
+
/** Every scope pipelines' RPCs use. */
|
|
48
|
+
export const ALL_SCOPES = [
|
|
49
|
+
"pipelines:run:read",
|
|
50
|
+
"pipelines:run:edit",
|
|
51
|
+
"pipelines:package:read",
|
|
52
|
+
"pipelines:package:edit",
|
|
53
|
+
"pipelines:pool:read",
|
|
54
|
+
"pipelines:pool:edit",
|
|
55
|
+
"pipelines:worker:edit",
|
|
56
|
+
];
|
|
57
|
+
const TRANSIENT = [
|
|
58
|
+
FailureClass.ABANDONED,
|
|
59
|
+
FailureClass.RUNNER_CRASHED,
|
|
60
|
+
FailureClass.IMAGE_PULL,
|
|
61
|
+
FailureClass.STAGING_FAILED,
|
|
62
|
+
FailureClass.INTERNAL,
|
|
63
|
+
];
|
|
64
|
+
const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
65
|
+
const REF = /^\$\{\{\s*([^}]+?)\s*\}\}$/;
|
|
66
|
+
/**
|
|
67
|
+
* An in-memory pipelines-api for tests. Call {@link close} when done (it
|
|
68
|
+
* stops the timer that looks at leases and delays).
|
|
69
|
+
*/
|
|
70
|
+
export class FakePipelines {
|
|
71
|
+
/** Runs by id. */
|
|
72
|
+
runs = new Map();
|
|
73
|
+
/** Every call that reached the fake, in order. */
|
|
74
|
+
calls = [];
|
|
75
|
+
accounts;
|
|
76
|
+
opts;
|
|
77
|
+
packages = new Map();
|
|
78
|
+
namespaces = new Map();
|
|
79
|
+
pools = new Map();
|
|
80
|
+
attemptsById = new Map();
|
|
81
|
+
idempotency = new Map();
|
|
82
|
+
injected = new Map();
|
|
83
|
+
timer;
|
|
84
|
+
changed;
|
|
85
|
+
constructor(opts = {}) {
|
|
86
|
+
this.accounts = new Map(Object.entries(opts.accounts ?? { [FAKE_TOKEN]: { tenant: FAKE_TENANT, scopes: ALL_SCOPES } }));
|
|
87
|
+
const leaseMs = opts.leaseMs ?? 60_000;
|
|
88
|
+
this.opts = {
|
|
89
|
+
leaseMs,
|
|
90
|
+
heartbeatIntervalMs: opts.heartbeatIntervalMs ?? leaseMs / 4,
|
|
91
|
+
cancelGraceMs: opts.cancelGraceMs ?? 10_000,
|
|
92
|
+
defaultTimeoutMs: opts.defaultTimeoutMs ?? 0,
|
|
93
|
+
defaultRetries: opts.defaultRetries ?? 2,
|
|
94
|
+
retryBackoffMs: opts.retryBackoffMs ?? 0,
|
|
95
|
+
logMaxBytes: opts.logMaxBytes ?? 0,
|
|
96
|
+
tickMs: opts.tickMs ?? 10,
|
|
97
|
+
};
|
|
98
|
+
this.changed = signal();
|
|
99
|
+
this.timer = setInterval(() => this.tick(), this.opts.tickMs);
|
|
100
|
+
this.timer.unref();
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* A transport to the fake that authenticates with `token` (default
|
|
104
|
+
* {@link FAKE_TOKEN}) through the SDK's own auth interceptor, so token
|
|
105
|
+
* refresh behaves as it does against the real API.
|
|
106
|
+
*/
|
|
107
|
+
transport(token = FAKE_TOKEN, interceptors = []) {
|
|
108
|
+
const provider = typeof token === "string" ? () => token : token;
|
|
109
|
+
return this.buildRouter([authInterceptor(tokenSource(provider)), ...interceptors]);
|
|
110
|
+
}
|
|
111
|
+
/** Adds or replaces an account. */
|
|
112
|
+
setAccount(token, account) {
|
|
113
|
+
this.accounts.set(token, account);
|
|
114
|
+
}
|
|
115
|
+
/** Forgets a token: calls bearing it answer `unauthenticated`. */
|
|
116
|
+
revoke(token) {
|
|
117
|
+
this.accounts.delete(token);
|
|
118
|
+
}
|
|
119
|
+
/** Makes the next `times` calls of `method` fail with `code`. */
|
|
120
|
+
inject(method, code, times = 1, message = "injected by the fake") {
|
|
121
|
+
const list = this.injected.get(method) ?? [];
|
|
122
|
+
for (let i = 0; i < times; i++)
|
|
123
|
+
list.push(new ConnectError(message, code));
|
|
124
|
+
this.injected.set(method, list);
|
|
125
|
+
}
|
|
126
|
+
/** How many calls of `method` reached the fake. */
|
|
127
|
+
count(method) {
|
|
128
|
+
return this.calls.filter((c) => c.method === method).length;
|
|
129
|
+
}
|
|
130
|
+
/** One run, or throws. */
|
|
131
|
+
run(runId) {
|
|
132
|
+
const r = this.runs.get(runId);
|
|
133
|
+
if (!r)
|
|
134
|
+
throw new Error(`FakePipelines: no run ${runId}`);
|
|
135
|
+
return r;
|
|
136
|
+
}
|
|
137
|
+
/** One attempt, or throws. */
|
|
138
|
+
attempt(attemptId) {
|
|
139
|
+
const a = this.attemptsById.get(attemptId);
|
|
140
|
+
if (!a)
|
|
141
|
+
throw new Error(`FakePipelines: no attempt ${attemptId}`);
|
|
142
|
+
return a;
|
|
143
|
+
}
|
|
144
|
+
/** Every attempt of a run, in creation order. */
|
|
145
|
+
attemptsOf(runId) {
|
|
146
|
+
return [...this.run(runId).nodes.values()].flatMap((n) => n.attempts);
|
|
147
|
+
}
|
|
148
|
+
/** An attempt's log as text. */
|
|
149
|
+
logOf(attemptId) {
|
|
150
|
+
const dec = new TextDecoder();
|
|
151
|
+
return this.attempt(attemptId)
|
|
152
|
+
.chunks.map((c) => dec.decode(c))
|
|
153
|
+
.join("");
|
|
154
|
+
}
|
|
155
|
+
/** A registered pool, by name. */
|
|
156
|
+
pool(name) {
|
|
157
|
+
return [...this.pools.values()].find((p) => p.name === name);
|
|
158
|
+
}
|
|
159
|
+
/** Whether `name@version` was published. */
|
|
160
|
+
published(name, version) {
|
|
161
|
+
return this.packages.get(name)?.versions.get(version);
|
|
162
|
+
}
|
|
163
|
+
/** Waits until a run is terminal and returns it. */
|
|
164
|
+
async waitForRun(runId, timeoutMs = 10_000) {
|
|
165
|
+
return this.waitFor(() => {
|
|
166
|
+
const r = this.run(runId);
|
|
167
|
+
return terminal(r.state) ? r : undefined;
|
|
168
|
+
}, timeoutMs);
|
|
169
|
+
}
|
|
170
|
+
/** Waits until `check` returns something other than undefined. */
|
|
171
|
+
async waitFor(check, timeoutMs = 10_000) {
|
|
172
|
+
const until = Date.now() + timeoutMs;
|
|
173
|
+
for (;;) {
|
|
174
|
+
const v = check();
|
|
175
|
+
if (v !== undefined)
|
|
176
|
+
return v;
|
|
177
|
+
if (Date.now() > until)
|
|
178
|
+
throw new Error("FakePipelines.waitFor: timed out");
|
|
179
|
+
await new Promise((r) => setTimeout(r, this.opts.tickMs));
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/** Lapses an attempt's lease now, as the reaper would at its expiry. */
|
|
183
|
+
expireLease(attemptId) {
|
|
184
|
+
this.attempt(attemptId).leaseExpiresAt = 0;
|
|
185
|
+
this.tick();
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Settles an external (image) node, standing in for a runner: succeeded
|
|
189
|
+
* with `kv`, or failed with `failureClass`.
|
|
190
|
+
*/
|
|
191
|
+
completeNode(runId, nodeId, result = {}) {
|
|
192
|
+
const run = this.run(runId);
|
|
193
|
+
const node = run.nodes.get(nodeId);
|
|
194
|
+
if (!node || node.handler)
|
|
195
|
+
throw new Error(`FakePipelines.completeNode: ${nodeId} is not an external node of ${runId}`);
|
|
196
|
+
if (node.state !== NodeState.READY)
|
|
197
|
+
throw new Error(`FakePipelines.completeNode: ${nodeId} is not ready`);
|
|
198
|
+
if (result.failureClass) {
|
|
199
|
+
node.state = NodeState.FAILED;
|
|
200
|
+
node.failureClass = result.failureClass;
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
node.state = NodeState.SUCCEEDED;
|
|
204
|
+
node.kv = result.kv ?? {};
|
|
205
|
+
}
|
|
206
|
+
this.advance(run);
|
|
207
|
+
}
|
|
208
|
+
/** Stops the fake's timer. */
|
|
209
|
+
close() {
|
|
210
|
+
clearInterval(this.timer);
|
|
211
|
+
this.changed.resolve();
|
|
212
|
+
}
|
|
213
|
+
// ---- the router -------------------------------------------------------
|
|
214
|
+
buildRouter(interceptors) {
|
|
215
|
+
return createRouterTransport((router) => {
|
|
216
|
+
router.service(WorkerService, {
|
|
217
|
+
claimAttempts: async (req, ctx) => {
|
|
218
|
+
const who = this.enter("ClaimAttempts", ctx, "pipelines:worker:edit");
|
|
219
|
+
if (req.selector.case !== "handlers")
|
|
220
|
+
throw new ConnectError("an SDK worker claims by handler", Code.PermissionDenied);
|
|
221
|
+
const sel = req.selector.value;
|
|
222
|
+
const pool = this.pools.get(sel.poolId);
|
|
223
|
+
const denied = new ConnectError("the caller's owner does not own that sdk pool or those handlers", Code.PermissionDenied);
|
|
224
|
+
if (!pool || pool.kind !== PoolKind.SDK || pool.owner?.tenant !== who.tenant)
|
|
225
|
+
throw denied;
|
|
226
|
+
for (const h of sel.handlers) {
|
|
227
|
+
if (this.namespaces.get(h.split("/")[0]) !== who.tenant || !pool.handlers.includes(h))
|
|
228
|
+
throw denied;
|
|
229
|
+
}
|
|
230
|
+
if (req.freeSlots < 1 || req.freeSlots > 256)
|
|
231
|
+
throw new ConnectError("free_slots: 1 to 256", Code.InvalidArgument);
|
|
232
|
+
const until = Date.now() + Math.min(req.wait ? durationMs(req.wait) : 30_000, 30_000);
|
|
233
|
+
// One timer and one abort listener for the whole hold.
|
|
234
|
+
const stop = Promise.race([sleepUntil(until), aborted(ctx.signal)]);
|
|
235
|
+
for (;;) {
|
|
236
|
+
const claimed = this.claim(pool, sel.handlers, sel.workerId, req.freeSlots);
|
|
237
|
+
if (claimed.length > 0 || Date.now() >= until || ctx.signal.aborted) {
|
|
238
|
+
return { attempts: claimed.map((a) => this.describe(a)) };
|
|
239
|
+
}
|
|
240
|
+
await Promise.race([this.changed.promise, stop]);
|
|
241
|
+
}
|
|
242
|
+
},
|
|
243
|
+
startAttempt: (req, ctx) => {
|
|
244
|
+
const who = this.enter("StartAttempt", ctx, "pipelines:worker:edit");
|
|
245
|
+
const a = this.held(who, req.ref);
|
|
246
|
+
const run = this.run(a.runId);
|
|
247
|
+
if (a.state === AttemptState.RUNNING)
|
|
248
|
+
return this.started(a);
|
|
249
|
+
if (a.state !== AttemptState.CLAIMED)
|
|
250
|
+
throw stale();
|
|
251
|
+
if (run.cancelRequestedAt !== undefined) {
|
|
252
|
+
throw new ConnectError("a cancel was requested: do not start the step", Code.FailedPrecondition);
|
|
253
|
+
}
|
|
254
|
+
a.state = AttemptState.RUNNING;
|
|
255
|
+
a.leaseExpiresAt = Date.now() + this.opts.leaseMs;
|
|
256
|
+
const node = run.nodes.get(a.nodeId);
|
|
257
|
+
node.state = NodeState.RUNNING;
|
|
258
|
+
a.startedAt = Date.now();
|
|
259
|
+
return this.started(a);
|
|
260
|
+
},
|
|
261
|
+
heartbeat: (req, ctx) => {
|
|
262
|
+
const who = this.enter("Heartbeat", ctx, "pipelines:worker:edit");
|
|
263
|
+
const a = this.held(who, req.ref);
|
|
264
|
+
if (a.state !== AttemptState.CLAIMED && a.state !== AttemptState.RUNNING)
|
|
265
|
+
throw stale();
|
|
266
|
+
a.leaseExpiresAt = Date.now() + this.opts.leaseMs;
|
|
267
|
+
const run = this.run(a.runId);
|
|
268
|
+
const cancel = run.cancelRequestedAt !== undefined;
|
|
269
|
+
return {
|
|
270
|
+
cancel,
|
|
271
|
+
cancelReason: cancel ? run.cancelReason || "the run was cancelled" : "",
|
|
272
|
+
leaseExpiresAt: timestampFromMs(a.leaseExpiresAt),
|
|
273
|
+
};
|
|
274
|
+
},
|
|
275
|
+
appendLogs: (req, ctx) => {
|
|
276
|
+
const who = this.enter("AppendLogs", ctx, "pipelines:worker:edit");
|
|
277
|
+
const a = this.held(who, req.ref);
|
|
278
|
+
if (a.state !== AttemptState.CLAIMED && a.state !== AttemptState.RUNNING)
|
|
279
|
+
throw stale();
|
|
280
|
+
const next = BigInt(a.chunks.length);
|
|
281
|
+
if (req.seq > next)
|
|
282
|
+
throw new ConnectError(`the next expected seq is ${next}`, Code.OutOfRange);
|
|
283
|
+
if (req.data.length < 1 || req.data.length > 1 << 20)
|
|
284
|
+
throw new ConnectError("data: 1 byte to 1 MiB", Code.InvalidArgument);
|
|
285
|
+
if (req.seq === next && !a.logTruncated) {
|
|
286
|
+
a.chunks.push(req.data);
|
|
287
|
+
a.logFinal ||= req.final;
|
|
288
|
+
const size = a.chunks.reduce((n, c) => n + c.length, 0);
|
|
289
|
+
if (this.opts.logMaxBytes > 0 && size >= this.opts.logMaxBytes)
|
|
290
|
+
a.logTruncated = true;
|
|
291
|
+
}
|
|
292
|
+
return { nextSeq: BigInt(a.chunks.length), truncated: a.logTruncated };
|
|
293
|
+
},
|
|
294
|
+
completeAttempt: (req, ctx) => {
|
|
295
|
+
const who = this.enter("CompleteAttempt", ctx, "pipelines:worker:edit");
|
|
296
|
+
const a = this.held(who, req.ref);
|
|
297
|
+
if (a.completedAt !== undefined)
|
|
298
|
+
return { state: a.state };
|
|
299
|
+
if (a.state !== AttemptState.RUNNING)
|
|
300
|
+
throw stale();
|
|
301
|
+
if (req.termination === Termination.UNSPECIFIED)
|
|
302
|
+
throw new ConnectError("termination: required", Code.InvalidArgument);
|
|
303
|
+
if (req.message.length > 2048)
|
|
304
|
+
throw new ConnectError("message: at most 2048", Code.InvalidArgument);
|
|
305
|
+
const kv = {};
|
|
306
|
+
const entries = Object.entries(req.kv);
|
|
307
|
+
if (entries.length > 100)
|
|
308
|
+
throw new ConnectError("kv: at most 100 pairs", Code.InvalidArgument);
|
|
309
|
+
for (const [k, v] of entries) {
|
|
310
|
+
if (k.length > 64 || !KV_KEY.test(k))
|
|
311
|
+
throw new ConnectError(`kv: ${k} is not a valid key`, Code.InvalidArgument);
|
|
312
|
+
kv[k] = toJson(ValueSchema, v);
|
|
313
|
+
}
|
|
314
|
+
const [state, failureClass] = outcome(req.termination, req.exitCode);
|
|
315
|
+
a.state = state;
|
|
316
|
+
a.failureClass = failureClass;
|
|
317
|
+
a.termination = req.termination;
|
|
318
|
+
a.exitCode = req.exitCode;
|
|
319
|
+
a.message = req.message;
|
|
320
|
+
a.kv = state === AttemptState.SUCCEEDED ? kv : {};
|
|
321
|
+
a.completedAt = Date.now();
|
|
322
|
+
this.settle(a);
|
|
323
|
+
return { state: a.state };
|
|
324
|
+
},
|
|
325
|
+
});
|
|
326
|
+
router.service(RunsService, {
|
|
327
|
+
createJob: (req, ctx) => {
|
|
328
|
+
const who = this.enter("CreateJob", ctx, "pipelines:run:edit");
|
|
329
|
+
this.actsFor(who, req.tenant);
|
|
330
|
+
const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
|
|
331
|
+
if (existing)
|
|
332
|
+
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
333
|
+
this.requireHandler(req.handler);
|
|
334
|
+
const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, {}, req.delay ? durationMs(req.delay) : 0);
|
|
335
|
+
this.addNode(run, { id: "job", handler: req.handler, uses: "", with: req.payload ?? {}, needs: [], retry: undefined, timeoutMs: 0 });
|
|
336
|
+
this.advance(run);
|
|
337
|
+
return { run: this.runMessage(run), deduplicated: false };
|
|
338
|
+
},
|
|
339
|
+
createRun: (req, ctx) => {
|
|
340
|
+
const who = this.enter("CreateRun", ctx, "pipelines:run:edit");
|
|
341
|
+
this.actsFor(who, req.tenant);
|
|
342
|
+
const existing = this.idempotency.get(`${who.product ?? who.tenant}\u0000${req.idempotencyKey}`);
|
|
343
|
+
if (existing)
|
|
344
|
+
return { run: this.runMessage(this.run(existing)), deduplicated: true };
|
|
345
|
+
if (req.source.case !== "inlineDefinition")
|
|
346
|
+
throw new ConnectError("only inline definitions", Code.Unimplemented);
|
|
347
|
+
const def = req.source.value;
|
|
348
|
+
if (def.nodes.length === 0)
|
|
349
|
+
throw new ConnectError("definition: no nodes", Code.InvalidArgument);
|
|
350
|
+
const run = this.newRun(who, req.tenant, req.idempotencyKey, req.resource, req.variables ?? {}, 0);
|
|
351
|
+
for (const n of def.nodes)
|
|
352
|
+
this.addNode(run, nodeSpec(n, (h) => this.requireHandler(h)));
|
|
353
|
+
for (const n of run.nodes.values()) {
|
|
354
|
+
for (const need of n.needs)
|
|
355
|
+
if (!run.nodes.has(need))
|
|
356
|
+
throw new ConnectError(`node ${n.id}: unknown need ${need}`, Code.InvalidArgument);
|
|
357
|
+
}
|
|
358
|
+
this.advance(run);
|
|
359
|
+
return { run: this.runMessage(run), deduplicated: false };
|
|
360
|
+
},
|
|
361
|
+
getRun: (req, ctx) => {
|
|
362
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
363
|
+
return { run: this.runMessage(this.visible(who, req.runId)) };
|
|
364
|
+
},
|
|
365
|
+
getRunGraph: (req, ctx) => {
|
|
366
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
367
|
+
const run = this.visible(who, req.runId);
|
|
368
|
+
return { run: this.runMessage(run), nodes: [...run.nodes.values()].map((n) => this.nodeMessage(n)) };
|
|
369
|
+
},
|
|
370
|
+
listRuns: (req, ctx) => {
|
|
371
|
+
const who = this.enter("GetRun", ctx, "pipelines:run:read");
|
|
372
|
+
this.actsFor(who, req.tenant);
|
|
373
|
+
const runs = [...this.runs.values()]
|
|
374
|
+
.filter((r) => r.tenant === req.tenant && (req.states.length === 0 || req.states.includes(r.state)))
|
|
375
|
+
.filter((r) => !req.resource || (r.resource?.kind === req.resource.kind && r.resource.id === req.resource.id))
|
|
376
|
+
.reverse();
|
|
377
|
+
return { runs: runs.map((r) => this.runMessage(r)), nextCursor: "" };
|
|
378
|
+
},
|
|
379
|
+
cancelRun: (req, ctx) => {
|
|
380
|
+
const who = this.enter("CancelRun", ctx, "pipelines:run:edit");
|
|
381
|
+
const run = this.visible(who, req.runId);
|
|
382
|
+
this.cancel(run, req.reason);
|
|
383
|
+
return { run: this.runMessage(run) };
|
|
384
|
+
},
|
|
385
|
+
});
|
|
386
|
+
router.service(RegistryService, {
|
|
387
|
+
publishVersion: (req, ctx) => {
|
|
388
|
+
const who = this.enter("PublishVersion", ctx, "pipelines:package:edit");
|
|
389
|
+
const namespace = req.name.split("/")[0];
|
|
390
|
+
const owner = this.namespaces.get(namespace);
|
|
391
|
+
if (owner !== undefined && owner !== who.tenant) {
|
|
392
|
+
throw new ConnectError(`the ${namespace} namespace is not this team's`, Code.PermissionDenied);
|
|
393
|
+
}
|
|
394
|
+
if (req.tenant !== who.tenant)
|
|
395
|
+
throw new ConnectError("tenant: the caller does not act for it", Code.PermissionDenied);
|
|
396
|
+
if (!req.manifest)
|
|
397
|
+
throw new ConnectError("manifest: required", Code.InvalidArgument);
|
|
398
|
+
this.namespaces.set(namespace, who.tenant);
|
|
399
|
+
const pkg = this.packages.get(req.name) ?? { tenant: who.tenant, versions: new Map() };
|
|
400
|
+
const prior = pkg.versions.get(req.version);
|
|
401
|
+
if (prior && JSON.stringify(prior) !== JSON.stringify(req.manifest)) {
|
|
402
|
+
throw new ConnectError(`${req.name}@${req.version} exists with different content`, Code.AlreadyExists);
|
|
403
|
+
}
|
|
404
|
+
pkg.versions.set(req.version, req.manifest);
|
|
405
|
+
this.packages.set(req.name, pkg);
|
|
406
|
+
return { version: { name: req.name, version: req.version, manifest: req.manifest, state: VersionState.READY } };
|
|
407
|
+
},
|
|
408
|
+
});
|
|
409
|
+
router.service(PoolsService, {
|
|
410
|
+
setPool: (req, ctx) => {
|
|
411
|
+
const who = this.enter("SetPool", ctx, "pipelines:pool:edit");
|
|
412
|
+
const p = req.pool;
|
|
413
|
+
if (!p)
|
|
414
|
+
throw new ConnectError("pool: required", Code.InvalidArgument);
|
|
415
|
+
if (p.owner?.kind !== PoolOwnerKind.TENANT || p.owner.tenant !== who.tenant) {
|
|
416
|
+
throw new ConnectError("pool.owner: the caller does not act for it", Code.PermissionDenied);
|
|
417
|
+
}
|
|
418
|
+
if (p.kind === PoolKind.SDK && p.handlers.length === 0) {
|
|
419
|
+
throw new ConnectError("pool.handlers: an sdk pool names the handlers it serves", Code.InvalidArgument);
|
|
420
|
+
}
|
|
421
|
+
for (const h of p.handlers) {
|
|
422
|
+
if (this.namespaces.get(h.split("/")[0]) !== who.tenant) {
|
|
423
|
+
throw new ConnectError(`pool.handlers: the namespace of ${h} is not this team's`, Code.PermissionDenied);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
const existing = [...this.pools.values()].find((x) => x.owner?.tenant === p.owner?.tenant && x.name === p.name);
|
|
427
|
+
const pool = create(PoolSchema, { ...p, id: existing?.id ?? randomUUID() });
|
|
428
|
+
this.pools.set(pool.id, pool);
|
|
429
|
+
this.notify();
|
|
430
|
+
return { pool, created: !existing };
|
|
431
|
+
},
|
|
432
|
+
});
|
|
433
|
+
}, { transport: { interceptors } });
|
|
434
|
+
}
|
|
435
|
+
// ---- auth -------------------------------------------------------------
|
|
436
|
+
enter(method, ctx, scope) {
|
|
437
|
+
const header = ctx.requestHeader.get("Authorization") ?? "";
|
|
438
|
+
const token = header.startsWith("Bearer ") ? header.slice(7) : "";
|
|
439
|
+
this.calls.push({ method, token });
|
|
440
|
+
const queue = this.injected.get(method);
|
|
441
|
+
const injected = queue?.shift();
|
|
442
|
+
if (injected)
|
|
443
|
+
throw injected;
|
|
444
|
+
const who = this.accounts.get(token);
|
|
445
|
+
if (!who)
|
|
446
|
+
throw new ConnectError("a valid token is required", Code.Unauthenticated);
|
|
447
|
+
if (!who.scopes.includes(scope))
|
|
448
|
+
throw new ConnectError(`the token lacks ${scope}`, Code.PermissionDenied);
|
|
449
|
+
return who;
|
|
450
|
+
}
|
|
451
|
+
actsFor(who, tenant) {
|
|
452
|
+
if (who.tenant !== tenant)
|
|
453
|
+
throw new ConnectError("the caller may not act for that tenant", Code.PermissionDenied);
|
|
454
|
+
}
|
|
455
|
+
visible(who, runId) {
|
|
456
|
+
const run = this.runs.get(runId);
|
|
457
|
+
if (!run || run.tenant !== who.tenant)
|
|
458
|
+
throw new ConnectError("no such run", Code.NotFound);
|
|
459
|
+
return run;
|
|
460
|
+
}
|
|
461
|
+
/** The server's held(): the attempt at this epoch, held by a pool of the caller's team. */
|
|
462
|
+
held(who, ref) {
|
|
463
|
+
if (!ref)
|
|
464
|
+
throw new ConnectError("ref: required", Code.InvalidArgument);
|
|
465
|
+
const a = this.attemptsById.get(ref.attemptId);
|
|
466
|
+
if (!a || a.epoch !== ref.claimEpoch || !a.poolId)
|
|
467
|
+
throw stale();
|
|
468
|
+
if (this.pools.get(a.poolId)?.owner?.tenant !== who.tenant)
|
|
469
|
+
throw stale();
|
|
470
|
+
return a;
|
|
471
|
+
}
|
|
472
|
+
requireHandler(handler) {
|
|
473
|
+
const pkg = this.packages.get(handler);
|
|
474
|
+
if (!pkg || pkg.versions.size === 0 || [...pkg.versions.values()].some((m) => m.kind !== PackageKind.WORKER)) {
|
|
475
|
+
throw new ConnectError(`handler ${handler} is not a published worker package`, Code.InvalidArgument);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
// ---- the scheduler ----------------------------------------------------
|
|
479
|
+
newRun(who, tenant, key, resource, variables, delayMs) {
|
|
480
|
+
const now = Date.now();
|
|
481
|
+
const run = {
|
|
482
|
+
id: randomUUID(),
|
|
483
|
+
tenant,
|
|
484
|
+
product: who.product ?? who.tenant,
|
|
485
|
+
idempotencyKey: key,
|
|
486
|
+
createdBy: `urn:fake:sa:${who.tenant}`,
|
|
487
|
+
resource: resource ? { kind: resource.kind, id: resource.id, ...(resource.json ? { json: resource.json } : {}) } : undefined,
|
|
488
|
+
variables,
|
|
489
|
+
state: RunState.QUEUED,
|
|
490
|
+
failureClass: FailureClass.UNSPECIFIED,
|
|
491
|
+
failedNodeId: "",
|
|
492
|
+
notBefore: now + delayMs,
|
|
493
|
+
createdAt: now,
|
|
494
|
+
startedAt: undefined,
|
|
495
|
+
finishedAt: undefined,
|
|
496
|
+
cancelRequestedAt: undefined,
|
|
497
|
+
cancelReason: "",
|
|
498
|
+
nodes: new Map(),
|
|
499
|
+
};
|
|
500
|
+
this.runs.set(run.id, run);
|
|
501
|
+
this.idempotency.set(`${run.product}\u0000${key}`, run.id);
|
|
502
|
+
return run;
|
|
503
|
+
}
|
|
504
|
+
addNode(run, spec) {
|
|
505
|
+
const refs = new Set(spec.needs);
|
|
506
|
+
for (const v of Object.values(spec.with)) {
|
|
507
|
+
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
508
|
+
const node = m?.[1]?.match(/^(?:steps|needs)\.([^.]+)\./)?.[1];
|
|
509
|
+
if (node)
|
|
510
|
+
refs.add(node);
|
|
511
|
+
}
|
|
512
|
+
run.nodes.set(spec.id, {
|
|
513
|
+
id: spec.id,
|
|
514
|
+
handler: spec.handler,
|
|
515
|
+
uses: spec.uses,
|
|
516
|
+
state: NodeState.PENDING,
|
|
517
|
+
failureClass: FailureClass.UNSPECIFIED,
|
|
518
|
+
kv: {},
|
|
519
|
+
attempts: [],
|
|
520
|
+
needs: [...refs],
|
|
521
|
+
with: spec.with,
|
|
522
|
+
retry: spec.retry ?? { max: this.opts.defaultRetries, when: [] },
|
|
523
|
+
timeoutMs: spec.timeoutMs || this.opts.defaultTimeoutMs,
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
/** Makes ready what can be, skips what cannot run, and settles the run. */
|
|
527
|
+
advance(run) {
|
|
528
|
+
if (terminal(run.state))
|
|
529
|
+
return;
|
|
530
|
+
if (run.notBefore > Date.now() && run.cancelRequestedAt === undefined)
|
|
531
|
+
return;
|
|
532
|
+
for (let changed = true; changed;) {
|
|
533
|
+
changed = false;
|
|
534
|
+
for (const n of run.nodes.values()) {
|
|
535
|
+
if (n.state !== NodeState.PENDING)
|
|
536
|
+
continue;
|
|
537
|
+
const needs = n.needs.map((id) => run.nodes.get(id));
|
|
538
|
+
if (run.cancelRequestedAt !== undefined) {
|
|
539
|
+
n.state = NodeState.SKIPPED;
|
|
540
|
+
changed = true;
|
|
541
|
+
}
|
|
542
|
+
else if (needs.some((x) => x.state === NodeState.FAILED || x.state === NodeState.SKIPPED || x.state === NodeState.CANCELLED)) {
|
|
543
|
+
n.state = NodeState.SKIPPED;
|
|
544
|
+
changed = true;
|
|
545
|
+
}
|
|
546
|
+
else if (needs.every((x) => x.state === NodeState.SUCCEEDED)) {
|
|
547
|
+
n.state = NodeState.READY;
|
|
548
|
+
run.startedAt ??= Date.now();
|
|
549
|
+
if (run.state === RunState.QUEUED)
|
|
550
|
+
run.state = RunState.RUNNING;
|
|
551
|
+
if (n.handler) {
|
|
552
|
+
const inputs = this.resolve(run, n);
|
|
553
|
+
if (typeof inputs === "string") {
|
|
554
|
+
n.state = NodeState.FAILED;
|
|
555
|
+
n.failureClass = FailureClass.INVALID_IO;
|
|
556
|
+
}
|
|
557
|
+
else {
|
|
558
|
+
this.offer(run, n, inputs, 0);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
changed = true;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
const nodes = [...run.nodes.values()];
|
|
566
|
+
if (nodes.every((n) => terminalNode(n.state))) {
|
|
567
|
+
run.finishedAt = Date.now();
|
|
568
|
+
const failed = nodes.find((n) => n.state === NodeState.FAILED);
|
|
569
|
+
if (run.cancelRequestedAt !== undefined) {
|
|
570
|
+
run.state = RunState.CANCELLED;
|
|
571
|
+
}
|
|
572
|
+
else if (failed) {
|
|
573
|
+
run.state = RunState.FAILED;
|
|
574
|
+
run.failureClass = failed.failureClass;
|
|
575
|
+
run.failedNodeId = failed.id;
|
|
576
|
+
}
|
|
577
|
+
else {
|
|
578
|
+
run.state = RunState.SUCCEEDED;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
resolve(run, n) {
|
|
583
|
+
const out = {};
|
|
584
|
+
for (const [k, v] of Object.entries(n.with)) {
|
|
585
|
+
const m = typeof v === "string" ? REF.exec(v) : null;
|
|
586
|
+
if (!m) {
|
|
587
|
+
out[k] = v;
|
|
588
|
+
continue;
|
|
589
|
+
}
|
|
590
|
+
const path = m[1].split(".");
|
|
591
|
+
if ((path[0] === "steps" || path[0] === "needs") && path[2] === "outputs" && path[3] === "kv" && path[4]) {
|
|
592
|
+
const value = run.nodes.get(path[1])?.kv[path[4]];
|
|
593
|
+
if (value === undefined)
|
|
594
|
+
return `${m[1]} does not resolve`;
|
|
595
|
+
out[k] = value;
|
|
596
|
+
}
|
|
597
|
+
else if (path[0] === "variables" && path[1] && run.variables[path[1]] !== undefined) {
|
|
598
|
+
out[k] = run.variables[path[1]];
|
|
599
|
+
}
|
|
600
|
+
else {
|
|
601
|
+
return `${m[1]} does not resolve`;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
return out;
|
|
605
|
+
}
|
|
606
|
+
offer(run, n, inputs, delayMs) {
|
|
607
|
+
const a = {
|
|
608
|
+
id: randomUUID(),
|
|
609
|
+
runId: run.id,
|
|
610
|
+
nodeId: n.id,
|
|
611
|
+
number: n.attempts.length + 1,
|
|
612
|
+
state: AttemptState.OFFERED,
|
|
613
|
+
epoch: 0n,
|
|
614
|
+
workerId: "",
|
|
615
|
+
poolId: "",
|
|
616
|
+
leaseExpiresAt: 0,
|
|
617
|
+
notBefore: Date.now() + delayMs,
|
|
618
|
+
inputs,
|
|
619
|
+
termination: Termination.UNSPECIFIED,
|
|
620
|
+
failureClass: FailureClass.UNSPECIFIED,
|
|
621
|
+
exitCode: undefined,
|
|
622
|
+
message: "",
|
|
623
|
+
kv: {},
|
|
624
|
+
chunks: [],
|
|
625
|
+
logFinal: false,
|
|
626
|
+
logTruncated: false,
|
|
627
|
+
completedAt: undefined,
|
|
628
|
+
};
|
|
629
|
+
n.attempts.push(a);
|
|
630
|
+
this.attemptsById.set(a.id, a);
|
|
631
|
+
this.notify();
|
|
632
|
+
}
|
|
633
|
+
claim(pool, handlers, workerId, max) {
|
|
634
|
+
const now = Date.now();
|
|
635
|
+
const out = [];
|
|
636
|
+
for (const run of this.runs.values()) {
|
|
637
|
+
if (run.tenant !== pool.owner?.tenant || run.cancelRequestedAt !== undefined || run.notBefore > now)
|
|
638
|
+
continue;
|
|
639
|
+
for (const n of run.nodes.values()) {
|
|
640
|
+
if (!handlers.includes(n.handler))
|
|
641
|
+
continue;
|
|
642
|
+
for (const a of n.attempts) {
|
|
643
|
+
if (out.length >= max)
|
|
644
|
+
return out;
|
|
645
|
+
if (a.state !== AttemptState.OFFERED || a.notBefore > now)
|
|
646
|
+
continue;
|
|
647
|
+
a.state = AttemptState.CLAIMED;
|
|
648
|
+
a.epoch += 1n;
|
|
649
|
+
a.workerId = workerId;
|
|
650
|
+
a.poolId = pool.id;
|
|
651
|
+
a.leaseExpiresAt = now + this.opts.leaseMs;
|
|
652
|
+
out.push(a);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
return out;
|
|
657
|
+
}
|
|
658
|
+
describe(a) {
|
|
659
|
+
const run = this.run(a.runId);
|
|
660
|
+
const n = run.nodes.get(a.nodeId);
|
|
661
|
+
const pkg = this.packages.get(n.handler);
|
|
662
|
+
const version = pkg ? [...pkg.versions.keys()].at(-1) : "0.0.0";
|
|
663
|
+
return create(ClaimedAttemptSchema, {
|
|
664
|
+
ref: { attemptId: a.id, claimEpoch: a.epoch },
|
|
665
|
+
runId: run.id,
|
|
666
|
+
nodeId: n.id,
|
|
667
|
+
number: a.number,
|
|
668
|
+
tenant: run.tenant,
|
|
669
|
+
leaseExpiresAt: timestampFromMs(a.leaseExpiresAt),
|
|
670
|
+
heartbeatInterval: durationFromMs(this.opts.heartbeatIntervalMs),
|
|
671
|
+
package: { name: n.handler, version },
|
|
672
|
+
...(pkg?.versions.get(version) ? { manifest: pkg.versions.get(version) } : {}),
|
|
673
|
+
inputs: a.inputs,
|
|
674
|
+
...(n.timeoutMs > 0 ? { timeout: durationFromMs(n.timeoutMs) } : {}),
|
|
675
|
+
cancelGrace: durationFromMs(this.opts.cancelGraceMs),
|
|
676
|
+
...(run.resource ? { resource: run.resource } : {}),
|
|
677
|
+
variables: run.variables,
|
|
678
|
+
runStartedAt: timestampFromMs(run.startedAt ?? Date.now()),
|
|
679
|
+
logMaxBytes: BigInt(this.opts.logMaxBytes),
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
started(a) {
|
|
683
|
+
const n = this.run(a.runId).nodes.get(a.nodeId);
|
|
684
|
+
const startedAt = a.startedAt ?? Date.now();
|
|
685
|
+
return {
|
|
686
|
+
attemptTokenSecret: `fake-attempt-token.${a.id}.${a.epoch}`,
|
|
687
|
+
...(n.timeoutMs > 0
|
|
688
|
+
? {
|
|
689
|
+
deadline: timestampFromMs(startedAt + n.timeoutMs),
|
|
690
|
+
tokenExpiresAt: timestampFromMs(startedAt + n.timeoutMs + this.opts.cancelGraceMs),
|
|
691
|
+
}
|
|
692
|
+
: { tokenExpiresAt: timestampFromMs(Date.now() + 86_400_000) }),
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
/** After an attempt ends: settle the node, or offer a retry. */
|
|
696
|
+
settle(a) {
|
|
697
|
+
const run = this.run(a.runId);
|
|
698
|
+
const n = run.nodes.get(a.nodeId);
|
|
699
|
+
if (a.state === AttemptState.SUCCEEDED) {
|
|
700
|
+
n.state = NodeState.SUCCEEDED;
|
|
701
|
+
n.kv = a.kv;
|
|
702
|
+
}
|
|
703
|
+
else if (a.state === AttemptState.CANCELLED || run.cancelRequestedAt !== undefined) {
|
|
704
|
+
n.state = NodeState.CANCELLED;
|
|
705
|
+
n.failureClass = FailureClass.CANCELLED;
|
|
706
|
+
}
|
|
707
|
+
else {
|
|
708
|
+
const when = n.retry.when.length > 0 ? n.retry.when : TRANSIENT;
|
|
709
|
+
const retries = n.attempts.length - 1;
|
|
710
|
+
if (when.includes(a.failureClass) && a.failureClass !== FailureClass.CANCELLED && retries < n.retry.max) {
|
|
711
|
+
this.offer(run, n, a.inputs, this.opts.retryBackoffMs);
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
n.state = NodeState.FAILED;
|
|
715
|
+
n.failureClass = a.failureClass;
|
|
716
|
+
}
|
|
717
|
+
this.advance(run);
|
|
718
|
+
this.notify();
|
|
719
|
+
}
|
|
720
|
+
cancel(run, reason) {
|
|
721
|
+
if (terminal(run.state) || run.cancelRequestedAt !== undefined)
|
|
722
|
+
return;
|
|
723
|
+
run.cancelRequestedAt = Date.now();
|
|
724
|
+
run.cancelReason = reason;
|
|
725
|
+
run.state = RunState.CANCELLING;
|
|
726
|
+
for (const n of run.nodes.values()) {
|
|
727
|
+
for (const a of n.attempts) {
|
|
728
|
+
if (a.state === AttemptState.OFFERED) {
|
|
729
|
+
a.state = AttemptState.CANCELLED;
|
|
730
|
+
a.failureClass = FailureClass.CANCELLED;
|
|
731
|
+
a.completedAt = Date.now();
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
const live = n.attempts.some((a) => a.state === AttemptState.CLAIMED || a.state === AttemptState.RUNNING);
|
|
735
|
+
if (!terminalNode(n.state) && !live) {
|
|
736
|
+
n.state = n.state === NodeState.PENDING ? NodeState.SKIPPED : NodeState.CANCELLED;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
this.advance(run);
|
|
740
|
+
this.notify();
|
|
741
|
+
}
|
|
742
|
+
/** The reaper and the delay clock. */
|
|
743
|
+
tick() {
|
|
744
|
+
const now = Date.now();
|
|
745
|
+
for (const run of this.runs.values()) {
|
|
746
|
+
if (run.state === RunState.QUEUED && run.notBefore <= now && run.cancelRequestedAt === undefined)
|
|
747
|
+
this.advance(run);
|
|
748
|
+
for (const n of run.nodes.values()) {
|
|
749
|
+
for (const a of n.attempts) {
|
|
750
|
+
if (a.leaseExpiresAt > now)
|
|
751
|
+
continue;
|
|
752
|
+
if (a.state === AttemptState.CLAIMED) {
|
|
753
|
+
// Nothing ran: offered again, epoch bumped, no retry spent.
|
|
754
|
+
a.state = run.cancelRequestedAt !== undefined ? AttemptState.CANCELLED : AttemptState.OFFERED;
|
|
755
|
+
a.epoch += 1n;
|
|
756
|
+
a.workerId = "";
|
|
757
|
+
a.poolId = "";
|
|
758
|
+
if (a.state === AttemptState.CANCELLED) {
|
|
759
|
+
a.completedAt = now;
|
|
760
|
+
this.settle(a);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
else if (a.state === AttemptState.RUNNING) {
|
|
764
|
+
a.state = AttemptState.ABANDONED;
|
|
765
|
+
a.failureClass = FailureClass.ABANDONED;
|
|
766
|
+
a.completedAt = now;
|
|
767
|
+
this.settle(a);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
this.notify();
|
|
773
|
+
}
|
|
774
|
+
notify() {
|
|
775
|
+
const prev = this.changed;
|
|
776
|
+
this.changed = signal();
|
|
777
|
+
prev.resolve();
|
|
778
|
+
}
|
|
779
|
+
// ---- messages ---------------------------------------------------------
|
|
780
|
+
runMessage(r) {
|
|
781
|
+
return create(RunSchema, {
|
|
782
|
+
id: r.id,
|
|
783
|
+
tenant: r.tenant,
|
|
784
|
+
product: r.product,
|
|
785
|
+
idempotencyKey: r.idempotencyKey,
|
|
786
|
+
...(r.resource ? { resource: r.resource } : {}),
|
|
787
|
+
variables: r.variables,
|
|
788
|
+
state: r.state,
|
|
789
|
+
failureClass: r.failureClass,
|
|
790
|
+
failedNodeId: r.failedNodeId,
|
|
791
|
+
createdBy: r.createdBy,
|
|
792
|
+
createdAt: timestampFromMs(r.createdAt),
|
|
793
|
+
...(r.notBefore > r.createdAt ? { notBefore: timestampFromMs(r.notBefore) } : {}),
|
|
794
|
+
...(r.startedAt ? { startedAt: timestampFromMs(r.startedAt) } : {}),
|
|
795
|
+
...(r.finishedAt ? { finishedAt: timestampFromMs(r.finishedAt) } : {}),
|
|
796
|
+
...(r.cancelRequestedAt ? { cancelRequestedAt: timestampFromMs(r.cancelRequestedAt) } : {}),
|
|
797
|
+
cancelReason: r.cancelReason,
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
nodeMessage(n) {
|
|
801
|
+
const settled = n.attempts.find((a) => a.state === AttemptState.SUCCEEDED) ?? n.attempts.at(-1);
|
|
802
|
+
return create(NodeRunSchema, {
|
|
803
|
+
nodeId: n.id,
|
|
804
|
+
state: n.state,
|
|
805
|
+
failureClass: n.failureClass,
|
|
806
|
+
settledAttemptId: terminalNode(n.state) ? (settled?.id ?? "") : "",
|
|
807
|
+
kv: Object.fromEntries(Object.entries(n.kv).map(([k, v]) => [k, jsonValue(v)])),
|
|
808
|
+
attempts: n.attempts.map((a) => create(AttemptSchema, {
|
|
809
|
+
id: a.id,
|
|
810
|
+
runId: a.runId,
|
|
811
|
+
nodeId: a.nodeId,
|
|
812
|
+
number: a.number,
|
|
813
|
+
state: a.state,
|
|
814
|
+
poolId: a.poolId,
|
|
815
|
+
runnerId: a.workerId,
|
|
816
|
+
failureClass: a.failureClass,
|
|
817
|
+
termination: a.termination,
|
|
818
|
+
...(a.exitCode !== undefined ? { exitCode: a.exitCode } : {}),
|
|
819
|
+
message: a.message,
|
|
820
|
+
logBytes: BigInt(a.chunks.reduce((s, c) => s + c.length, 0)),
|
|
821
|
+
logTruncated: a.logTruncated,
|
|
822
|
+
})),
|
|
823
|
+
});
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
function nodeSpec(n, requireHandler) {
|
|
827
|
+
let handler = "";
|
|
828
|
+
let uses = "";
|
|
829
|
+
if (n.action.case === "handler") {
|
|
830
|
+
handler = n.action.value;
|
|
831
|
+
requireHandler(handler);
|
|
832
|
+
}
|
|
833
|
+
else if (n.action.case === "uses") {
|
|
834
|
+
uses = `${n.action.value.name}@${n.action.value.version}`;
|
|
835
|
+
}
|
|
836
|
+
else {
|
|
837
|
+
throw new ConnectError(`node ${n.id}: only handler and uses nodes`, Code.Unimplemented);
|
|
838
|
+
}
|
|
839
|
+
return {
|
|
840
|
+
id: n.id,
|
|
841
|
+
handler,
|
|
842
|
+
uses,
|
|
843
|
+
with: Object.fromEntries(Object.entries(n.with).map(([k, v]) => [k, toJson(ValueSchema, v)])),
|
|
844
|
+
needs: n.needs.map((x) => x.node),
|
|
845
|
+
retry: n.retry ? { max: n.retry.max, when: n.retry.when } : undefined,
|
|
846
|
+
timeoutMs: n.timeout ? durationMs(n.timeout) : 0,
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
/** The server's dispatch.Outcome. */
|
|
850
|
+
function outcome(t, exitCode) {
|
|
851
|
+
switch (t) {
|
|
852
|
+
case Termination.EXITED:
|
|
853
|
+
return exitCode === undefined || exitCode === 0
|
|
854
|
+
? [AttemptState.SUCCEEDED, FailureClass.UNSPECIFIED]
|
|
855
|
+
: [AttemptState.FAILED, FailureClass.EXIT_CODE];
|
|
856
|
+
case Termination.OOM_KILLED:
|
|
857
|
+
return [AttemptState.FAILED, FailureClass.OOM];
|
|
858
|
+
case Termination.TIMED_OUT:
|
|
859
|
+
return [AttemptState.FAILED, FailureClass.TIMEOUT];
|
|
860
|
+
case Termination.CANCELLED:
|
|
861
|
+
return [AttemptState.CANCELLED, FailureClass.CANCELLED];
|
|
862
|
+
case Termination.IMAGE_PULL_FAILED:
|
|
863
|
+
return [AttemptState.FAILED, FailureClass.IMAGE_PULL];
|
|
864
|
+
case Termination.STAGING_FAILED:
|
|
865
|
+
return [AttemptState.FAILED, FailureClass.STAGING_FAILED];
|
|
866
|
+
case Termination.SECRETS_FAILED:
|
|
867
|
+
return [AttemptState.FAILED, FailureClass.SECRETS_UNAVAILABLE];
|
|
868
|
+
case Termination.HANDLER_ERROR:
|
|
869
|
+
return [AttemptState.FAILED, FailureClass.HANDLER_ERROR];
|
|
870
|
+
default:
|
|
871
|
+
return [AttemptState.FAILED, FailureClass.RUNNER_CRASHED];
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
function stale() {
|
|
875
|
+
return new ConnectError("the attempt is not held by this caller at this claim epoch: stop it and report nothing more", Code.FailedPrecondition);
|
|
876
|
+
}
|
|
877
|
+
function terminal(s) {
|
|
878
|
+
return s === RunState.SUCCEEDED || s === RunState.FAILED || s === RunState.CANCELLED;
|
|
879
|
+
}
|
|
880
|
+
function terminalNode(s) {
|
|
881
|
+
return (s === NodeState.SUCCEEDED ||
|
|
882
|
+
s === NodeState.FAILED ||
|
|
883
|
+
s === NodeState.FAILED_ALLOWED ||
|
|
884
|
+
s === NodeState.SKIPPED ||
|
|
885
|
+
s === NodeState.CANCELLED);
|
|
886
|
+
}
|
|
887
|
+
function jsonValue(v) {
|
|
888
|
+
return fromJson(ValueSchema, v);
|
|
889
|
+
}
|
|
890
|
+
function signal() {
|
|
891
|
+
let resolve;
|
|
892
|
+
const promise = new Promise((r) => (resolve = r));
|
|
893
|
+
return { promise, resolve };
|
|
894
|
+
}
|
|
895
|
+
function sleepUntil(at) {
|
|
896
|
+
return new Promise((r) => {
|
|
897
|
+
const t = setTimeout(r, Math.max(0, at - Date.now()));
|
|
898
|
+
t.unref();
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
function aborted(s) {
|
|
902
|
+
return new Promise((r) => {
|
|
903
|
+
if (s.aborted)
|
|
904
|
+
r();
|
|
905
|
+
else
|
|
906
|
+
s.addEventListener("abort", () => r(), { once: true });
|
|
907
|
+
});
|
|
908
|
+
}
|
|
909
|
+
//# sourceMappingURL=index.js.map
|