@kici-dev/orchestrator 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/dispatcher.d.ts +9 -0
- package/dist/agent/host-roster-reaper.d.ts +42 -0
- package/dist/agent/host-roster.d.ts +110 -0
- package/dist/agent/registry.d.ts +65 -2
- package/dist/app.d.ts +3 -0
- package/dist/cli/commands/host.d.ts +13 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +23 -0
- package/dist/cli/service/compose.d.ts +2 -1
- package/dist/cli/service/index.d.ts +2 -2
- package/dist/cli/service/instance/manifest.d.ts +12 -0
- package/dist/cli/service/instance/resolve.d.ts +12 -2
- package/dist/cli/service/launchd.d.ts +4 -1
- package/dist/cli/service/platform-detect.d.ts +5 -0
- package/dist/cli/service/systemd.d.ts +2 -1
- package/dist/cli/service/types.d.ts +37 -0
- package/dist/cli/service/windows.d.ts +2 -1
- package/dist/cli.js +797 -202
- package/dist/cluster/coordinator.d.ts +5 -1
- package/dist/config/schema.d.ts +4 -0
- package/dist/config/types.d.ts +9 -0
- package/dist/config.d.ts +6 -0
- package/dist/db/migrations/039_host_roster.d.ts +19 -0
- package/dist/db/migrations/040_runsonall_pin.d.ts +4 -0
- package/dist/db/migrations/041_wave_gated.d.ts +4 -0
- package/dist/db/migrations/042_dispatch_queue_patterns.d.ts +4 -0
- package/dist/db/types.d.ts +61 -0
- package/dist/diagnostics/fleet-collector.d.ts +1 -1
- package/dist/environments/held-runs.d.ts +9 -0
- package/dist/lockfile-redos-guard.d.ts +19 -0
- package/dist/metrics/prometheus.d.ts +13 -0
- package/dist/metrics/scheduled-jobs.d.ts +2 -2
- package/dist/orchestrator-core.d.ts +46 -1
- package/dist/pipeline/dispatch-matched-workflow.d.ts +42 -1
- package/dist/pipeline/processor.d.ts +25 -0
- package/dist/pipeline/wave-scheduler.d.ts +60 -0
- package/dist/queue/job-queue.d.ts +61 -2
- package/dist/reporting/execution-tracker.d.ts +28 -0
- package/dist/server.js +52760 -51263
- package/dist/standalone.js +1953 -500
- package/installer-image-digests.json +3 -3
- package/package.json +22 -22
- package/sbom.spdx.json +3227 -6682
package/dist/standalone.js
CHANGED
|
@@ -8,7 +8,7 @@ import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/s
|
|
|
8
8
|
import WebSocket from "ws";
|
|
9
9
|
import * as fs from "node:fs";
|
|
10
10
|
import { chmodSync, closeSync, createReadStream, createWriteStream, mkdirSync, mkdtempSync, openSync, promises, readFileSync, rmSync, statSync, unlinkSync, unwatchFile, watchFile, writeFileSync } from "node:fs";
|
|
11
|
-
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CacheRefScope, CheckRunConclusion, EnvDeleteErrorCode, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, FanoutError, HoldScope, IfFailedPolicy, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, LockFileParseError, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, TimeoutReason, TriggerSource, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_DISPATCH_ACK_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, materializeFanout, materializeResolvedMatrix, matrixEnvelopeFields, minAccessLogWarmDays, minSecretAuditLogWarmDays, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve } from "@kici-dev/engine";
|
|
11
|
+
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CacheRefScope, CheckRunConclusion, EnvDeleteErrorCode, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, FanoutError, HoldScope, IfFailedPolicy, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, LockFileParseError, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, TimeoutReason, TriggerSource, VariantKind, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_DISPATCH_ACK_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, matcherSatisfiedBy, materializeFanout, materializeResolvedHosts, materializeResolvedMatrix, matrixEnvelopeFields, minAccessLogWarmDays, minSecretAuditLogWarmDays, partitionMatchers, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve } from "@kici-dev/engine";
|
|
12
12
|
import { access, appendFile, chmod, constants, copyFile, link, mkdir, mkdtemp, open, readFile, readdir, rename, rm, stat, statfs, writeFile } from "node:fs/promises";
|
|
13
13
|
import * as path$1 from "node:path";
|
|
14
14
|
import path, { dirname, join, relative, resolve } from "node:path";
|
|
@@ -19,6 +19,7 @@ import { serve } from "@hono/node-server";
|
|
|
19
19
|
import { parse } from "yaml";
|
|
20
20
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
21
21
|
import { LRUCache } from "lru-cache";
|
|
22
|
+
import { assertMatchersSafe } from "@kici-dev/engine/labels/compile";
|
|
22
23
|
import { bodyLimit } from "hono/body-limit";
|
|
23
24
|
import { createNodeWebSocket } from "@hono/node-ws";
|
|
24
25
|
import { getConnInfo } from "@hono/node-server/conninfo";
|
|
@@ -41,7 +42,7 @@ import { JSONPath } from "jsonpath-plus";
|
|
|
41
42
|
import { execFile, spawn } from "node:child_process";
|
|
42
43
|
import { promisify } from "node:util";
|
|
43
44
|
import vm from "node:vm";
|
|
44
|
-
import
|
|
45
|
+
import { ZipArchive } from "archiver";
|
|
45
46
|
import { createInterface } from "node:readline";
|
|
46
47
|
import Docker from "dockerode";
|
|
47
48
|
import http from "node:http";
|
|
@@ -206,6 +207,9 @@ var init_config$5 = __esmMin((() => {
|
|
|
206
207
|
pgCustomerSecrets: z.enum(["true", "false"]).default("true").transform((v) => v === "true"),
|
|
207
208
|
agentAuth: z.enum(["token", "none"]).default("token"),
|
|
208
209
|
agentTokenTtlMs: z.coerce.number().default(36e5),
|
|
210
|
+
rosterGraceMs: z.coerce.number().int().min(1e3).default(3e5),
|
|
211
|
+
rosterTtlMs: z.coerce.number().int().min(1e3).default(18e5),
|
|
212
|
+
maxFanoutHosts: z.coerce.number().int().min(1).default(1024),
|
|
209
213
|
eventRouterMaxChainDepth: z.coerce.number().default(10),
|
|
210
214
|
eventRouterRateLimitPerWorkflowPerMinute: z.coerce.number().default(100),
|
|
211
215
|
eventRouterEventTtlSeconds: z.coerce.number().default(604800),
|
|
@@ -393,6 +397,9 @@ var init_config$5 = __esmMin((() => {
|
|
|
393
397
|
pgCustomerSecrets: "KICI_PG_CUSTOMER_SECRETS",
|
|
394
398
|
agentAuth: "KICI_AGENT_AUTH",
|
|
395
399
|
agentTokenTtlMs: "KICI_AGENT_TOKEN_TTL_MS",
|
|
400
|
+
rosterGraceMs: "KICI_ROSTER_GRACE_MS",
|
|
401
|
+
rosterTtlMs: "KICI_ROSTER_TTL_MS",
|
|
402
|
+
maxFanoutHosts: "KICI_MAX_FANOUT_HOSTS",
|
|
396
403
|
eventRouterMaxChainDepth: "KICI_EVENT_ROUTER_MAX_CHAIN_DEPTH",
|
|
397
404
|
eventRouterRateLimitPerWorkflowPerMinute: "KICI_EVENT_ROUTER_RATE_LIMIT_PER_WORKFLOW_PER_MINUTE",
|
|
398
405
|
eventRouterEventTtlSeconds: "KICI_EVENT_ROUTER_EVENT_TTL_SECONDS",
|
|
@@ -1053,12 +1060,12 @@ var init_peer_credentials = __esmMin((() => {
|
|
|
1053
1060
|
* Authentication uses ECDH key exchange followed by join token (first connect)
|
|
1054
1061
|
* or HMAC credential proof (reconnection).
|
|
1055
1062
|
*/
|
|
1056
|
-
var logger$
|
|
1063
|
+
var logger$89, SOFTWARE_VERSION$1, PeerClient$1;
|
|
1057
1064
|
var init_peer_client = __esmMin((() => {
|
|
1058
1065
|
init_peer_crypto();
|
|
1059
1066
|
init_peer_credentials();
|
|
1060
|
-
logger$
|
|
1061
|
-
SOFTWARE_VERSION$1 = "0.1.
|
|
1067
|
+
logger$89 = createLogger({ prefix: "peer-client" });
|
|
1068
|
+
SOFTWARE_VERSION$1 = "0.1.19";
|
|
1062
1069
|
PeerClient$1 = class {
|
|
1063
1070
|
ws = null;
|
|
1064
1071
|
_state = "disconnected";
|
|
@@ -1132,7 +1139,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1132
1139
|
*/
|
|
1133
1140
|
connect() {
|
|
1134
1141
|
if (this._state !== "disconnected") {
|
|
1135
|
-
logger$
|
|
1142
|
+
logger$89.warn("connect() called while not disconnected", { state: this._state });
|
|
1136
1143
|
return;
|
|
1137
1144
|
}
|
|
1138
1145
|
this.intentionalDisconnect = false;
|
|
@@ -1266,20 +1273,20 @@ var init_peer_client = __esmMin((() => {
|
|
|
1266
1273
|
}
|
|
1267
1274
|
});
|
|
1268
1275
|
} catch (err) {
|
|
1269
|
-
logger$
|
|
1276
|
+
logger$89.error("Failed to create peer WebSocket", { error: toErrorMessage(err) });
|
|
1270
1277
|
this._state = "disconnected";
|
|
1271
1278
|
this.scheduleReconnect();
|
|
1272
1279
|
return;
|
|
1273
1280
|
}
|
|
1274
1281
|
this.ws.on("open", () => {
|
|
1275
1282
|
this._state = "handshaking";
|
|
1276
|
-
logger$
|
|
1283
|
+
logger$89.info("Connected to peer, waiting for ECDH handshake", { url: this.url });
|
|
1277
1284
|
});
|
|
1278
1285
|
this.ws.on("message", (data) => {
|
|
1279
1286
|
this.handleMessage(data);
|
|
1280
1287
|
});
|
|
1281
1288
|
this.ws.on("close", (code, reason) => {
|
|
1282
|
-
logger$
|
|
1289
|
+
logger$89.info("Peer connection closed", {
|
|
1283
1290
|
code,
|
|
1284
1291
|
reason: reason.toString(),
|
|
1285
1292
|
targetInstanceId: this._targetInstanceId
|
|
@@ -1292,7 +1299,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1292
1299
|
if (!this.intentionalDisconnect) this.scheduleReconnect();
|
|
1293
1300
|
});
|
|
1294
1301
|
this.ws.on("error", (err) => {
|
|
1295
|
-
logger$
|
|
1302
|
+
logger$89.error("Peer WebSocket error", { error: err.message });
|
|
1296
1303
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
|
|
1297
1304
|
});
|
|
1298
1305
|
}
|
|
@@ -1303,12 +1310,12 @@ var init_peer_client = __esmMin((() => {
|
|
|
1303
1310
|
try {
|
|
1304
1311
|
parsed = JSON.parse(raw);
|
|
1305
1312
|
} catch {
|
|
1306
|
-
logger$
|
|
1313
|
+
logger$89.warn("Malformed JSON during handshake");
|
|
1307
1314
|
return;
|
|
1308
1315
|
}
|
|
1309
1316
|
const hello = peerHelloSchema.safeParse(parsed);
|
|
1310
1317
|
if (!hello.success) {
|
|
1311
|
-
logger$
|
|
1318
|
+
logger$89.warn("Expected peer.hello, got invalid message");
|
|
1312
1319
|
return;
|
|
1313
1320
|
}
|
|
1314
1321
|
const ecdh = generateEcdhKeyPair();
|
|
@@ -1317,7 +1324,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1317
1324
|
try {
|
|
1318
1325
|
this.sessionKey = deriveSessionKey(ecdh.privateKey, serverPubKey, nonce);
|
|
1319
1326
|
} catch (err) {
|
|
1320
|
-
logger$
|
|
1327
|
+
logger$89.error("ECDH key derivation failed", { error: toErrorMessage(err) });
|
|
1321
1328
|
if (this.ws) this.ws.close(1e3, "Key derivation failed");
|
|
1322
1329
|
return;
|
|
1323
1330
|
}
|
|
@@ -1327,48 +1334,48 @@ var init_peer_client = __esmMin((() => {
|
|
|
1327
1334
|
}));
|
|
1328
1335
|
this._state = "authenticating";
|
|
1329
1336
|
this.sendAuthRequest(nonce).catch((err) => {
|
|
1330
|
-
logger$
|
|
1337
|
+
logger$89.error("Failed to send auth request", { error: toErrorMessage(err) });
|
|
1331
1338
|
if (this.ws) this.ws.close(1e3, "Auth request failed");
|
|
1332
1339
|
});
|
|
1333
1340
|
return;
|
|
1334
1341
|
}
|
|
1335
1342
|
if (this._state === "authenticating") {
|
|
1336
1343
|
if (!this.sessionKey) {
|
|
1337
|
-
logger$
|
|
1344
|
+
logger$89.warn("No session key for auth response decryption");
|
|
1338
1345
|
return;
|
|
1339
1346
|
}
|
|
1340
1347
|
let decrypted;
|
|
1341
1348
|
try {
|
|
1342
1349
|
decrypted = decryptMessage(raw, this.sessionKey);
|
|
1343
1350
|
} catch {
|
|
1344
|
-
logger$
|
|
1351
|
+
logger$89.warn("Failed to decrypt auth response");
|
|
1345
1352
|
return;
|
|
1346
1353
|
}
|
|
1347
1354
|
let parsed;
|
|
1348
1355
|
try {
|
|
1349
1356
|
parsed = JSON.parse(decrypted);
|
|
1350
1357
|
} catch {
|
|
1351
|
-
logger$
|
|
1358
|
+
logger$89.warn("Malformed JSON in decrypted auth response");
|
|
1352
1359
|
return;
|
|
1353
1360
|
}
|
|
1354
1361
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
1355
1362
|
if (!msgResult.success) {
|
|
1356
|
-
logger$
|
|
1363
|
+
logger$89.warn("Invalid auth response", { errors: msgResult.error.issues });
|
|
1357
1364
|
return;
|
|
1358
1365
|
}
|
|
1359
1366
|
if (msgResult.data.type !== "peer.auth.response") {
|
|
1360
|
-
logger$
|
|
1367
|
+
logger$89.warn("Expected peer.auth.response, got", { type: msgResult.data.type });
|
|
1361
1368
|
return;
|
|
1362
1369
|
}
|
|
1363
1370
|
const msg = msgResult.data;
|
|
1364
1371
|
if (msg.accepted) {
|
|
1365
|
-
if (msg.softwareVersion) logger$
|
|
1372
|
+
if (msg.softwareVersion) logger$89.info("Coordinator software version", {
|
|
1366
1373
|
localVersion: SOFTWARE_VERSION$1,
|
|
1367
1374
|
coordinatorVersion: msg.softwareVersion
|
|
1368
1375
|
});
|
|
1369
1376
|
this._targetInstanceId = msg.instanceId ?? null;
|
|
1370
1377
|
if (msg.instanceId) this.onAuthenticated?.(msg.instanceId);
|
|
1371
|
-
logger$
|
|
1378
|
+
logger$89.info("Peer auth accepted", {
|
|
1372
1379
|
targetInstanceId: msg.instanceId,
|
|
1373
1380
|
agentCount: msg.agents?.length ?? 0,
|
|
1374
1381
|
scalerBackends: msg.scalerCapacity?.length ?? 0
|
|
@@ -1381,7 +1388,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1381
1388
|
role: msg.role ?? "coordinator",
|
|
1382
1389
|
issuedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1383
1390
|
}).catch((err) => {
|
|
1384
|
-
logger$
|
|
1391
|
+
logger$89.error("Failed to persist credential file", { error: toErrorMessage(err) });
|
|
1385
1392
|
});
|
|
1386
1393
|
this.peerRegistry.addPeer({
|
|
1387
1394
|
instanceId: msg.instanceId,
|
|
@@ -1409,12 +1416,12 @@ var init_peer_client = __esmMin((() => {
|
|
|
1409
1416
|
}
|
|
1410
1417
|
this.startHeartbeat();
|
|
1411
1418
|
} else {
|
|
1412
|
-
logger$
|
|
1419
|
+
logger$89.error("Peer auth rejected", { reason: msg.reason });
|
|
1413
1420
|
if (msg.reason === "Invalid proof" || msg.reason === "Unknown credential" || msg.reason === "Credential revoked") try {
|
|
1414
1421
|
unlinkSync(this.credentialFile);
|
|
1415
|
-
logger$
|
|
1422
|
+
logger$89.warn("Deleted stale credential file after server rejection", { reason: msg.reason });
|
|
1416
1423
|
} catch (err) {
|
|
1417
|
-
if (err.code !== "ENOENT") logger$
|
|
1424
|
+
if (err.code !== "ENOENT") logger$89.warn("Failed to delete stale credential file", {
|
|
1418
1425
|
error: toErrorMessage(err),
|
|
1419
1426
|
path: this.credentialFile
|
|
1420
1427
|
});
|
|
@@ -1428,19 +1435,19 @@ var init_peer_client = __esmMin((() => {
|
|
|
1428
1435
|
try {
|
|
1429
1436
|
decrypted = decryptMessage(raw, this.sessionKey);
|
|
1430
1437
|
} catch {
|
|
1431
|
-
logger$
|
|
1438
|
+
logger$89.warn("Failed to decrypt message from peer");
|
|
1432
1439
|
return;
|
|
1433
1440
|
}
|
|
1434
1441
|
let parsed;
|
|
1435
1442
|
try {
|
|
1436
1443
|
parsed = JSON.parse(decrypted);
|
|
1437
1444
|
} catch {
|
|
1438
|
-
logger$
|
|
1445
|
+
logger$89.warn("Malformed JSON from peer");
|
|
1439
1446
|
return;
|
|
1440
1447
|
}
|
|
1441
1448
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
1442
1449
|
if (!msgResult.success) {
|
|
1443
|
-
logger$
|
|
1450
|
+
logger$89.warn("Invalid message from peer", { errors: msgResult.error.issues });
|
|
1444
1451
|
return;
|
|
1445
1452
|
}
|
|
1446
1453
|
this.routeMessage(msgResult.data);
|
|
@@ -1482,7 +1489,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1482
1489
|
softwareVersion: SOFTWARE_VERSION$1,
|
|
1483
1490
|
role: this.role
|
|
1484
1491
|
};
|
|
1485
|
-
logger$
|
|
1492
|
+
logger$89.info("Sending credential-based auth request", {
|
|
1486
1493
|
targetUrl: this.url,
|
|
1487
1494
|
credentialInstanceId: cred.instanceId
|
|
1488
1495
|
});
|
|
@@ -1496,10 +1503,10 @@ var init_peer_client = __esmMin((() => {
|
|
|
1496
1503
|
softwareVersion: SOFTWARE_VERSION$1,
|
|
1497
1504
|
role: this.role
|
|
1498
1505
|
};
|
|
1499
|
-
logger$
|
|
1506
|
+
logger$89.info("Sending token-based auth request");
|
|
1500
1507
|
this.ws.send(encryptMessage(JSON.stringify(authRequest), this.sessionKey));
|
|
1501
1508
|
} else {
|
|
1502
|
-
logger$
|
|
1509
|
+
logger$89.error("No auth method available: no credential file and no join token");
|
|
1503
1510
|
if (this.ws.readyState === WebSocket.OPEN) this.ws.close(1e3, "No auth method");
|
|
1504
1511
|
}
|
|
1505
1512
|
}
|
|
@@ -1510,7 +1517,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1510
1517
|
break;
|
|
1511
1518
|
case "job.reroute":
|
|
1512
1519
|
this.onJobReroute(msg).catch((err) => {
|
|
1513
|
-
logger$
|
|
1520
|
+
logger$89.error("Error handling job reroute", { error: toErrorMessage(err) });
|
|
1514
1521
|
});
|
|
1515
1522
|
break;
|
|
1516
1523
|
case "job.reroute.ack": {
|
|
@@ -1518,7 +1525,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1518
1525
|
if (waiter) {
|
|
1519
1526
|
clearTimeout(waiter.timer);
|
|
1520
1527
|
this.ackWaiters.delete(msg.messageId);
|
|
1521
|
-
if (!msg.accepted) logger$
|
|
1528
|
+
if (!msg.accepted) logger$89.info("Reroute ACK rejected by peer", {
|
|
1522
1529
|
targetInstanceId: this._targetInstanceId,
|
|
1523
1530
|
reason: msg.reason
|
|
1524
1531
|
});
|
|
@@ -1548,7 +1555,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1548
1555
|
if (this.onPeerCacheUploadRequest) this.onPeerCacheUploadRequest(msg).then((response) => {
|
|
1549
1556
|
this.send(response);
|
|
1550
1557
|
}).catch((err) => {
|
|
1551
|
-
logger$
|
|
1558
|
+
logger$89.error("Error handling cache upload request", { error: toErrorMessage(err) });
|
|
1552
1559
|
});
|
|
1553
1560
|
break;
|
|
1554
1561
|
case "peer.cache.upload.response": {
|
|
@@ -1561,7 +1568,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1561
1568
|
break;
|
|
1562
1569
|
}
|
|
1563
1570
|
case "peer.auth.request":
|
|
1564
|
-
logger$
|
|
1571
|
+
logger$89.warn("Unexpected peer.auth.request on outgoing connection");
|
|
1565
1572
|
break;
|
|
1566
1573
|
case "raft.vote.response":
|
|
1567
1574
|
this.onRaftVoteResponse?.(msg);
|
|
@@ -1593,7 +1600,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1593
1600
|
handler(msg).then((result) => {
|
|
1594
1601
|
sendReply(result);
|
|
1595
1602
|
}).catch((err) => {
|
|
1596
|
-
logger$
|
|
1603
|
+
logger$89.error("Error executing peer config reload", { error: toErrorMessage(err) });
|
|
1597
1604
|
sendReply({
|
|
1598
1605
|
success: false,
|
|
1599
1606
|
errors: [toErrorMessage(err)]
|
|
@@ -1643,7 +1650,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1643
1650
|
this.cancelReconnect();
|
|
1644
1651
|
const delay = this.getReconnectDelay();
|
|
1645
1652
|
this.reconnectAttempts++;
|
|
1646
|
-
logger$
|
|
1653
|
+
logger$89.info("Scheduling peer reconnect", {
|
|
1647
1654
|
attempt: this.reconnectAttempts,
|
|
1648
1655
|
delayMs: Math.round(delay)
|
|
1649
1656
|
});
|
|
@@ -1950,7 +1957,7 @@ function createPeerHandler(deps) {
|
|
|
1950
1957
|
break;
|
|
1951
1958
|
case "job.reroute":
|
|
1952
1959
|
onJobReroute(msg).catch((err) => {
|
|
1953
|
-
logger$
|
|
1960
|
+
logger$88.error("Error handling job reroute from peer", { error: toErrorMessage(err) });
|
|
1954
1961
|
});
|
|
1955
1962
|
break;
|
|
1956
1963
|
case "job.reroute.ack": {
|
|
@@ -1987,7 +1994,7 @@ function createPeerHandler(deps) {
|
|
|
1987
1994
|
if (onPeerCacheUploadRequest) onPeerCacheUploadRequest(msg, conn.peerInstanceId).then((response) => {
|
|
1988
1995
|
sendEncryptedMessage(conn.ws, conn.sessionKey, response);
|
|
1989
1996
|
}).catch((err) => {
|
|
1990
|
-
logger$
|
|
1997
|
+
logger$88.error("Error handling cache upload request from peer", {
|
|
1991
1998
|
peerId: conn.peerInstanceId,
|
|
1992
1999
|
error: toErrorMessage(err)
|
|
1993
2000
|
});
|
|
@@ -2008,13 +2015,13 @@ function createPeerHandler(deps) {
|
|
|
2008
2015
|
});
|
|
2009
2016
|
break;
|
|
2010
2017
|
case "peer.cache.upload.response":
|
|
2011
|
-
logger$
|
|
2018
|
+
logger$88.warn("Unexpected peer.cache.upload.response on incoming connection");
|
|
2012
2019
|
break;
|
|
2013
2020
|
case "peer.auth.request":
|
|
2014
|
-
logger$
|
|
2021
|
+
logger$88.warn("Unexpected peer.auth.request after authentication");
|
|
2015
2022
|
break;
|
|
2016
2023
|
case "peer.auth.response":
|
|
2017
|
-
logger$
|
|
2024
|
+
logger$88.warn("Unexpected peer.auth.response on incoming connection");
|
|
2018
2025
|
break;
|
|
2019
2026
|
case "raft.vote.response":
|
|
2020
2027
|
onRaftVoteResponse?.(msg);
|
|
@@ -2046,7 +2053,7 @@ function createPeerHandler(deps) {
|
|
|
2046
2053
|
handler(msg).then((result) => {
|
|
2047
2054
|
sendResponse(result);
|
|
2048
2055
|
}).catch((err) => {
|
|
2049
|
-
logger$
|
|
2056
|
+
logger$88.error("Error executing peer config reload", {
|
|
2050
2057
|
peerId: conn.peerInstanceId,
|
|
2051
2058
|
error: toErrorMessage(err)
|
|
2052
2059
|
});
|
|
@@ -2088,7 +2095,7 @@ function createPeerHandler(deps) {
|
|
|
2088
2095
|
function handleConnection(ws, remoteIp) {
|
|
2089
2096
|
const ip = remoteIp ?? "unknown";
|
|
2090
2097
|
if (isRateLimited(ip)) {
|
|
2091
|
-
logger$
|
|
2098
|
+
logger$88.warn("Rate limited peer connection attempt", { ip });
|
|
2092
2099
|
ws.close(WS_CLOSE_UNAUTHORIZED, "Rate limited");
|
|
2093
2100
|
return;
|
|
2094
2101
|
}
|
|
@@ -2098,7 +2105,7 @@ function createPeerHandler(deps) {
|
|
|
2098
2105
|
let handshakeNonce = null;
|
|
2099
2106
|
const authTimer = setTimeout(() => {
|
|
2100
2107
|
if (!authenticated) {
|
|
2101
|
-
logger$
|
|
2108
|
+
logger$88.warn("Peer auth timeout, closing connection");
|
|
2102
2109
|
ws.close(WS_CLOSE_AUTH_TIMEOUT, "Auth timeout");
|
|
2103
2110
|
}
|
|
2104
2111
|
}, authTimeoutMs);
|
|
@@ -2118,12 +2125,12 @@ function createPeerHandler(deps) {
|
|
|
2118
2125
|
try {
|
|
2119
2126
|
parsed = JSON.parse(raw);
|
|
2120
2127
|
} catch {
|
|
2121
|
-
logger$
|
|
2128
|
+
logger$88.warn("Malformed JSON during ECDH handshake");
|
|
2122
2129
|
return;
|
|
2123
2130
|
}
|
|
2124
2131
|
const helloResp = peerHelloResponseSchema.safeParse(parsed);
|
|
2125
2132
|
if (!helloResp.success) {
|
|
2126
|
-
logger$
|
|
2133
|
+
logger$88.warn("Expected peer.hello.response, got invalid message");
|
|
2127
2134
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Expected hello response");
|
|
2128
2135
|
clearTimeout(authTimer);
|
|
2129
2136
|
return;
|
|
@@ -2131,7 +2138,7 @@ function createPeerHandler(deps) {
|
|
|
2131
2138
|
try {
|
|
2132
2139
|
sessionKey = deriveSessionKey(ecdh.privateKey, Buffer.from(helloResp.data.ephemeralPublicKey, "base64"), nonce);
|
|
2133
2140
|
} catch (err) {
|
|
2134
|
-
logger$
|
|
2141
|
+
logger$88.warn("ECDH key derivation failed", { error: toErrorMessage(err) });
|
|
2135
2142
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Key derivation failed");
|
|
2136
2143
|
clearTimeout(authTimer);
|
|
2137
2144
|
return;
|
|
@@ -2141,7 +2148,7 @@ function createPeerHandler(deps) {
|
|
|
2141
2148
|
}
|
|
2142
2149
|
if (!authenticated) {
|
|
2143
2150
|
if (!sessionKey) {
|
|
2144
|
-
logger$
|
|
2151
|
+
logger$88.warn("No session key for auth decryption");
|
|
2145
2152
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "No session key");
|
|
2146
2153
|
clearTimeout(authTimer);
|
|
2147
2154
|
return;
|
|
@@ -2150,7 +2157,7 @@ function createPeerHandler(deps) {
|
|
|
2150
2157
|
try {
|
|
2151
2158
|
decrypted = decryptMessage(raw, sessionKey);
|
|
2152
2159
|
} catch (err) {
|
|
2153
|
-
logger$
|
|
2160
|
+
logger$88.warn("Failed to decrypt auth request", { error: toErrorMessage(err) });
|
|
2154
2161
|
recordFailedAuth(ip);
|
|
2155
2162
|
ws.close(WS_CLOSE_UNAUTHORIZED, "Decryption failed");
|
|
2156
2163
|
clearTimeout(authTimer);
|
|
@@ -2160,14 +2167,14 @@ function createPeerHandler(deps) {
|
|
|
2160
2167
|
try {
|
|
2161
2168
|
authParsed = JSON.parse(decrypted);
|
|
2162
2169
|
} catch {
|
|
2163
|
-
logger$
|
|
2170
|
+
logger$88.warn("Malformed JSON in decrypted auth request");
|
|
2164
2171
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Invalid auth format");
|
|
2165
2172
|
clearTimeout(authTimer);
|
|
2166
2173
|
return;
|
|
2167
2174
|
}
|
|
2168
2175
|
const authMsg = peerAuthRequestSchema.safeParse(authParsed);
|
|
2169
2176
|
if (!authMsg.success) {
|
|
2170
|
-
logger$
|
|
2177
|
+
logger$88.warn("Invalid peer.auth.request format", { errors: authMsg.error.issues });
|
|
2171
2178
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Invalid auth request");
|
|
2172
2179
|
clearTimeout(authTimer);
|
|
2173
2180
|
return;
|
|
@@ -2177,7 +2184,7 @@ function createPeerHandler(deps) {
|
|
|
2177
2184
|
authenticated = true;
|
|
2178
2185
|
peerInstanceId = authMsg.data.instanceId;
|
|
2179
2186
|
clearTimeout(authTimer);
|
|
2180
|
-
logger$
|
|
2187
|
+
logger$88.info("Peer authenticated", { peerInstanceId });
|
|
2181
2188
|
peerRegistry.addPeer({
|
|
2182
2189
|
instanceId: peerInstanceId,
|
|
2183
2190
|
connectionId: randomUUID(),
|
|
@@ -2199,7 +2206,7 @@ function createPeerHandler(deps) {
|
|
|
2199
2206
|
connections.set(peerInstanceId, conn);
|
|
2200
2207
|
startHeartbeat(conn);
|
|
2201
2208
|
}).catch((err) => {
|
|
2202
|
-
logger$
|
|
2209
|
+
logger$88.error("Unexpected error during auth handling", { error: toErrorMessage(err) });
|
|
2203
2210
|
ws.close(WS_CLOSE_UNAUTHORIZED, "Auth error");
|
|
2204
2211
|
clearTimeout(authTimer);
|
|
2205
2212
|
});
|
|
@@ -2210,19 +2217,19 @@ function createPeerHandler(deps) {
|
|
|
2210
2217
|
try {
|
|
2211
2218
|
decrypted = decryptMessage(raw, sessionKey);
|
|
2212
2219
|
} catch {
|
|
2213
|
-
logger$
|
|
2220
|
+
logger$88.warn("Failed to decrypt message from authenticated peer");
|
|
2214
2221
|
return;
|
|
2215
2222
|
}
|
|
2216
2223
|
let parsed;
|
|
2217
2224
|
try {
|
|
2218
2225
|
parsed = JSON.parse(decrypted);
|
|
2219
2226
|
} catch {
|
|
2220
|
-
logger$
|
|
2227
|
+
logger$88.warn("Malformed JSON from peer connection");
|
|
2221
2228
|
return;
|
|
2222
2229
|
}
|
|
2223
2230
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
2224
2231
|
if (!msgResult.success) {
|
|
2225
|
-
logger$
|
|
2232
|
+
logger$88.warn("Invalid message from peer", { errors: msgResult.error.issues });
|
|
2226
2233
|
return;
|
|
2227
2234
|
}
|
|
2228
2235
|
const conn = connections.get(peerInstanceId);
|
|
@@ -2231,14 +2238,14 @@ function createPeerHandler(deps) {
|
|
|
2231
2238
|
ws.on("close", () => {
|
|
2232
2239
|
clearTimeout(authTimer);
|
|
2233
2240
|
if (peerInstanceId) {
|
|
2234
|
-
logger$
|
|
2241
|
+
logger$88.info("Peer disconnected", { peerInstanceId });
|
|
2235
2242
|
rejectLogsCollectForPeer(peerInstanceId);
|
|
2236
2243
|
const conn = connections.get(peerInstanceId);
|
|
2237
2244
|
if (conn) cleanupConnection(conn);
|
|
2238
2245
|
}
|
|
2239
2246
|
});
|
|
2240
2247
|
ws.on("error", (err) => {
|
|
2241
|
-
logger$
|
|
2248
|
+
logger$88.error("Peer connection error", { error: toErrorMessage(err) });
|
|
2242
2249
|
});
|
|
2243
2250
|
}
|
|
2244
2251
|
/**
|
|
@@ -2247,7 +2254,7 @@ function createPeerHandler(deps) {
|
|
|
2247
2254
|
*/
|
|
2248
2255
|
async function handleAuth(ws, sessionKey, nonce, authMsg, ip) {
|
|
2249
2256
|
if (authMsg.protocolVersion < MIN_PROTOCOL_VERSION) {
|
|
2250
|
-
logger$
|
|
2257
|
+
logger$88.warn("Peer protocol version below minimum", {
|
|
2251
2258
|
peerInstanceId: authMsg.instanceId,
|
|
2252
2259
|
received: authMsg.protocolVersion,
|
|
2253
2260
|
minimum: MIN_PROTOCOL_VERSION
|
|
@@ -2263,7 +2270,7 @@ function createPeerHandler(deps) {
|
|
|
2263
2270
|
ws.close(WS_CLOSE_PROTOCOL_ERROR, "Unsupported protocol version");
|
|
2264
2271
|
return false;
|
|
2265
2272
|
}
|
|
2266
|
-
if (authMsg.softwareVersion) logger$
|
|
2273
|
+
if (authMsg.softwareVersion) logger$88.info("Peer software version", {
|
|
2267
2274
|
peerInstanceId: authMsg.instanceId,
|
|
2268
2275
|
localVersion: SOFTWARE_VERSION,
|
|
2269
2276
|
remoteVersion: authMsg.softwareVersion
|
|
@@ -2271,7 +2278,7 @@ function createPeerHandler(deps) {
|
|
|
2271
2278
|
if (authMsg.token) try {
|
|
2272
2279
|
const result = await tokenManager.validateAndConsumeToken(authMsg.token, instanceId);
|
|
2273
2280
|
if (!acceptedRoles.includes(result.routing.role)) {
|
|
2274
|
-
logger$
|
|
2281
|
+
logger$88.warn("Peer role mismatch", {
|
|
2275
2282
|
peerInstanceId: authMsg.instanceId,
|
|
2276
2283
|
accepted: acceptedRoles,
|
|
2277
2284
|
actual: result.routing.role
|
|
@@ -2315,7 +2322,7 @@ function createPeerHandler(deps) {
|
|
|
2315
2322
|
const existing = await credentialStore.findByInstanceId(authMsg.instanceId);
|
|
2316
2323
|
if (existing && !existing.revokedAt && existing.sourceTokenHash === presentedHash) {
|
|
2317
2324
|
if (!acceptedRoles.includes(parsed.routing.role)) {
|
|
2318
|
-
logger$
|
|
2325
|
+
logger$88.warn("Peer role mismatch on token retry", {
|
|
2319
2326
|
peerInstanceId: authMsg.instanceId,
|
|
2320
2327
|
accepted: acceptedRoles,
|
|
2321
2328
|
actual: parsed.routing.role
|
|
@@ -2339,7 +2346,7 @@ function createPeerHandler(deps) {
|
|
|
2339
2346
|
routingKeys: [parsed.routing.routingKey],
|
|
2340
2347
|
sourceTokenHash: presentedHash
|
|
2341
2348
|
});
|
|
2342
|
-
logger$
|
|
2349
|
+
logger$88.info("Peer idempotent token retry accepted", {
|
|
2343
2350
|
peerInstanceId: authMsg.instanceId,
|
|
2344
2351
|
sourceTokenHash: presentedHash
|
|
2345
2352
|
});
|
|
@@ -2358,12 +2365,12 @@ function createPeerHandler(deps) {
|
|
|
2358
2365
|
return true;
|
|
2359
2366
|
}
|
|
2360
2367
|
} catch (recoveryErr) {
|
|
2361
|
-
logger$
|
|
2368
|
+
logger$88.warn("Peer token idempotent recovery failed", {
|
|
2362
2369
|
peerInstanceId: authMsg.instanceId,
|
|
2363
2370
|
error: toErrorMessage(recoveryErr)
|
|
2364
2371
|
});
|
|
2365
2372
|
}
|
|
2366
|
-
logger$
|
|
2373
|
+
logger$88.warn("Peer token validation failed", {
|
|
2367
2374
|
peerInstanceId: authMsg.instanceId,
|
|
2368
2375
|
error: toErrorMessage(err)
|
|
2369
2376
|
});
|
|
@@ -2380,7 +2387,7 @@ function createPeerHandler(deps) {
|
|
|
2380
2387
|
else if (authMsg.proof) {
|
|
2381
2388
|
const stored = await credentialStore.findByInstanceId(authMsg.instanceId);
|
|
2382
2389
|
if (!stored) {
|
|
2383
|
-
logger$
|
|
2390
|
+
logger$88.warn("Peer credential not found", { peerInstanceId: authMsg.instanceId });
|
|
2384
2391
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2385
2392
|
type: "peer.auth.response",
|
|
2386
2393
|
accepted: false,
|
|
@@ -2392,7 +2399,7 @@ function createPeerHandler(deps) {
|
|
|
2392
2399
|
return false;
|
|
2393
2400
|
}
|
|
2394
2401
|
if (stored.revokedAt) {
|
|
2395
|
-
logger$
|
|
2402
|
+
logger$88.warn("Peer credential revoked", { peerInstanceId: authMsg.instanceId });
|
|
2396
2403
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2397
2404
|
type: "peer.auth.response",
|
|
2398
2405
|
accepted: false,
|
|
@@ -2407,7 +2414,7 @@ function createPeerHandler(deps) {
|
|
|
2407
2414
|
const expectedProof = createHmac("sha256", Buffer.from(stored.credentialHash, "hex")).update(nonceB64 + ":" + authMsg.instanceId).digest();
|
|
2408
2415
|
const proofBuffer = Buffer.from(authMsg.proof, "hex");
|
|
2409
2416
|
if (proofBuffer.length !== expectedProof.length || !timingSafeEqual(proofBuffer, expectedProof)) {
|
|
2410
|
-
logger$
|
|
2417
|
+
logger$88.warn("Peer HMAC proof invalid", { peerInstanceId: authMsg.instanceId });
|
|
2411
2418
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2412
2419
|
type: "peer.auth.response",
|
|
2413
2420
|
accepted: false,
|
|
@@ -2431,7 +2438,7 @@ function createPeerHandler(deps) {
|
|
|
2431
2438
|
});
|
|
2432
2439
|
return true;
|
|
2433
2440
|
} else {
|
|
2434
|
-
logger$
|
|
2441
|
+
logger$88.warn("Peer auth request missing token and proof", { peerInstanceId: authMsg.instanceId });
|
|
2435
2442
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2436
2443
|
type: "peer.auth.response",
|
|
2437
2444
|
accepted: false,
|
|
@@ -2579,12 +2586,12 @@ function createPeerHandler(deps) {
|
|
|
2579
2586
|
closeAllInbound
|
|
2580
2587
|
};
|
|
2581
2588
|
}
|
|
2582
|
-
var logger$
|
|
2589
|
+
var logger$88, SOFTWARE_VERSION, RATE_LIMIT_MAX, RATE_LIMIT_WINDOW_MS;
|
|
2583
2590
|
var init_peer_handler = __esmMin((() => {
|
|
2584
2591
|
init_peer_crypto();
|
|
2585
2592
|
init_join_token();
|
|
2586
|
-
logger$
|
|
2587
|
-
SOFTWARE_VERSION = "0.1.
|
|
2593
|
+
logger$88 = createLogger({ prefix: "peer-handler" });
|
|
2594
|
+
SOFTWARE_VERSION = "0.1.19";
|
|
2588
2595
|
RATE_LIMIT_MAX = 5;
|
|
2589
2596
|
RATE_LIMIT_WINDOW_MS = 6e4;
|
|
2590
2597
|
}));
|
|
@@ -2606,9 +2613,9 @@ var init_peer_handler = __esmMin((() => {
|
|
|
2606
2613
|
* heartbeat (peer.heartbeat every 30s). Raft heartbeat is purely for leader
|
|
2607
2614
|
* liveness detection.
|
|
2608
2615
|
*/
|
|
2609
|
-
var logger$
|
|
2616
|
+
var logger$87, RaftNode;
|
|
2610
2617
|
var init_raft = __esmMin((() => {
|
|
2611
|
-
logger$
|
|
2618
|
+
logger$87 = createLogger({ prefix: "raft" });
|
|
2612
2619
|
RaftNode = class {
|
|
2613
2620
|
instanceId;
|
|
2614
2621
|
stateStore;
|
|
@@ -2659,7 +2666,7 @@ var init_raft = __esmMin((() => {
|
|
|
2659
2666
|
this.leaderId = state.leaderId;
|
|
2660
2667
|
this.role = "follower";
|
|
2661
2668
|
this.startedAt = Date.now();
|
|
2662
|
-
logger$
|
|
2669
|
+
logger$87.info("Raft node started", {
|
|
2663
2670
|
instanceId: this.instanceId,
|
|
2664
2671
|
currentTerm: this.currentTerm,
|
|
2665
2672
|
leaderId: this.leaderId
|
|
@@ -2677,7 +2684,7 @@ var init_raft = __esmMin((() => {
|
|
|
2677
2684
|
votedFor: this.votedFor,
|
|
2678
2685
|
leaderId: this.leaderId
|
|
2679
2686
|
});
|
|
2680
|
-
logger$
|
|
2687
|
+
logger$87.info("Raft node stopped", { instanceId: this.instanceId });
|
|
2681
2688
|
}
|
|
2682
2689
|
/** Whether this node is the current leader. */
|
|
2683
2690
|
isLeader() {
|
|
@@ -2703,11 +2710,11 @@ var init_raft = __esmMin((() => {
|
|
|
2703
2710
|
*/
|
|
2704
2711
|
handlePeerLeaving(instanceId) {
|
|
2705
2712
|
if (this.role === "leader") {
|
|
2706
|
-
logger$
|
|
2713
|
+
logger$87.debug("Ignoring peer.leaving (we are leader)", { leavingPeer: instanceId });
|
|
2707
2714
|
return;
|
|
2708
2715
|
}
|
|
2709
2716
|
if (instanceId === this.leaderId) {
|
|
2710
|
-
logger$
|
|
2717
|
+
logger$87.info("Current leader is leaving, starting immediate election", {
|
|
2711
2718
|
leavingPeer: instanceId,
|
|
2712
2719
|
term: this.currentTerm
|
|
2713
2720
|
});
|
|
@@ -2715,7 +2722,7 @@ var init_raft = __esmMin((() => {
|
|
|
2715
2722
|
this.consecutiveUnansweredElections = 0;
|
|
2716
2723
|
this.clearElectionTimer();
|
|
2717
2724
|
this.startElection();
|
|
2718
|
-
} else logger$
|
|
2725
|
+
} else logger$87.debug("Non-leader peer leaving", { leavingPeer: instanceId });
|
|
2719
2726
|
}
|
|
2720
2727
|
/**
|
|
2721
2728
|
* Reset the election timer with a randomized timeout.
|
|
@@ -2763,7 +2770,7 @@ var init_raft = __esmMin((() => {
|
|
|
2763
2770
|
this.votedFor = this.instanceId;
|
|
2764
2771
|
this.role = "candidate";
|
|
2765
2772
|
this.votesReceived = new Set([this.instanceId]);
|
|
2766
|
-
logger$
|
|
2773
|
+
logger$87.info("Starting election", {
|
|
2767
2774
|
instanceId: this.instanceId,
|
|
2768
2775
|
term: this.currentTerm
|
|
2769
2776
|
});
|
|
@@ -2772,12 +2779,12 @@ var init_raft = __esmMin((() => {
|
|
|
2772
2779
|
votedFor: this.votedFor,
|
|
2773
2780
|
leaderId: this.leaderId
|
|
2774
2781
|
}).catch((err) => {
|
|
2775
|
-
logger$
|
|
2782
|
+
logger$87.error("Failed to persist election state", { error: toErrorMessage(err) });
|
|
2776
2783
|
});
|
|
2777
2784
|
const connectedPeerCount = this.peerRegistry.getConnectedCoordinatorPeerCount();
|
|
2778
2785
|
if (connectedPeerCount === 0) {
|
|
2779
2786
|
if (this.gracePeriodMs > 0 && Date.now() - this.startedAt < this.gracePeriodMs) {
|
|
2780
|
-
logger$
|
|
2787
|
+
logger$87.info("Dormant mode: deferring self-election (grace period active)", {
|
|
2781
2788
|
instanceId: this.instanceId,
|
|
2782
2789
|
elapsedMs: Date.now() - this.startedAt,
|
|
2783
2790
|
gracePeriodMs: this.gracePeriodMs
|
|
@@ -2785,7 +2792,7 @@ var init_raft = __esmMin((() => {
|
|
|
2785
2792
|
this.resetElectionTimer();
|
|
2786
2793
|
return;
|
|
2787
2794
|
}
|
|
2788
|
-
logger$
|
|
2795
|
+
logger$87.info("No peers connected, self-electing as leader", {
|
|
2789
2796
|
instanceId: this.instanceId,
|
|
2790
2797
|
term: this.currentTerm
|
|
2791
2798
|
});
|
|
@@ -2793,7 +2800,7 @@ var init_raft = __esmMin((() => {
|
|
|
2793
2800
|
return;
|
|
2794
2801
|
}
|
|
2795
2802
|
if (this.consecutiveUnansweredElections >= 2) {
|
|
2796
|
-
logger$
|
|
2803
|
+
logger$87.warn("No vote responses after consecutive elections, forcing self-election", {
|
|
2797
2804
|
instanceId: this.instanceId,
|
|
2798
2805
|
term: this.currentTerm,
|
|
2799
2806
|
consecutiveUnanswered: this.consecutiveUnansweredElections,
|
|
@@ -2828,14 +2835,14 @@ var init_raft = __esmMin((() => {
|
|
|
2828
2835
|
votedFor: this.votedFor,
|
|
2829
2836
|
leaderId: this.leaderId
|
|
2830
2837
|
}).catch((err) => {
|
|
2831
|
-
logger$
|
|
2838
|
+
logger$87.error("Failed to persist vote", { error: toErrorMessage(err) });
|
|
2832
2839
|
});
|
|
2833
2840
|
this.resetElectionTimer();
|
|
2834
|
-
logger$
|
|
2841
|
+
logger$87.info("Vote granted", {
|
|
2835
2842
|
candidateId: msg.candidateId,
|
|
2836
2843
|
term: msg.term
|
|
2837
2844
|
});
|
|
2838
|
-
} else logger$
|
|
2845
|
+
} else logger$87.info("Vote denied", {
|
|
2839
2846
|
candidateId: msg.candidateId,
|
|
2840
2847
|
term: msg.term,
|
|
2841
2848
|
currentTerm: this.currentTerm,
|
|
@@ -2860,7 +2867,7 @@ var init_raft = __esmMin((() => {
|
|
|
2860
2867
|
this.consecutiveUnansweredElections = 0;
|
|
2861
2868
|
if (msg.voteGranted) {
|
|
2862
2869
|
this.votesReceived.add(msg.voterId);
|
|
2863
|
-
logger$
|
|
2870
|
+
logger$87.info("Vote received", {
|
|
2864
2871
|
from: msg.voterId,
|
|
2865
2872
|
term: msg.term,
|
|
2866
2873
|
totalVotes: this.votesReceived.size
|
|
@@ -2887,7 +2894,7 @@ var init_raft = __esmMin((() => {
|
|
|
2887
2894
|
this.leaderId = this.instanceId;
|
|
2888
2895
|
this.consecutiveUnansweredElections = 0;
|
|
2889
2896
|
this.clearElectionTimer();
|
|
2890
|
-
logger$
|
|
2897
|
+
logger$87.info("Became leader", {
|
|
2891
2898
|
instanceId: this.instanceId,
|
|
2892
2899
|
term: this.currentTerm
|
|
2893
2900
|
});
|
|
@@ -2897,7 +2904,7 @@ var init_raft = __esmMin((() => {
|
|
|
2897
2904
|
votedFor: this.votedFor,
|
|
2898
2905
|
leaderId: this.leaderId
|
|
2899
2906
|
}).catch((err) => {
|
|
2900
|
-
logger$
|
|
2907
|
+
logger$87.error("Failed to persist leader state", { error: toErrorMessage(err) });
|
|
2901
2908
|
});
|
|
2902
2909
|
this.onBecomeLeader();
|
|
2903
2910
|
}
|
|
@@ -2918,16 +2925,16 @@ var init_raft = __esmMin((() => {
|
|
|
2918
2925
|
votedFor: this.votedFor,
|
|
2919
2926
|
leaderId: this.leaderId
|
|
2920
2927
|
}).catch((err) => {
|
|
2921
|
-
logger$
|
|
2928
|
+
logger$87.error("Failed to persist step-down state", { error: toErrorMessage(err) });
|
|
2922
2929
|
});
|
|
2923
2930
|
if (wasLeader) {
|
|
2924
|
-
logger$
|
|
2931
|
+
logger$87.info("Lost leadership", {
|
|
2925
2932
|
instanceId: this.instanceId,
|
|
2926
2933
|
newTerm,
|
|
2927
2934
|
newLeaderId
|
|
2928
2935
|
});
|
|
2929
2936
|
this.onLoseLeadership();
|
|
2930
|
-
} else logger$
|
|
2937
|
+
} else logger$87.info("Stepped down", {
|
|
2931
2938
|
instanceId: this.instanceId,
|
|
2932
2939
|
newTerm,
|
|
2933
2940
|
newLeaderId
|
|
@@ -3035,9 +3042,9 @@ var init_raft_state = __esmMin((() => {
|
|
|
3035
3042
|
}));
|
|
3036
3043
|
//#endregion
|
|
3037
3044
|
//#region src/cluster/orphan-recovery.ts
|
|
3038
|
-
var logger$
|
|
3045
|
+
var logger$86, DEFAULT_STALE_THRESHOLD_MS, DEFAULT_JOB_STUCK_THRESHOLD_MS, OrphanRecovery;
|
|
3039
3046
|
var init_orphan_recovery = __esmMin((() => {
|
|
3040
|
-
logger$
|
|
3047
|
+
logger$86 = createLogger({ prefix: "orphan-recovery" });
|
|
3041
3048
|
DEFAULT_STALE_THRESHOLD_MS = 300 * 1e3;
|
|
3042
3049
|
DEFAULT_JOB_STUCK_THRESHOLD_MS = 180 * 1e3;
|
|
3043
3050
|
OrphanRecovery = class {
|
|
@@ -3066,10 +3073,10 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3066
3073
|
if (this.scanTimer) return;
|
|
3067
3074
|
this.scanTimer = setInterval(() => {
|
|
3068
3075
|
this.scanForOrphans().catch((err) => {
|
|
3069
|
-
logger$
|
|
3076
|
+
logger$86.error("Orphan recovery scan failed", { error: toErrorMessage(err) });
|
|
3070
3077
|
});
|
|
3071
3078
|
}, this.scanIntervalMs);
|
|
3072
|
-
logger$
|
|
3079
|
+
logger$86.info("Orphan recovery started", {
|
|
3073
3080
|
scanIntervalMs: this.scanIntervalMs,
|
|
3074
3081
|
staleThresholdMs: this.staleThresholdMs
|
|
3075
3082
|
});
|
|
@@ -3082,7 +3089,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3082
3089
|
clearInterval(this.scanTimer);
|
|
3083
3090
|
this.scanTimer = null;
|
|
3084
3091
|
}
|
|
3085
|
-
logger$
|
|
3092
|
+
logger$86.info("Orphan recovery stopped");
|
|
3086
3093
|
}
|
|
3087
3094
|
/**
|
|
3088
3095
|
* Scan for orphaned runs and recover them.
|
|
@@ -3100,7 +3107,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3100
3107
|
"sha"
|
|
3101
3108
|
]).where("status", "=", ExecutionRunStatus.enum.running).where("started_at", "<", staleThreshold).execute();
|
|
3102
3109
|
if (staleRuns.length === 0) return;
|
|
3103
|
-
logger$
|
|
3110
|
+
logger$86.info("Found potentially orphaned runs", { count: staleRuns.length });
|
|
3104
3111
|
for (const run of staleRuns) await this.recoverRun(run);
|
|
3105
3112
|
}
|
|
3106
3113
|
/**
|
|
@@ -3108,7 +3115,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3108
3115
|
*/
|
|
3109
3116
|
async recoverRun(run) {
|
|
3110
3117
|
if (this.isCoordinatorStillConnected(run.routing_key)) return;
|
|
3111
|
-
logger$
|
|
3118
|
+
logger$86.info("Recovering orphan run", {
|
|
3112
3119
|
runId: run.run_id,
|
|
3113
3120
|
routingKey: run.routing_key,
|
|
3114
3121
|
workflowName: run.workflow_name
|
|
@@ -3125,7 +3132,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3125
3132
|
completed_at: /* @__PURE__ */ new Date(),
|
|
3126
3133
|
failure_reason: "No jobs found for orphaned run"
|
|
3127
3134
|
}).where("run_id", "=", run.run_id).where("status", "=", ExecutionRunStatus.enum.running).execute();
|
|
3128
|
-
logger$
|
|
3135
|
+
logger$86.info("Finalized orphan run with no jobs", { runId: run.run_id });
|
|
3129
3136
|
this.executionTracker.emitInfraEvent(run.run_id, "orchestrator.run.orphan_recovered", { metadata: {
|
|
3130
3137
|
routingKey: run.routing_key,
|
|
3131
3138
|
reason: "No jobs found"
|
|
@@ -3152,7 +3159,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3152
3159
|
reason: errorMsg
|
|
3153
3160
|
}
|
|
3154
3161
|
});
|
|
3155
|
-
logger$
|
|
3162
|
+
logger$86.info("Marked stuck job as failed", {
|
|
3156
3163
|
runId: run.run_id,
|
|
3157
3164
|
jobId: job.job_id,
|
|
3158
3165
|
jobName: job.job_name
|
|
@@ -3206,9 +3213,9 @@ function agentSatisfiesMandatoryLabels(agent, requiredLabels) {
|
|
|
3206
3213
|
const required = new Set(requiredLabels);
|
|
3207
3214
|
return mandatory.every((m) => required.has(m));
|
|
3208
3215
|
}
|
|
3209
|
-
var logger$
|
|
3216
|
+
var logger$85, DEFAULT_ACK_TIMEOUT_MS, DEFAULT_MAX_HOPS, NAK_BACKOFF_BASE_MS, NAK_BACKOFF_MAX_MS, RunCoordinator;
|
|
3210
3217
|
var init_coordinator = __esmMin((() => {
|
|
3211
|
-
logger$
|
|
3218
|
+
logger$85 = createLogger({ prefix: "coordinator" });
|
|
3212
3219
|
DEFAULT_ACK_TIMEOUT_MS = 15e3;
|
|
3213
3220
|
DEFAULT_MAX_HOPS = 3;
|
|
3214
3221
|
NAK_BACKOFF_BASE_MS = 1e3;
|
|
@@ -3266,6 +3273,8 @@ var init_coordinator = __esmMin((() => {
|
|
|
3266
3273
|
workflowName: runContext.workflowName,
|
|
3267
3274
|
jobName: job.jobName,
|
|
3268
3275
|
runsOnLabels: flatLabels,
|
|
3276
|
+
runsOnPatterns: job.runsOnPatterns,
|
|
3277
|
+
excludePatterns: job.excludePatterns,
|
|
3269
3278
|
excludeLabels: job.excludeLabels,
|
|
3270
3279
|
jobConfig: job.jobConfig,
|
|
3271
3280
|
repoUrl: job.repoUrl,
|
|
@@ -3317,7 +3326,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3317
3326
|
}));
|
|
3318
3327
|
await Promise.all(reroutePromises);
|
|
3319
3328
|
}
|
|
3320
|
-
logger$
|
|
3329
|
+
logger$85.info("Jobs routed", {
|
|
3321
3330
|
runId: runContext.runId,
|
|
3322
3331
|
local: result.localJobs.length,
|
|
3323
3332
|
rerouted: result.reroutedJobs.length,
|
|
@@ -3346,6 +3355,8 @@ var init_coordinator = __esmMin((() => {
|
|
|
3346
3355
|
jobName: msg.jobName,
|
|
3347
3356
|
runsOnLabels: flatLabels,
|
|
3348
3357
|
excludeLabels: msg.excludeLabels,
|
|
3358
|
+
runsOnPatterns: msg.runsOnPatterns,
|
|
3359
|
+
excludePatterns: msg.excludePatterns,
|
|
3349
3360
|
jobConfig: msg.jobConfig ?? msg.payload,
|
|
3350
3361
|
repoUrl: msg.repoUrl ?? "",
|
|
3351
3362
|
ref: msg.ref ?? "",
|
|
@@ -3378,7 +3389,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3378
3389
|
runId: msg.runId
|
|
3379
3390
|
});
|
|
3380
3391
|
} catch (err) {
|
|
3381
|
-
logger$
|
|
3392
|
+
logger$85.warn("Failed to create check runs for rerouted job", {
|
|
3382
3393
|
runId: msg.runId,
|
|
3383
3394
|
error: toErrorMessage(err)
|
|
3384
3395
|
});
|
|
@@ -3395,7 +3406,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3395
3406
|
jobName: msg.jobName
|
|
3396
3407
|
}], msg.routingKey);
|
|
3397
3408
|
} catch (err) {
|
|
3398
|
-
logger$
|
|
3409
|
+
logger$85.warn("Failed to register rerouted execution for tracking", {
|
|
3399
3410
|
runId: msg.runId,
|
|
3400
3411
|
error: toErrorMessage(err)
|
|
3401
3412
|
});
|
|
@@ -3414,7 +3425,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3414
3425
|
*/
|
|
3415
3426
|
onPeerJobProgress(msg) {
|
|
3416
3427
|
if (this.executionTracker) (msg.kind === "job" ? this.executionTracker.onJobStatus(msg.runId, msg.jobId, msg.state, msg.timestamp, void 0, msg.data) : this.executionTracker.onStepStatus(msg.runId, msg.jobId, msg.stepIndex, msg.stepName, msg.state, msg.timestamp, msg.data)).catch((err) => {
|
|
3417
|
-
logger$
|
|
3428
|
+
logger$85.error("Failed to track peer job progress", {
|
|
3418
3429
|
error: toErrorMessage(err),
|
|
3419
3430
|
runId: msg.runId,
|
|
3420
3431
|
jobId: msg.jobId,
|
|
@@ -3428,7 +3439,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3428
3439
|
if (runJobs.size === 0) this.reroutedJobs.delete(msg.runId);
|
|
3429
3440
|
}
|
|
3430
3441
|
}
|
|
3431
|
-
logger$
|
|
3442
|
+
logger$85.debug("Peer job progress", {
|
|
3432
3443
|
runId: msg.runId,
|
|
3433
3444
|
jobId: msg.jobId,
|
|
3434
3445
|
kind: msg.kind,
|
|
@@ -3461,7 +3472,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3461
3472
|
*/
|
|
3462
3473
|
onPeerJobComplete(runId, jobId, status, timestamp, data) {
|
|
3463
3474
|
if (this.executionTracker) this.executionTracker.onJobStatus(runId, jobId, status, timestamp, void 0, data).catch((err) => {
|
|
3464
|
-
logger$
|
|
3475
|
+
logger$85.error("Failed to track peer job completion", {
|
|
3465
3476
|
error: toErrorMessage(err),
|
|
3466
3477
|
runId,
|
|
3467
3478
|
jobId
|
|
@@ -3500,14 +3511,14 @@ var init_coordinator = __esmMin((() => {
|
|
|
3500
3511
|
jobId,
|
|
3501
3512
|
reason
|
|
3502
3513
|
};
|
|
3503
|
-
if (!(client ? client.send(cancelMsg) : this.sendToPeerViaHandler?.(peerId, cancelMsg) ?? false)) logger$
|
|
3514
|
+
if (!(client ? client.send(cancelMsg) : this.sendToPeerViaHandler?.(peerId, cancelMsg) ?? false)) logger$85.warn("Failed to send cancel to peer", {
|
|
3504
3515
|
peerId,
|
|
3505
3516
|
runId,
|
|
3506
3517
|
jobId
|
|
3507
3518
|
});
|
|
3508
3519
|
}
|
|
3509
3520
|
}
|
|
3510
|
-
logger$
|
|
3521
|
+
logger$85.info("Cancel propagated to peers", {
|
|
3511
3522
|
runId,
|
|
3512
3523
|
reason,
|
|
3513
3524
|
peerCount: peerJobs.size
|
|
@@ -3527,7 +3538,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3527
3538
|
if (this.staleEvictionTimer) clearInterval(this.staleEvictionTimer);
|
|
3528
3539
|
this.staleEvictionTimer = setInterval(() => {
|
|
3529
3540
|
const evicted = this.peerRegistry.evictStalePeers(staleTimeoutMs);
|
|
3530
|
-
if (evicted.length > 0) logger$
|
|
3541
|
+
if (evicted.length > 0) logger$85.warn("Evicted stale peers", {
|
|
3531
3542
|
evicted,
|
|
3532
3543
|
count: evicted.length
|
|
3533
3544
|
});
|
|
@@ -3564,7 +3575,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3564
3575
|
if (peers.length === 0) {
|
|
3565
3576
|
const peersWithLabels = this.peerRegistry.findPeersWithLabels(labelSets);
|
|
3566
3577
|
const allPeers = this.peerRegistry.getConnectedPeers();
|
|
3567
|
-
logger$
|
|
3578
|
+
logger$85.debug("Reroute failed — peer registry state", {
|
|
3568
3579
|
jobName: job.jobName,
|
|
3569
3580
|
requiredLabels: labelSets,
|
|
3570
3581
|
connectedPeers: allPeers.map((p) => ({
|
|
@@ -3594,7 +3605,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3594
3605
|
for (const peer of sortedPeers) {
|
|
3595
3606
|
const nakEntry = this.nakTracker.get(peer.instanceId);
|
|
3596
3607
|
if (nakEntry && nakEntry.backoffUntil > now) {
|
|
3597
|
-
logger$
|
|
3608
|
+
logger$85.debug("Skipping peer in NAK backoff", {
|
|
3598
3609
|
peerId: peer.instanceId,
|
|
3599
3610
|
jobName: job.jobName,
|
|
3600
3611
|
backoffRemainingMs: nakEntry.backoffUntil - now
|
|
@@ -3604,7 +3615,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3604
3615
|
const client = this.getPeerClient(peer.instanceId);
|
|
3605
3616
|
const canSendViaHandler = !client && this.sendAndWaitAckViaHandler;
|
|
3606
3617
|
if (!client && !canSendViaHandler) {
|
|
3607
|
-
logger$
|
|
3618
|
+
logger$85.warn("No PeerClient for peer (found in registry but no connection)", {
|
|
3608
3619
|
peerId: peer.instanceId,
|
|
3609
3620
|
jobName: job.jobName
|
|
3610
3621
|
});
|
|
@@ -3625,6 +3636,8 @@ var init_coordinator = __esmMin((() => {
|
|
|
3625
3636
|
workflowName: runContext.workflowName,
|
|
3626
3637
|
runsOnLabels: labelSets,
|
|
3627
3638
|
excludeLabels: job.excludeLabels,
|
|
3639
|
+
runsOnPatterns: job.runsOnPatterns,
|
|
3640
|
+
excludePatterns: job.excludePatterns,
|
|
3628
3641
|
triedConnections: [this.instanceId],
|
|
3629
3642
|
maxHops: DEFAULT_MAX_HOPS,
|
|
3630
3643
|
coordinatorId: this.instanceId,
|
|
@@ -3645,7 +3658,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3645
3658
|
if (client ? await client.sendAndWaitAck(rerouteMsg, this.ackTimeoutMs) : await this.sendAndWaitAckViaHandler(peer.instanceId, rerouteMsg, this.ackTimeoutMs)) {
|
|
3646
3659
|
this.nakTracker.delete(peer.instanceId);
|
|
3647
3660
|
this.trackReroutedJob(runContext.runId, allocatedJobId, peer.instanceId, job.jobName);
|
|
3648
|
-
logger$
|
|
3661
|
+
logger$85.info("Job rerouted to peer", {
|
|
3649
3662
|
runId: runContext.runId,
|
|
3650
3663
|
jobName: job.jobName,
|
|
3651
3664
|
jobId: allocatedJobId,
|
|
@@ -3663,7 +3676,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3663
3676
|
count: nakCount,
|
|
3664
3677
|
backoffUntil: Date.now() + backoffMs
|
|
3665
3678
|
});
|
|
3666
|
-
logger$
|
|
3679
|
+
logger$85.warn("Peer NAKed job", {
|
|
3667
3680
|
peerId: peer.instanceId,
|
|
3668
3681
|
jobName: job.jobName,
|
|
3669
3682
|
nakCount,
|
|
@@ -3893,6 +3906,39 @@ function resolveDataDir(explicit) {
|
|
|
3893
3906
|
var init_data_dir = __esmMin((() => {}));
|
|
3894
3907
|
//#endregion
|
|
3895
3908
|
//#region src/metrics/prometheus.ts
|
|
3909
|
+
function meter$1() {
|
|
3910
|
+
if (!_meter$1) _meter$1 = createMeter("kici-orchestrator");
|
|
3911
|
+
return _meter$1;
|
|
3912
|
+
}
|
|
3913
|
+
/**
|
|
3914
|
+
* A counter whose real OTel instrument is created on first `.add()` — after
|
|
3915
|
+
* `initTelemetry()` has registered the global MeterProvider. The returned
|
|
3916
|
+
* value satisfies the OTel `Counter` interface, so call sites are unchanged.
|
|
3917
|
+
*/
|
|
3918
|
+
function lazyCounter$1(name, options) {
|
|
3919
|
+
let inst;
|
|
3920
|
+
const get = () => inst ??= meter$1().createCounter(name, options);
|
|
3921
|
+
return { add: (...args) => get().add(...args) };
|
|
3922
|
+
}
|
|
3923
|
+
/**
|
|
3924
|
+
* A histogram whose real OTel instrument is created on first `.record()` —
|
|
3925
|
+
* after `initTelemetry()`. Satisfies the OTel `Histogram` interface.
|
|
3926
|
+
*/
|
|
3927
|
+
function lazyHistogram$1(name, options) {
|
|
3928
|
+
let inst;
|
|
3929
|
+
const get = () => inst ??= meter$1().createHistogram(name, options);
|
|
3930
|
+
return { record: (...args) => get().record(...args) };
|
|
3931
|
+
}
|
|
3932
|
+
/**
|
|
3933
|
+
* Queue an observable gauge for registration. The gauge is created and its
|
|
3934
|
+
* callback attached lazily inside registerOrchestratorMetrics(), so the
|
|
3935
|
+
* instrument binds to the real MeterProvider rather than the no-op one.
|
|
3936
|
+
*/
|
|
3937
|
+
function defineObservableGauge(name, options, observe) {
|
|
3938
|
+
_observableGauges.push(() => {
|
|
3939
|
+
meter$1().createObservableGauge(name, options).addCallback(observe);
|
|
3940
|
+
});
|
|
3941
|
+
}
|
|
3896
3942
|
/** Set the current number of active agents. */
|
|
3897
3943
|
function setAgentsActive(value) {
|
|
3898
3944
|
_agentsActiveValue = value;
|
|
@@ -3901,6 +3947,10 @@ function setAgentsActive(value) {
|
|
|
3901
3947
|
function setConfigVersion(value) {
|
|
3902
3948
|
_configVersionValue = value;
|
|
3903
3949
|
}
|
|
3950
|
+
/** Set the current number of declared (static) roster hosts that are unreachable. */
|
|
3951
|
+
function setDeclaredHostsUnreachable(value) {
|
|
3952
|
+
_declaredHostsUnreachableValue = value;
|
|
3953
|
+
}
|
|
3904
3954
|
/** Set the current number of stale runs detected. */
|
|
3905
3955
|
function setStaleRunsCurrent(value) {
|
|
3906
3956
|
_staleRunsCurrentValue = value;
|
|
@@ -3958,21 +4008,32 @@ function setDispatchQueueDepthBreakdown(snapshot) {
|
|
|
3958
4008
|
function setEventDlqDepth(value) {
|
|
3959
4009
|
_eventDlqDepthValue = value;
|
|
3960
4010
|
}
|
|
3961
|
-
|
|
4011
|
+
/**
|
|
4012
|
+
* Register every queued orchestrator observable gauge on the real meter.
|
|
4013
|
+
*
|
|
4014
|
+
* Must run AFTER `initTelemetry()` has wired the global MeterProvider —
|
|
4015
|
+
* `createApp()` calls it once during bootstrap. Registering the gauges at
|
|
4016
|
+
* module-eval time instead would bind them to the no-op provider (the
|
|
4017
|
+
* bundler hoists some module init above the entry's `initTelemetry()` call),
|
|
4018
|
+
* leaving every `kici_orch_*` gauge absent from the /metrics scrape and the
|
|
4019
|
+
* Platform push. Idempotent: repeat calls are no-ops.
|
|
4020
|
+
*/
|
|
4021
|
+
function registerOrchestratorMetrics() {
|
|
4022
|
+
if (_observableGaugesRegistered) return;
|
|
4023
|
+
_observableGaugesRegistered = true;
|
|
4024
|
+
for (const register of _observableGauges) register();
|
|
4025
|
+
}
|
|
4026
|
+
var _meter$1, _observableGauges, _agentsActiveValue, _configVersionValue, _declaredHostsUnreachableValue, _staleRunsCurrentValue, _scalerUsageRows, _scalerSpawnRefusalsValue, _queueDepthBreakdown, _everSeenQueueLabels, webhooksReceivedTotal, webhooksProcessedTotal, triggerMatchDurationSeconds, dedupHitsTotal, pgPoolClientErrorsTotal, sourceCacheHitsTotal, sourceCacheMissesTotal, depCacheHitsTotal, depCacheMissesTotal, buildDurationSeconds, executionsTotal, executionDurationSeconds, stepsTotal, githubCheckRunTotal, logChunksReceivedTotal, logBytesStoredTotal, scalerConfigReloadsTotal, ScalerSpawnFailureBound, scalerSpawnFailuresTotal, configReloadTotal, staleRunsDetectedTotal, staleDetectionDurationSeconds, crossSourceFanoutSize, crossSourceErrorsTotal, universalGitRegistrationErrorsTotal, trustMatchRefusedNoIdTotal, eventDispatchSuccessTotal, eventRetryTotal, eventDlqTotal, eventLeaseExpirationsTotal, eventAttemptsHistogram, _eventDlqDepthValue, InstallSecretsDecisionReason, InstallSecretsChannel, installSecretsDecisionsTotal, installSecretsRegistryUsedTotal, installSecretsContributorStrippedTotal, installSecretsTokenResolutionDurationSeconds, _observableGaugesRegistered;
|
|
3962
4027
|
var init_prometheus = __esmMin((() => {
|
|
3963
|
-
|
|
4028
|
+
_observableGauges = [];
|
|
3964
4029
|
_agentsActiveValue = 0;
|
|
3965
|
-
|
|
3966
|
-
result.observe(_agentsActiveValue);
|
|
3967
|
-
});
|
|
4030
|
+
defineObservableGauge("kici_orch_agents_active", { description: "Current number of active agents connected" }, (result) => result.observe(_agentsActiveValue));
|
|
3968
4031
|
_configVersionValue = 0;
|
|
3969
|
-
|
|
3970
|
-
|
|
3971
|
-
});
|
|
4032
|
+
defineObservableGauge("kici_orch_config_version", { description: "Current shared config version number" }, (result) => result.observe(_configVersionValue));
|
|
4033
|
+
_declaredHostsUnreachableValue = 0;
|
|
4034
|
+
defineObservableGauge("kici_orch_declared_hosts_unreachable", { description: "Number of declared (static) roster hosts currently unreachable" }, (result) => result.observe(_declaredHostsUnreachableValue));
|
|
3972
4035
|
_staleRunsCurrentValue = 0;
|
|
3973
|
-
|
|
3974
|
-
result.observe(_staleRunsCurrentValue);
|
|
3975
|
-
});
|
|
4036
|
+
defineObservableGauge("kici_orch_stale_runs_current", { description: "Current number of stale runs detected in last scan" }, (result) => result.observe(_staleRunsCurrentValue));
|
|
3976
4037
|
_scalerUsageRows = [];
|
|
3977
4038
|
/**
|
|
3978
4039
|
* Current CPU reservations per scaler / machine pool.
|
|
@@ -3980,7 +4041,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
3980
4041
|
* - scaler: __global__ | container | firecracker | bare-metal | stateful
|
|
3981
4042
|
* - machinePool: optional pool name (operator-defined; capped at the Platform filter)
|
|
3982
4043
|
*/
|
|
3983
|
-
|
|
4044
|
+
defineObservableGauge("kici_orch_scaler_cpus_used", { description: "Current CPU reservations summed by scaler / pool. scaler=\"__global__\" is the orchestrator-wide total; machinePool=\"<name>\" rows reflect the on-disk ledger." }, (result) => {
|
|
3984
4045
|
for (const row of _scalerUsageRows) {
|
|
3985
4046
|
const attrs = { scaler: row.scaler };
|
|
3986
4047
|
if (row.machinePool) attrs.machinePool = row.machinePool;
|
|
@@ -3993,7 +4054,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
3993
4054
|
* - scaler: __global__ | container | firecracker | bare-metal | stateful
|
|
3994
4055
|
* - machinePool: optional pool name (operator-defined; capped at the Platform filter)
|
|
3995
4056
|
*/
|
|
3996
|
-
|
|
4057
|
+
defineObservableGauge("kici_orch_scaler_memory_bytes_used", { description: "Current memory reservations (bytes) summed by scaler / pool. scaler=\"__global__\" is the orchestrator-wide total; machinePool=\"<name>\" rows reflect the on-disk ledger." }, (result) => {
|
|
3997
4058
|
for (const row of _scalerUsageRows) {
|
|
3998
4059
|
const attrs = { scaler: row.scaler };
|
|
3999
4060
|
if (row.machinePool) attrs.machinePool = row.machinePool;
|
|
@@ -4001,9 +4062,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4001
4062
|
}
|
|
4002
4063
|
});
|
|
4003
4064
|
_scalerSpawnRefusalsValue = 0;
|
|
4004
|
-
|
|
4005
|
-
result.observe(_scalerSpawnRefusalsValue);
|
|
4006
|
-
});
|
|
4065
|
+
defineObservableGauge("kici_orch_scaler_spawn_refusals_total", { description: "Cumulative count of scaler spawn requests refused due to resource caps (maxAgents, resourceCap, globalResourceCap, machinePool)." }, (result) => result.observe(_scalerSpawnRefusalsValue));
|
|
4007
4066
|
_queueDepthBreakdown = {
|
|
4008
4067
|
byStatus: {
|
|
4009
4068
|
pending: 0,
|
|
@@ -4017,7 +4076,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4017
4076
|
* Labels:
|
|
4018
4077
|
* - status: pending | dispatched
|
|
4019
4078
|
*/
|
|
4020
|
-
|
|
4079
|
+
defineObservableGauge("kici_orch_dispatch_queue_depth", { description: "Current dispatch_queue depth per status (pending, dispatched)" }, (result) => {
|
|
4021
4080
|
result.observe(_queueDepthBreakdown.byStatus.pending, { status: "pending" });
|
|
4022
4081
|
result.observe(_queueDepthBreakdown.byStatus.dispatched, { status: "dispatched" });
|
|
4023
4082
|
});
|
|
@@ -4027,15 +4086,15 @@ var init_prometheus = __esmMin((() => {
|
|
|
4027
4086
|
* - status: pending (only emitted status)
|
|
4028
4087
|
* - label: a runs_on label string from the workflow's runtime config
|
|
4029
4088
|
*/
|
|
4030
|
-
|
|
4089
|
+
defineObservableGauge("kici_orch_dispatch_queue_depth_by_label", { description: "Current pending dispatch_queue depth per runs_on label (multi-label jobs fan out)" }, (result) => {
|
|
4031
4090
|
for (const [label, count] of Object.entries(_queueDepthBreakdown.byLabel)) result.observe(count, {
|
|
4032
4091
|
status: "pending",
|
|
4033
4092
|
label
|
|
4034
4093
|
});
|
|
4035
4094
|
});
|
|
4036
|
-
webhooksReceivedTotal =
|
|
4037
|
-
webhooksProcessedTotal =
|
|
4038
|
-
triggerMatchDurationSeconds =
|
|
4095
|
+
webhooksReceivedTotal = lazyCounter$1("kici_orch_webhooks_received_total", { description: "Total number of webhooks received" });
|
|
4096
|
+
webhooksProcessedTotal = lazyCounter$1("kici_orch_webhooks_processed_total", { description: "Total number of webhooks processed" });
|
|
4097
|
+
triggerMatchDurationSeconds = lazyHistogram$1("kici_orch_trigger_match_duration_seconds", {
|
|
4039
4098
|
description: "Duration of trigger matching operations in seconds",
|
|
4040
4099
|
advice: { explicitBucketBoundaries: [
|
|
4041
4100
|
.001,
|
|
@@ -4048,13 +4107,13 @@ var init_prometheus = __esmMin((() => {
|
|
|
4048
4107
|
5
|
|
4049
4108
|
] }
|
|
4050
4109
|
});
|
|
4051
|
-
dedupHitsTotal =
|
|
4052
|
-
pgPoolClientErrorsTotal =
|
|
4053
|
-
sourceCacheHitsTotal =
|
|
4054
|
-
sourceCacheMissesTotal =
|
|
4055
|
-
depCacheHitsTotal =
|
|
4056
|
-
depCacheMissesTotal =
|
|
4057
|
-
buildDurationSeconds =
|
|
4110
|
+
dedupHitsTotal = lazyCounter$1("kici_orch_dedup_hits_total", { description: "Total number of deduplication cache hits" });
|
|
4111
|
+
pgPoolClientErrorsTotal = lazyCounter$1("kici_orch_pg_pool_client_errors_total", { description: "Total pg connection errors absorbed without a process restart" });
|
|
4112
|
+
sourceCacheHitsTotal = lazyCounter$1("kici_orch_source_cache_hits_total", { description: "Total number of source tarball cache hits" });
|
|
4113
|
+
sourceCacheMissesTotal = lazyCounter$1("kici_orch_source_cache_misses_total", { description: "Total number of source tarball cache misses" });
|
|
4114
|
+
depCacheHitsTotal = lazyCounter$1("kici_orch_dep_cache_hits_total", { description: "Total number of dep cache hits" });
|
|
4115
|
+
depCacheMissesTotal = lazyCounter$1("kici_orch_dep_cache_misses_total", { description: "Total number of dep cache misses" });
|
|
4116
|
+
buildDurationSeconds = lazyHistogram$1("kici_orch_build_duration_seconds", {
|
|
4058
4117
|
description: "Duration of build agent operations in seconds",
|
|
4059
4118
|
advice: { explicitBucketBoundaries: [
|
|
4060
4119
|
1,
|
|
@@ -4067,8 +4126,8 @@ var init_prometheus = __esmMin((() => {
|
|
|
4067
4126
|
600
|
|
4068
4127
|
] }
|
|
4069
4128
|
});
|
|
4070
|
-
executionsTotal =
|
|
4071
|
-
executionDurationSeconds =
|
|
4129
|
+
executionsTotal = lazyCounter$1("kici_orch_executions_total", { description: "Total number of execution runs" });
|
|
4130
|
+
executionDurationSeconds = lazyHistogram$1("kici_orch_execution_duration_seconds", {
|
|
4072
4131
|
description: "Duration of execution runs in seconds",
|
|
4073
4132
|
advice: { explicitBucketBoundaries: [
|
|
4074
4133
|
1,
|
|
@@ -4082,19 +4141,19 @@ var init_prometheus = __esmMin((() => {
|
|
|
4082
4141
|
1800
|
|
4083
4142
|
] }
|
|
4084
4143
|
});
|
|
4085
|
-
stepsTotal =
|
|
4086
|
-
githubCheckRunTotal =
|
|
4087
|
-
logChunksReceivedTotal =
|
|
4088
|
-
logBytesStoredTotal =
|
|
4089
|
-
scalerConfigReloadsTotal =
|
|
4144
|
+
stepsTotal = lazyCounter$1("kici_orch_steps_total", { description: "Total number of steps executed" });
|
|
4145
|
+
githubCheckRunTotal = lazyCounter$1("kici_orch_github_check_run_total", { description: "Total number of GitHub check run API calls" });
|
|
4146
|
+
logChunksReceivedTotal = lazyCounter$1("kici_orch_log_chunks_received_total", { description: "Total number of log chunks received from agents" });
|
|
4147
|
+
logBytesStoredTotal = lazyCounter$1("kici_orch_log_bytes_stored_total", { description: "Total bytes of log data written to storage" });
|
|
4148
|
+
scalerConfigReloadsTotal = lazyCounter$1("kici_orch_scaler_config_reloads_total", { description: "Total number of scaler config reload operations" });
|
|
4090
4149
|
ScalerSpawnFailureBound = {
|
|
4091
4150
|
Bound: "true",
|
|
4092
4151
|
Unbound: "false"
|
|
4093
4152
|
};
|
|
4094
|
-
scalerSpawnFailuresTotal =
|
|
4095
|
-
configReloadTotal =
|
|
4096
|
-
staleRunsDetectedTotal =
|
|
4097
|
-
staleDetectionDurationSeconds =
|
|
4153
|
+
scalerSpawnFailuresTotal = lazyCounter$1("kici_orch_scaler_spawn_failures_total", { description: "Total scaler agent spawn failures (job-bound and warm-pool)" });
|
|
4154
|
+
configReloadTotal = lazyCounter$1("kici_orch_config_reload_total", { description: "Total number of config reload operations" });
|
|
4155
|
+
staleRunsDetectedTotal = lazyCounter$1("kici_orch_stale_runs_detected_total", { description: "Total number of stale runs detected and marked as failed" });
|
|
4156
|
+
staleDetectionDurationSeconds = lazyHistogram$1("kici_orch_stale_detection_duration_seconds", {
|
|
4098
4157
|
description: "Time between job becoming stale and detection (seconds)",
|
|
4099
4158
|
advice: { explicitBucketBoundaries: [
|
|
4100
4159
|
10,
|
|
@@ -4105,7 +4164,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4105
4164
|
600
|
|
4106
4165
|
] }
|
|
4107
4166
|
});
|
|
4108
|
-
crossSourceFanoutSize =
|
|
4167
|
+
crossSourceFanoutSize = lazyHistogram$1("kici_cross_source_fanout_size", {
|
|
4109
4168
|
description: "Number of webhook-trigger registrations matched when an inbound generic webhook fans out across sources in the same org",
|
|
4110
4169
|
advice: { explicitBucketBoundaries: [
|
|
4111
4170
|
0,
|
|
@@ -4118,18 +4177,16 @@ var init_prometheus = __esmMin((() => {
|
|
|
4118
4177
|
100
|
|
4119
4178
|
] }
|
|
4120
4179
|
});
|
|
4121
|
-
crossSourceErrorsTotal =
|
|
4122
|
-
universalGitRegistrationErrorsTotal =
|
|
4123
|
-
trustMatchRefusedNoIdTotal =
|
|
4124
|
-
eventDispatchSuccessTotal =
|
|
4125
|
-
eventRetryTotal =
|
|
4126
|
-
eventDlqTotal =
|
|
4127
|
-
eventLeaseExpirationsTotal =
|
|
4128
|
-
eventAttemptsHistogram =
|
|
4180
|
+
crossSourceErrorsTotal = lazyCounter$1("kici_cross_source_errors_total", { description: "Errors encountered during cross-source webhook dispatch" });
|
|
4181
|
+
universalGitRegistrationErrorsTotal = lazyCounter$1("kici_universal_git_registration_errors_total", { description: "Errors encountered while registering universal-git provider bundles" });
|
|
4182
|
+
trustMatchRefusedNoIdTotal = lazyCounter$1("kici_orch_trust_match_refused_no_id_total", { description: "Trust-resolution attempts refused because the provider numeric id was missing or did not match" });
|
|
4183
|
+
eventDispatchSuccessTotal = lazyCounter$1("kici_orch_event_dispatch_success_total", { description: "Internal events delivered to all matching workflows successfully (after the lease commits processed=true)" });
|
|
4184
|
+
eventRetryTotal = lazyCounter$1("kici_orch_event_retry_total", { description: "Internal-event dispatch failures that triggered a retry (lease released, next_retry_at scheduled)" });
|
|
4185
|
+
eventDlqTotal = lazyCounter$1("kici_orch_event_dlq_total", { description: "Internal events moved to the DLQ after exhausting retries" });
|
|
4186
|
+
eventLeaseExpirationsTotal = lazyCounter$1("kici_orch_event_lease_expirations_total", { description: "Dispatch leases that timed out before the holding node finalised them (signals node crash mid-dispatch)" });
|
|
4187
|
+
eventAttemptsHistogram = lazyHistogram$1("kici_orch_event_attempts", { description: "Distribution of dispatch attempts at terminal outcome (success or DLQ)" });
|
|
4129
4188
|
_eventDlqDepthValue = 0;
|
|
4130
|
-
|
|
4131
|
-
result.observe(_eventDlqDepthValue);
|
|
4132
|
-
});
|
|
4189
|
+
defineObservableGauge("kici_orch_event_dlq_depth", { description: "Current count of events sitting in the DLQ awaiting operator triage" }, (result) => result.observe(_eventDlqDepthValue));
|
|
4133
4190
|
InstallSecretsDecisionReason = {
|
|
4134
4191
|
Ok: "ok",
|
|
4135
4192
|
MalformedRef: "malformed_ref",
|
|
@@ -4147,10 +4204,10 @@ var init_prometheus = __esmMin((() => {
|
|
|
4147
4204
|
Registries: "registries",
|
|
4148
4205
|
InstallEnv: "install_env"
|
|
4149
4206
|
};
|
|
4150
|
-
installSecretsDecisionsTotal =
|
|
4151
|
-
installSecretsRegistryUsedTotal =
|
|
4152
|
-
installSecretsContributorStrippedTotal =
|
|
4153
|
-
installSecretsTokenResolutionDurationSeconds =
|
|
4207
|
+
installSecretsDecisionsTotal = lazyCounter$1("kici_orch_install_secrets_decisions_total", { description: "Total install-secrets resolution decisions (pass / reject / hold + reason)" });
|
|
4208
|
+
installSecretsRegistryUsedTotal = lazyCounter$1("kici_orch_install_secrets_npm_registry_used_total", { description: "Total registry / installEnv entries successfully resolved per channel / provider / scope" });
|
|
4209
|
+
installSecretsContributorStrippedTotal = lazyCounter$1("kici_orch_install_secrets_contributor_stripped_total", { description: "Dispatches where registry tokens were stripped because the contributor tier was not `trusted`" });
|
|
4210
|
+
installSecretsTokenResolutionDurationSeconds = lazyHistogram$1("kici_orch_install_secrets_token_resolution_duration_seconds", {
|
|
4154
4211
|
description: "Duration of per-environment secret resolution during install-secrets evaluation",
|
|
4155
4212
|
advice: { explicitBucketBoundaries: [
|
|
4156
4213
|
.001,
|
|
@@ -4163,6 +4220,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4163
4220
|
5
|
|
4164
4221
|
] }
|
|
4165
4222
|
});
|
|
4223
|
+
_observableGaugesRegistered = false;
|
|
4166
4224
|
}));
|
|
4167
4225
|
//#endregion
|
|
4168
4226
|
//#region src/config/reload.ts
|
|
@@ -4428,6 +4486,8 @@ var init_schema = __esmMin((() => {
|
|
|
4428
4486
|
}).optional(),
|
|
4429
4487
|
agentAuth: z.enum(["token", "none"]).optional(),
|
|
4430
4488
|
agentTokenTtlMs: z.coerce.number().optional(),
|
|
4489
|
+
rosterGraceMs: z.coerce.number().optional(),
|
|
4490
|
+
rosterTtlMs: z.coerce.number().optional(),
|
|
4431
4491
|
queue: z.object({
|
|
4432
4492
|
maxDepth: z.coerce.number().optional(),
|
|
4433
4493
|
timeoutMs: z.coerce.number().optional(),
|
|
@@ -4504,6 +4564,8 @@ var init_schema = __esmMin((() => {
|
|
|
4504
4564
|
}).optional(),
|
|
4505
4565
|
agentAuth: z.enum(["token", "none"]).default("token"),
|
|
4506
4566
|
agentTokenTtlMs: z.coerce.number().default(36e5),
|
|
4567
|
+
rosterGraceMs: z.coerce.number().default(3e5),
|
|
4568
|
+
rosterTtlMs: z.coerce.number().default(18e5),
|
|
4507
4569
|
queueMaxDepth: z.coerce.number().default(1e3),
|
|
4508
4570
|
queueTimeoutMs: z.coerce.number().default(36e5),
|
|
4509
4571
|
/**
|
|
@@ -4925,6 +4987,9 @@ function getDefaults() {
|
|
|
4925
4987
|
return {
|
|
4926
4988
|
agentAuth: "token",
|
|
4927
4989
|
agentTokenTtlMs: 36e5,
|
|
4990
|
+
rosterGraceMs: 3e5,
|
|
4991
|
+
rosterTtlMs: 18e5,
|
|
4992
|
+
maxFanoutHosts: 1024,
|
|
4928
4993
|
queue: {
|
|
4929
4994
|
maxDepth: 1e3,
|
|
4930
4995
|
timeoutMs: 36e5,
|
|
@@ -4974,6 +5039,9 @@ function flattenToAppConfig(merged) {
|
|
|
4974
5039
|
if (merged.storage) flat.storage = merged.storage;
|
|
4975
5040
|
flat.agentAuth = merged.agentAuth ?? "token";
|
|
4976
5041
|
flat.agentTokenTtlMs = merged.agentTokenTtlMs ?? 36e5;
|
|
5042
|
+
flat.rosterGraceMs = merged.rosterGraceMs ?? 3e5;
|
|
5043
|
+
flat.rosterTtlMs = merged.rosterTtlMs ?? 18e5;
|
|
5044
|
+
flat.maxFanoutHosts = merged.maxFanoutHosts ?? 1024;
|
|
4977
5045
|
const queue = merged.queue;
|
|
4978
5046
|
flat.queueMaxDepth = queue?.maxDepth ?? 1e3;
|
|
4979
5047
|
flat.queueTimeoutMs = queue?.timeoutMs ?? 36e5;
|
|
@@ -5068,8 +5136,9 @@ function satisfiesMandatoryLabels(agent, requiredLabels) {
|
|
|
5068
5136
|
for (const label of agent.mandatoryLabels) if (!requiredSet.has(label)) return false;
|
|
5069
5137
|
return true;
|
|
5070
5138
|
}
|
|
5071
|
-
var AgentRegistry;
|
|
5139
|
+
var logger$84, AgentRegistry;
|
|
5072
5140
|
var init_registry = __esmMin((() => {
|
|
5141
|
+
logger$84 = createLogger({ prefix: "agent-registry" });
|
|
5073
5142
|
AgentRegistry = class {
|
|
5074
5143
|
/** Primary: agentId -> AgentEntry */
|
|
5075
5144
|
agents = /* @__PURE__ */ new Map();
|
|
@@ -5107,6 +5176,27 @@ var init_registry = __esmMin((() => {
|
|
|
5107
5176
|
*/
|
|
5108
5177
|
tokenExpiryTimers = /* @__PURE__ */ new Map();
|
|
5109
5178
|
/**
|
|
5179
|
+
* Per-agent last roster `last_seen` stamp time (epoch ms). Throttles the
|
|
5180
|
+
* coarse heartbeat write so we don't issue a DB write per agent per
|
|
5181
|
+
* heartbeat — `grace`/`ttl` are minute-scale, so a throttled stamp is
|
|
5182
|
+
* sufficient. Cleaned up alongside the agent in `removeFromIndexes`.
|
|
5183
|
+
*/
|
|
5184
|
+
lastRosterStampAt = /* @__PURE__ */ new Map();
|
|
5185
|
+
/** Minimum interval between coarse roster `last_seen` stamps per agent. */
|
|
5186
|
+
ROSTER_STAMP_THROTTLE_MS = 6e4;
|
|
5187
|
+
/** Optional host-roster reconciler + this instance's id (see AgentRegistryDeps). */
|
|
5188
|
+
rosterStore;
|
|
5189
|
+
instanceId;
|
|
5190
|
+
/**
|
|
5191
|
+
* The roster reconcile seam is optional: when no store is injected (workers
|
|
5192
|
+
* with no DB, unit tests), every reconcile hook is a no-op and the in-memory
|
|
5193
|
+
* registry behaves exactly as before.
|
|
5194
|
+
*/
|
|
5195
|
+
constructor(deps = {}) {
|
|
5196
|
+
this.rosterStore = deps.rosterStore;
|
|
5197
|
+
this.instanceId = deps.instanceId;
|
|
5198
|
+
}
|
|
5199
|
+
/**
|
|
5110
5200
|
* Register an agent. If the agentId already exists, updates the existing
|
|
5111
5201
|
* entry (agent reconnection scenario).
|
|
5112
5202
|
*/
|
|
@@ -5138,7 +5228,8 @@ var init_registry = __esmMin((() => {
|
|
|
5138
5228
|
runningAsUid: metadata?.runningAsUid ?? null,
|
|
5139
5229
|
memoryUsedMb: null,
|
|
5140
5230
|
memoryAvailableMb: null,
|
|
5141
|
-
uptimeSeconds: null
|
|
5231
|
+
uptimeSeconds: null,
|
|
5232
|
+
tokenAgentType: metadata?.tokenAgentType ?? null
|
|
5142
5233
|
};
|
|
5143
5234
|
this.agents.set(agentId, entry);
|
|
5144
5235
|
this.wsToAgentId.set(ws, agentId);
|
|
@@ -5158,6 +5249,22 @@ var init_registry = __esmMin((() => {
|
|
|
5158
5249
|
}
|
|
5159
5250
|
tokenAgents.add(agentId);
|
|
5160
5251
|
}
|
|
5252
|
+
if (this.rosterStore && this.instanceId) {
|
|
5253
|
+
const instanceId = this.instanceId;
|
|
5254
|
+
this.rosterStore.upsert({
|
|
5255
|
+
agentId,
|
|
5256
|
+
tokenId,
|
|
5257
|
+
lifecycleClass: metadata?.tokenAgentType ?? "ephemeral",
|
|
5258
|
+
labels,
|
|
5259
|
+
hostname: metadata?.hostname ?? null,
|
|
5260
|
+
platform,
|
|
5261
|
+
arch,
|
|
5262
|
+
instanceId
|
|
5263
|
+
}).catch((err) => logger$84.warn("host_roster upsert failed (best-effort)", {
|
|
5264
|
+
agentId,
|
|
5265
|
+
error: toErrorMessage(err)
|
|
5266
|
+
}));
|
|
5267
|
+
}
|
|
5161
5268
|
}
|
|
5162
5269
|
/**
|
|
5163
5270
|
* Unregister an agent by ID. Removes from all indexes.
|
|
@@ -5168,6 +5275,7 @@ var init_registry = __esmMin((() => {
|
|
|
5168
5275
|
if (!entry) return void 0;
|
|
5169
5276
|
this.removeFromIndexes(agentId, entry);
|
|
5170
5277
|
this.agents.delete(agentId);
|
|
5278
|
+
if (this.rosterStore && this.instanceId) this.rosterStore.markDisconnected(agentId, this.instanceId).catch(() => {});
|
|
5171
5279
|
return entry;
|
|
5172
5280
|
}
|
|
5173
5281
|
/**
|
|
@@ -5268,7 +5376,7 @@ var init_registry = __esmMin((() => {
|
|
|
5268
5376
|
* @param requiredLabels - All labels the agent must have (intersection semantics).
|
|
5269
5377
|
* @returns Array of matching available AgentEntry objects.
|
|
5270
5378
|
*/
|
|
5271
|
-
findAvailable(requiredLabels, excludeLabels = []) {
|
|
5379
|
+
findAvailable(requiredLabels, requiredPatterns = [], excludeLabels = [], excludePatterns = []) {
|
|
5272
5380
|
let candidates;
|
|
5273
5381
|
if (requiredLabels.length === 0) candidates = [...this.agents.values()].filter((e) => e.activeJobs < e.maxConcurrency);
|
|
5274
5382
|
else {
|
|
@@ -5287,14 +5395,29 @@ var init_registry = __esmMin((() => {
|
|
|
5287
5395
|
if (entry.activeJobs < entry.maxConcurrency) candidates.push(entry);
|
|
5288
5396
|
}
|
|
5289
5397
|
}
|
|
5398
|
+
if (requiredPatterns.length > 0) candidates = candidates.filter((a) => requiredPatterns.every((p) => matcherSatisfiedBy(p, a.labels)));
|
|
5290
5399
|
let filtered = candidates;
|
|
5291
5400
|
if (excludeLabels.length > 0) filtered = filtered.filter((agent) => {
|
|
5292
5401
|
for (const excluded of excludeLabels) if (agent.labels.has(excluded)) return false;
|
|
5293
5402
|
return true;
|
|
5294
5403
|
});
|
|
5404
|
+
if (excludePatterns.length > 0) filtered = filtered.filter((a) => !excludePatterns.some((p) => matcherSatisfiedBy(p, a.labels)));
|
|
5295
5405
|
return filtered.filter((agent) => satisfiesMandatoryLabels(agent, requiredLabels));
|
|
5296
5406
|
}
|
|
5297
5407
|
/**
|
|
5408
|
+
* Whether a specific agent satisfies the same label / exclude / mandatory
|
|
5409
|
+
* gate `findAvailable` applies per-agent, ignoring capacity. Used by the
|
|
5410
|
+
* host-fanout pin to verify the resolved agent still matches before pinning
|
|
5411
|
+
* (guards against roster label drift between resolution and dispatch).
|
|
5412
|
+
*/
|
|
5413
|
+
agentSatisfies(agent, requiredLabels, requiredPatterns = [], excludeLabels = [], excludePatterns = []) {
|
|
5414
|
+
for (const required of requiredLabels) if (!agent.labels.has(required)) return false;
|
|
5415
|
+
for (const p of requiredPatterns) if (!matcherSatisfiedBy(p, agent.labels)) return false;
|
|
5416
|
+
for (const excluded of excludeLabels) if (agent.labels.has(excluded)) return false;
|
|
5417
|
+
for (const p of excludePatterns) if (matcherSatisfiedBy(p, agent.labels)) return false;
|
|
5418
|
+
return satisfiesMandatoryLabels(agent, requiredLabels);
|
|
5419
|
+
}
|
|
5420
|
+
/**
|
|
5298
5421
|
* Check if ANY registered agent matches the required labels, regardless of capacity.
|
|
5299
5422
|
* Used to decide whether to queue (agent exists but busy) vs reject (no agent at all).
|
|
5300
5423
|
*
|
|
@@ -5305,23 +5428,20 @@ var init_registry = __esmMin((() => {
|
|
|
5305
5428
|
* an off-gate gated agent as "matching but busy" and skip the peer
|
|
5306
5429
|
* reroute path even though the local agent can never accept the job.
|
|
5307
5430
|
*/
|
|
5308
|
-
hasMatchingAgent(requiredLabels, excludeLabels = []) {
|
|
5431
|
+
hasMatchingAgent(requiredLabels, requiredPatterns = [], excludeLabels = [], excludePatterns = []) {
|
|
5309
5432
|
const candidateIds = this.intersectLabelCandidates(requiredLabels);
|
|
5310
5433
|
if (candidateIds === null) return false;
|
|
5434
|
+
const ok = (entry) => {
|
|
5435
|
+
if (excludeLabels.some((e) => entry.labels.has(e))) return false;
|
|
5436
|
+
if (requiredPatterns.some((p) => !matcherSatisfiedBy(p, entry.labels))) return false;
|
|
5437
|
+
if (excludePatterns.some((p) => matcherSatisfiedBy(p, entry.labels))) return false;
|
|
5438
|
+
return satisfiesMandatoryLabels(entry, requiredLabels);
|
|
5439
|
+
};
|
|
5311
5440
|
if (requiredLabels.length === 0) {
|
|
5312
|
-
for (const entry of this.agents.values())
|
|
5313
|
-
if (excludeLabels.some((e) => entry.labels.has(e))) continue;
|
|
5314
|
-
if (!satisfiesMandatoryLabels(entry, requiredLabels)) continue;
|
|
5315
|
-
return true;
|
|
5316
|
-
}
|
|
5441
|
+
for (const entry of this.agents.values()) if (ok(entry)) return true;
|
|
5317
5442
|
return false;
|
|
5318
5443
|
}
|
|
5319
|
-
for (const id of candidateIds)
|
|
5320
|
-
const entry = this.agents.get(id);
|
|
5321
|
-
if (excludeLabels.length > 0 && excludeLabels.some((e) => entry.labels.has(e))) continue;
|
|
5322
|
-
if (!satisfiesMandatoryLabels(entry, requiredLabels)) continue;
|
|
5323
|
-
return true;
|
|
5324
|
-
}
|
|
5444
|
+
for (const id of candidateIds) if (ok(this.agents.get(id))) return true;
|
|
5325
5445
|
return false;
|
|
5326
5446
|
}
|
|
5327
5447
|
/**
|
|
@@ -5371,6 +5491,16 @@ var init_registry = __esmMin((() => {
|
|
|
5371
5491
|
const entry = this.agents.get(agentId);
|
|
5372
5492
|
if (!entry) return false;
|
|
5373
5493
|
entry.lastHeartbeatAt = Date.now();
|
|
5494
|
+
if (this.rosterStore && this.instanceId) {
|
|
5495
|
+
const now = Date.now();
|
|
5496
|
+
const last = this.lastRosterStampAt.get(agentId) ?? now;
|
|
5497
|
+
if (!this.lastRosterStampAt.has(agentId)) this.lastRosterStampAt.set(agentId, now);
|
|
5498
|
+
else if (now - last >= this.ROSTER_STAMP_THROTTLE_MS) {
|
|
5499
|
+
this.lastRosterStampAt.set(agentId, now);
|
|
5500
|
+
const instanceId = this.instanceId;
|
|
5501
|
+
this.rosterStore.stampLastSeen(agentId, instanceId).catch(() => {});
|
|
5502
|
+
}
|
|
5503
|
+
}
|
|
5374
5504
|
return true;
|
|
5375
5505
|
}
|
|
5376
5506
|
/**
|
|
@@ -5405,6 +5535,7 @@ var init_registry = __esmMin((() => {
|
|
|
5405
5535
|
*/
|
|
5406
5536
|
removeFromIndexes(agentId, entry) {
|
|
5407
5537
|
this.wsToAgentId.delete(entry.ws);
|
|
5538
|
+
this.lastRosterStampAt.delete(agentId);
|
|
5408
5539
|
for (const label of entry.labels) {
|
|
5409
5540
|
const agentIds = this.labelIndex.get(label);
|
|
5410
5541
|
if (agentIds) {
|
|
@@ -5428,7 +5559,222 @@ var init_registry = __esmMin((() => {
|
|
|
5428
5559
|
}
|
|
5429
5560
|
}
|
|
5430
5561
|
};
|
|
5431
|
-
}))
|
|
5562
|
+
}));
|
|
5563
|
+
//#endregion
|
|
5564
|
+
//#region src/agent/host-roster.ts
|
|
5565
|
+
/**
|
|
5566
|
+
* The ONE status-derivation function — used by the store and the kici-admin
|
|
5567
|
+
* host CLI. `ready` requires the host to be genuinely live (connected to some
|
|
5568
|
+
* instance AND heartbeat fresh — the freshness check catches a crashed
|
|
5569
|
+
* instance that never cleared `connected_instance_id`). It never returns
|
|
5570
|
+
* `ready` for a not-currently-live host: a not-live `static` reads
|
|
5571
|
+
* `unreachable` (the declared-but-absent alarm), a not-live `ephemeral` reads
|
|
5572
|
+
* `stale` (scaled down, awaiting reap).
|
|
5573
|
+
*/
|
|
5574
|
+
function deriveHostStatus(row, nowMs, graceMs) {
|
|
5575
|
+
const ageMs = nowMs - new Date(row.last_seen).getTime();
|
|
5576
|
+
if (row.connected_instance_id !== null && ageMs <= graceMs) return "ready";
|
|
5577
|
+
if (row.lifecycle_class === "ephemeral") return "stale";
|
|
5578
|
+
return "unreachable";
|
|
5579
|
+
}
|
|
5580
|
+
var HostRosterStore;
|
|
5581
|
+
var init_host_roster = __esmMin((() => {
|
|
5582
|
+
HostRosterStore = class {
|
|
5583
|
+
db;
|
|
5584
|
+
constructor(db) {
|
|
5585
|
+
this.db = db;
|
|
5586
|
+
}
|
|
5587
|
+
/** Idempotent upsert on agent_id; stamps connected_instance_id + last_seen. */
|
|
5588
|
+
async upsert(input) {
|
|
5589
|
+
const labelsJson = JSON.stringify(input.labels);
|
|
5590
|
+
await this.db.insertInto("host_roster").values({
|
|
5591
|
+
agent_id: input.agentId,
|
|
5592
|
+
token_id: input.tokenId,
|
|
5593
|
+
lifecycle_class: input.lifecycleClass,
|
|
5594
|
+
labels: labelsJson,
|
|
5595
|
+
hostname: input.hostname,
|
|
5596
|
+
platform: input.platform,
|
|
5597
|
+
arch: input.arch,
|
|
5598
|
+
connected_instance_id: input.instanceId,
|
|
5599
|
+
last_seen: sql`now()`,
|
|
5600
|
+
updated_at: sql`now()`
|
|
5601
|
+
}).onConflict((oc) => oc.column("agent_id").doUpdateSet({
|
|
5602
|
+
token_id: input.tokenId,
|
|
5603
|
+
lifecycle_class: input.lifecycleClass,
|
|
5604
|
+
labels: labelsJson,
|
|
5605
|
+
hostname: input.hostname,
|
|
5606
|
+
platform: input.platform,
|
|
5607
|
+
arch: input.arch,
|
|
5608
|
+
connected_instance_id: input.instanceId,
|
|
5609
|
+
last_seen: sql`now()`,
|
|
5610
|
+
updated_at: sql`now()`
|
|
5611
|
+
})).execute();
|
|
5612
|
+
}
|
|
5613
|
+
/** Clear liveness on disconnect — but only if THIS instance still owns it. */
|
|
5614
|
+
async markDisconnected(agentId, instanceId) {
|
|
5615
|
+
await this.db.updateTable("host_roster").set({
|
|
5616
|
+
connected_instance_id: null,
|
|
5617
|
+
updated_at: sql`now()`
|
|
5618
|
+
}).where("agent_id", "=", agentId).where("connected_instance_id", "=", instanceId).execute();
|
|
5619
|
+
}
|
|
5620
|
+
/** Coarse heartbeat stamp — same owner-guard as markDisconnected. */
|
|
5621
|
+
async stampLastSeen(agentId, instanceId) {
|
|
5622
|
+
await this.db.updateTable("host_roster").set({ last_seen: sql`now()` }).where("agent_id", "=", agentId).where("connected_instance_id", "=", instanceId).execute();
|
|
5623
|
+
}
|
|
5624
|
+
/** Operator pre-declare of a static host before its agent dials in. */
|
|
5625
|
+
async declareStatic(input) {
|
|
5626
|
+
await this.db.insertInto("host_roster").values({
|
|
5627
|
+
agent_id: input.agentId,
|
|
5628
|
+
token_id: null,
|
|
5629
|
+
lifecycle_class: "static",
|
|
5630
|
+
labels: JSON.stringify(input.labels),
|
|
5631
|
+
hostname: input.hostname ?? null,
|
|
5632
|
+
connected_instance_id: null,
|
|
5633
|
+
last_seen: sql`now()`,
|
|
5634
|
+
updated_at: sql`now()`
|
|
5635
|
+
}).onConflict((oc) => oc.column("agent_id").doNothing()).execute();
|
|
5636
|
+
}
|
|
5637
|
+
async get(agentId) {
|
|
5638
|
+
return await this.db.selectFrom("host_roster").selectAll().where("agent_id", "=", agentId).executeTakeFirst() ?? null;
|
|
5639
|
+
}
|
|
5640
|
+
async listAll() {
|
|
5641
|
+
return this.db.selectFrom("host_roster").selectAll().orderBy("agent_id", "asc").execute();
|
|
5642
|
+
}
|
|
5643
|
+
/**
|
|
5644
|
+
* Resolve every roster host matching a `runsOnAll` predicate (OR-of-AND
|
|
5645
|
+
* include groups, minus exclude labels), tagged with its derived status. This
|
|
5646
|
+
* is the host-fanout resolver: it returns declared-but-absent static hosts
|
|
5647
|
+
* (status `unreachable`) so the caller can apply `onUnreachable` — the live
|
|
5648
|
+
* registry alone cannot name an expected-but-absent host.
|
|
5649
|
+
*/
|
|
5650
|
+
async findMatching(include, exclude, graceMs) {
|
|
5651
|
+
const rows = await this.db.selectFrom("host_roster").selectAll().execute();
|
|
5652
|
+
const now = Date.now();
|
|
5653
|
+
const out = [];
|
|
5654
|
+
for (const row of rows) {
|
|
5655
|
+
const labels = JSON.parse(row.labels);
|
|
5656
|
+
const set = new Set(labels);
|
|
5657
|
+
if (exclude.some((e) => matcherSatisfiedBy(e, set))) continue;
|
|
5658
|
+
if (include.length && !include.some((grp) => grp.every((m) => matcherSatisfiedBy(m, set)))) continue;
|
|
5659
|
+
out.push({
|
|
5660
|
+
agentId: row.agent_id,
|
|
5661
|
+
host: row.hostname ?? row.agent_id,
|
|
5662
|
+
labels,
|
|
5663
|
+
lifecycleClass: row.lifecycle_class,
|
|
5664
|
+
connectedInstanceId: row.connected_instance_id,
|
|
5665
|
+
status: deriveHostStatus(row, now, graceMs),
|
|
5666
|
+
platform: row.platform,
|
|
5667
|
+
arch: row.arch
|
|
5668
|
+
});
|
|
5669
|
+
}
|
|
5670
|
+
out.sort((a, b) => a.agentId.localeCompare(b.agentId));
|
|
5671
|
+
return out;
|
|
5672
|
+
}
|
|
5673
|
+
/**
|
|
5674
|
+
* Count `static` (declared) hosts whose derived status is `unreachable` —
|
|
5675
|
+
* the "declared-but-absent" alarm population. Reuses the single-source
|
|
5676
|
+
* {@link deriveHostStatus} so the count never diverges from what
|
|
5677
|
+
* `kici-admin host list` shows. A not-currently-connected static host reads
|
|
5678
|
+
* `unreachable` regardless of grace (only the connected-but-stale case
|
|
5679
|
+
* depends on `graceMs`).
|
|
5680
|
+
*/
|
|
5681
|
+
async countStaticUnreachable(graceMs) {
|
|
5682
|
+
const rows = await this.db.selectFrom("host_roster").selectAll().where("lifecycle_class", "=", "static").execute();
|
|
5683
|
+
const now = Date.now();
|
|
5684
|
+
return rows.filter((r) => deriveHostStatus(r, now, graceMs) === "unreachable").length;
|
|
5685
|
+
}
|
|
5686
|
+
/** Delete ephemeral rows whose last_seen is older than ttl. Returns count. */
|
|
5687
|
+
async reapEphemeralPastTtl(ttlMs) {
|
|
5688
|
+
const res = await this.db.deleteFrom("host_roster").where("lifecycle_class", "=", "ephemeral").where("last_seen", "<", sql`now() - (${ttlMs}::text || ' milliseconds')::interval`).executeTakeFirst();
|
|
5689
|
+
return Number(res.numDeletedRows ?? 0n);
|
|
5690
|
+
}
|
|
5691
|
+
};
|
|
5692
|
+
}));
|
|
5693
|
+
//#endregion
|
|
5694
|
+
//#region src/agent/host-roster-reaper.ts
|
|
5695
|
+
var logger$83, HostRosterReaper;
|
|
5696
|
+
var init_host_roster_reaper = __esmMin((() => {
|
|
5697
|
+
logger$83 = createLogger({ prefix: "host-roster-reaper" });
|
|
5698
|
+
HostRosterReaper = class {
|
|
5699
|
+
store;
|
|
5700
|
+
ttlMs;
|
|
5701
|
+
graceMs;
|
|
5702
|
+
scanIntervalMs;
|
|
5703
|
+
setUnreachableGauge;
|
|
5704
|
+
timer = null;
|
|
5705
|
+
isLeader = false;
|
|
5706
|
+
constructor(opts) {
|
|
5707
|
+
this.store = opts.store;
|
|
5708
|
+
this.ttlMs = opts.ttlMs;
|
|
5709
|
+
this.graceMs = opts.graceMs;
|
|
5710
|
+
this.scanIntervalMs = opts.scanIntervalMs;
|
|
5711
|
+
this.setUnreachableGauge = opts.setUnreachableGauge;
|
|
5712
|
+
}
|
|
5713
|
+
onBecomeLeader() {
|
|
5714
|
+
if (this.timer) {
|
|
5715
|
+
clearInterval(this.timer);
|
|
5716
|
+
this.timer = null;
|
|
5717
|
+
}
|
|
5718
|
+
this.isLeader = true;
|
|
5719
|
+
logger$83.info("Became leader, starting host roster reaper", {
|
|
5720
|
+
ttlMs: this.ttlMs,
|
|
5721
|
+
scanIntervalMs: this.scanIntervalMs
|
|
5722
|
+
});
|
|
5723
|
+
this.timer = setInterval(() => {
|
|
5724
|
+
this.tick().catch((err) => logger$83.error("roster reaper tick failed", { error: toErrorMessage(err) }));
|
|
5725
|
+
}, this.scanIntervalMs);
|
|
5726
|
+
this.timer.unref?.();
|
|
5727
|
+
}
|
|
5728
|
+
onLoseLeadership() {
|
|
5729
|
+
this.isLeader = false;
|
|
5730
|
+
if (this.timer) {
|
|
5731
|
+
clearInterval(this.timer);
|
|
5732
|
+
this.timer = null;
|
|
5733
|
+
}
|
|
5734
|
+
logger$83.info("Lost leadership, stopped host roster reaper");
|
|
5735
|
+
}
|
|
5736
|
+
stop() {
|
|
5737
|
+
if (this.timer) {
|
|
5738
|
+
clearInterval(this.timer);
|
|
5739
|
+
this.timer = null;
|
|
5740
|
+
}
|
|
5741
|
+
this.isLeader = false;
|
|
5742
|
+
}
|
|
5743
|
+
/** One reap pass. Public for tests. */
|
|
5744
|
+
async tick() {
|
|
5745
|
+
if (!this.isLeader) return;
|
|
5746
|
+
const deleted = await this.store.reapEphemeralPastTtl(this.ttlMs);
|
|
5747
|
+
if (deleted > 0) logger$83.info("Reaped expired ephemeral hosts", { deleted });
|
|
5748
|
+
const unreachable = await this.store.countStaticUnreachable(this.graceMs);
|
|
5749
|
+
this.setUnreachableGauge(unreachable);
|
|
5750
|
+
if (unreachable > 0) logger$83.warn("Declared hosts unreachable", { count: unreachable });
|
|
5751
|
+
}
|
|
5752
|
+
};
|
|
5753
|
+
}));
|
|
5754
|
+
//#endregion
|
|
5755
|
+
//#region src/queue/job-queue.ts
|
|
5756
|
+
/**
|
|
5757
|
+
* Parse a `dispatch_queue` jsonb pattern column into a `LabelMatcher[]`. Handles
|
|
5758
|
+
* both the auto-parsed array form (from the pg driver) and the JSON string form
|
|
5759
|
+
* (from tests / a non-parsing driver). A missing / malformed value yields `[]`.
|
|
5760
|
+
*/
|
|
5761
|
+
function parseMatcherColumn(v) {
|
|
5762
|
+
if (Array.isArray(v)) return v;
|
|
5763
|
+
if (typeof v === "string") return JSON.parse(v);
|
|
5764
|
+
return [];
|
|
5765
|
+
}
|
|
5766
|
+
/**
|
|
5767
|
+
* Whether an agent's label set satisfies a job's regex matchers: every
|
|
5768
|
+
* `runsOnPatterns` matcher must match some label AND no `excludePatterns`
|
|
5769
|
+
* matcher may match any label. The single matching authority is the engine's
|
|
5770
|
+
* `matcherSatisfiedBy` (JS RegExp) — never a Postgres `~`.
|
|
5771
|
+
*/
|
|
5772
|
+
function jobPatternsSatisfiedBy(job, labels) {
|
|
5773
|
+
if (!job.runsOnPatterns.every((p) => matcherSatisfiedBy(p, labels))) return false;
|
|
5774
|
+
if (job.excludePatterns.some((p) => matcherSatisfiedBy(p, labels))) return false;
|
|
5775
|
+
return true;
|
|
5776
|
+
}
|
|
5777
|
+
var JobQueue;
|
|
5432
5778
|
var init_job_queue = __esmMin((() => {
|
|
5433
5779
|
JobQueue = class {
|
|
5434
5780
|
db;
|
|
@@ -5480,7 +5826,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
5480
5826
|
deps_hash: job.depsHash ?? null,
|
|
5481
5827
|
request_id: job.requestId ?? null,
|
|
5482
5828
|
exclude_labels: JSON.stringify(job.excludeLabels ?? []),
|
|
5483
|
-
|
|
5829
|
+
runs_on_patterns: JSON.stringify(job.runsOnPatterns ?? []),
|
|
5830
|
+
exclude_patterns: JSON.stringify(job.excludePatterns ?? []),
|
|
5831
|
+
routing_key: job.routingKey,
|
|
5832
|
+
pinned_agent_id: job.pinnedAgentId ?? null
|
|
5484
5833
|
}).execute();
|
|
5485
5834
|
this.depthCache = null;
|
|
5486
5835
|
this.breakdownCache = null;
|
|
@@ -5507,18 +5856,79 @@ var init_job_queue = __esmMin((() => {
|
|
|
5507
5856
|
* (empty for static / non-scaler agents).
|
|
5508
5857
|
* @returns The matching job, or null if none found.
|
|
5509
5858
|
*/
|
|
5510
|
-
async dequeueForLabels(agentLabels, agentMandatoryLabels = []) {
|
|
5859
|
+
async dequeueForLabels(agentLabels, agentMandatoryLabels = [], agentId) {
|
|
5860
|
+
const fast = await this.claimPatternFree(agentLabels, agentMandatoryLabels, agentId);
|
|
5861
|
+
if (fast) return fast;
|
|
5862
|
+
return this.claimWithPatterns(agentLabels, agentMandatoryLabels, agentId);
|
|
5863
|
+
}
|
|
5864
|
+
/**
|
|
5865
|
+
* Build the shared drain WHERE chain (status / expiry / exact-label @> /
|
|
5866
|
+
* exclude-label / pin / mandatory-label gate) common to both drain passes.
|
|
5867
|
+
* The pattern columns are NOT filtered here — each pass adds its own
|
|
5868
|
+
* pattern-free / pattern-bearing guard on top.
|
|
5869
|
+
*/
|
|
5870
|
+
drainBaseQuery(agentLabels, agentMandatoryLabels, agentId) {
|
|
5511
5871
|
const agentLabelsJson = JSON.stringify(agentLabels);
|
|
5512
5872
|
const mandatoryLabelsJson = JSON.stringify(agentMandatoryLabels);
|
|
5513
5873
|
let query = this.db.selectFrom("dispatch_queue").selectAll().where("status", "=", "pending").where(sql`(expires_at IS NULL OR expires_at >= now())`).where(sql`${sql.lit(agentLabelsJson)}::jsonb @> runs_on_labels`).where(sql`NOT EXISTS (
|
|
5514
5874
|
SELECT 1 FROM jsonb_array_elements_text(exclude_labels) AS e
|
|
5515
5875
|
WHERE e.value = ANY(${sql.val(agentLabels)}::text[])
|
|
5516
5876
|
)`);
|
|
5877
|
+
query = query.where(sql`(pinned_agent_id IS NULL${agentId ? sql` OR pinned_agent_id = ${sql.val(agentId)}` : sql``})`);
|
|
5517
5878
|
if (agentMandatoryLabels.length > 0) query = query.where(sql`runs_on_labels @> ${sql.lit(mandatoryLabelsJson)}::jsonb`);
|
|
5518
|
-
|
|
5879
|
+
return query;
|
|
5880
|
+
}
|
|
5881
|
+
/**
|
|
5882
|
+
* Fast path: claim the oldest pending pattern-free row. The
|
|
5883
|
+
* `runs_on_patterns = '[]' AND exclude_patterns = '[]'` guard restricts this
|
|
5884
|
+
* pass to rows that need no JS post-filter, so the single-row atomic claim
|
|
5885
|
+
* (FOR UPDATE SKIP LOCKED) keeps the original hot-path semantics intact.
|
|
5886
|
+
*/
|
|
5887
|
+
async claimPatternFree(agentLabels, agentMandatoryLabels, agentId) {
|
|
5888
|
+
const row = await this.drainBaseQuery(agentLabels, agentMandatoryLabels, agentId).where(sql`runs_on_patterns = '[]'::jsonb AND exclude_patterns = '[]'::jsonb`).orderBy("created_at", "asc").limit(1).forUpdate().skipLocked().executeTakeFirst();
|
|
5519
5889
|
return row ? this.rowToQueuedJob(row) : null;
|
|
5520
5890
|
}
|
|
5521
5891
|
/**
|
|
5892
|
+
* Pattern path: load a small batch of pattern-bearing candidate rows, apply
|
|
5893
|
+
* the JS regex post-filter (matcherSatisfiedBy), and atomically claim the
|
|
5894
|
+
* first match by id with a conditional `status = Pending` guard. The claim is
|
|
5895
|
+
* a conditional UPDATE rather than relying on the SELECT lock alone because
|
|
5896
|
+
* the JS filter runs after the per-statement lock window has closed, so two
|
|
5897
|
+
* agents could both pass the filter for the same row; the `where status =
|
|
5898
|
+
* Pending` makes exactly one of them win. The claim transitions the row to
|
|
5899
|
+
* Dispatched, matching the value the caller-side markDispatched would set
|
|
5900
|
+
* (which then re-sets it idempotently).
|
|
5901
|
+
*/
|
|
5902
|
+
async claimWithPatterns(agentLabels, agentMandatoryLabels, agentId) {
|
|
5903
|
+
const labelSet = new Set(agentLabels);
|
|
5904
|
+
const rows = await this.drainBaseQuery(agentLabels, agentMandatoryLabels, agentId).where(sql`(runs_on_patterns <> '[]'::jsonb OR exclude_patterns <> '[]'::jsonb)`).orderBy("created_at", "asc").limit(10).forUpdate().skipLocked().execute();
|
|
5905
|
+
for (const row of rows) {
|
|
5906
|
+
const job = this.rowToQueuedJob(row);
|
|
5907
|
+
if (!jobPatternsSatisfiedBy(job, labelSet)) continue;
|
|
5908
|
+
if (((await this.db.updateTable("dispatch_queue").set({ status: "dispatched" }).where("id", "=", row.id).where("status", "=", "pending").executeTakeFirst()).numUpdatedRows ?? 0n) > 0n) return job;
|
|
5909
|
+
}
|
|
5910
|
+
return null;
|
|
5911
|
+
}
|
|
5912
|
+
/**
|
|
5913
|
+
* Atomically claim the oldest pending job pinned to a specific agent. Used by
|
|
5914
|
+
* the eager pin drain when the pinned agent (re)registers or frees a slot —
|
|
5915
|
+
* the host-fanout analog of `dispatchBoundJob`'s eager path. Ignores the exact
|
|
5916
|
+
* label gate: the pin was resolved against the roster at materialize time.
|
|
5917
|
+
*
|
|
5918
|
+
* Still applies the JS regex post-filter (`jobPatternsSatisfiedBy`) when
|
|
5919
|
+
* `agentLabels` is supplied, mirroring `dequeueById`: a pinned child whose
|
|
5920
|
+
* `runsOn`/`exclude` patterns no longer match the agent's current labels must
|
|
5921
|
+
* not be claimed. The single matching authority is the engine's
|
|
5922
|
+
* `matcherSatisfiedBy` (never a Postgres `~`).
|
|
5923
|
+
*/
|
|
5924
|
+
async dequeueByPinnedAgent(agentId, agentLabels) {
|
|
5925
|
+
const row = await this.db.selectFrom("dispatch_queue").selectAll().where("status", "=", "pending").where("pinned_agent_id", "=", agentId).where(sql`(expires_at IS NULL OR expires_at >= now())`).orderBy("created_at", "asc").limit(1).forUpdate().skipLocked().executeTakeFirst();
|
|
5926
|
+
if (!row) return null;
|
|
5927
|
+
const job = this.rowToQueuedJob(row);
|
|
5928
|
+
if (agentLabels && !jobPatternsSatisfiedBy(job, new Set(agentLabels))) return null;
|
|
5929
|
+
return job;
|
|
5930
|
+
}
|
|
5931
|
+
/**
|
|
5522
5932
|
* Atomically claim a specific pending job by ID, validating it still
|
|
5523
5933
|
* matches the agent's labels and isn't expired.
|
|
5524
5934
|
*
|
|
@@ -5544,7 +5954,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
5544
5954
|
)`);
|
|
5545
5955
|
if (agentMandatoryLabels.length > 0) query = query.where(sql`runs_on_labels @> ${sql.lit(mandatoryLabelsJson)}::jsonb`);
|
|
5546
5956
|
const row = await query.forUpdate().skipLocked().executeTakeFirst();
|
|
5547
|
-
|
|
5957
|
+
if (!row) return null;
|
|
5958
|
+
const job = this.rowToQueuedJob(row);
|
|
5959
|
+
if (!jobPatternsSatisfiedBy(job, new Set(agentLabels))) return null;
|
|
5960
|
+
return job;
|
|
5548
5961
|
}
|
|
5549
5962
|
/**
|
|
5550
5963
|
* Insert a job directly with status='dispatched' (bypasses the queue).
|
|
@@ -5576,7 +5989,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
5576
5989
|
deps_hash: job.depsHash ?? null,
|
|
5577
5990
|
request_id: job.requestId ?? null,
|
|
5578
5991
|
exclude_labels: JSON.stringify(job.excludeLabels ?? []),
|
|
5579
|
-
|
|
5992
|
+
runs_on_patterns: JSON.stringify(job.runsOnPatterns ?? []),
|
|
5993
|
+
exclude_patterns: JSON.stringify(job.excludePatterns ?? []),
|
|
5994
|
+
routing_key: job.routingKey,
|
|
5995
|
+
pinned_agent_id: job.pinnedAgentId ?? null
|
|
5580
5996
|
}).execute();
|
|
5581
5997
|
return id;
|
|
5582
5998
|
}
|
|
@@ -6006,7 +6422,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
6006
6422
|
depsHash: row.deps_hash ?? void 0,
|
|
6007
6423
|
requestId: row.request_id ?? void 0,
|
|
6008
6424
|
excludeLabels: typeof row.exclude_labels === "string" ? JSON.parse(row.exclude_labels) : Array.isArray(row.exclude_labels) ? row.exclude_labels : [],
|
|
6009
|
-
|
|
6425
|
+
runsOnPatterns: parseMatcherColumn(row.runs_on_patterns),
|
|
6426
|
+
excludePatterns: parseMatcherColumn(row.exclude_patterns),
|
|
6427
|
+
routingKey: row.routing_key,
|
|
6428
|
+
pinnedAgentId: row.pinned_agent_id ?? void 0
|
|
6010
6429
|
};
|
|
6011
6430
|
}
|
|
6012
6431
|
};
|
|
@@ -6111,14 +6530,66 @@ var init_cleanup$1 = __esmMin((() => {
|
|
|
6111
6530
|
* Prometheus can address multiple KiCI services from one dashboard.
|
|
6112
6531
|
* The orchestrator runs setInterval-driven schedulers (no pg-boss
|
|
6113
6532
|
* dependency) but the outward observability shape is identical.
|
|
6533
|
+
*
|
|
6534
|
+
* Lazy meter + instrument initialization: the `@kici-dev/shared` barrel
|
|
6535
|
+
* is imported statically at the top of the orchestrator entry points
|
|
6536
|
+
* (server.ts, standalone.ts), which evaluates this module BEFORE
|
|
6537
|
+
* `initTelemetry()` sets the global MeterProvider. Creating the meter or
|
|
6538
|
+
* any instrument at module-eval time would bind it to the no-op provider,
|
|
6539
|
+
* so its samples never reach the Prometheus exporter (the `/metrics`
|
|
6540
|
+
* scrape) or the Platform push. We therefore resolve the meter and every
|
|
6541
|
+
* instrument on first use — counters/histograms on first `.add()` /
|
|
6542
|
+
* `.record()`, observable gauges on first `.set()` — all of which happen
|
|
6543
|
+
* at job-run time, long after telemetry is wired up. Same hazard, same
|
|
6544
|
+
* fix as `prometheus.ts` and `@kici-dev/shared`'s cold-store metrics
|
|
6545
|
+
* module.
|
|
6546
|
+
*/
|
|
6547
|
+
function meter() {
|
|
6548
|
+
if (!_meter) _meter = createMeter("kici-orchestrator");
|
|
6549
|
+
return _meter;
|
|
6550
|
+
}
|
|
6551
|
+
/**
|
|
6552
|
+
* A counter whose real OTel instrument is created on first `.add()` —
|
|
6553
|
+
* after `initTelemetry()` has registered the global MeterProvider. The
|
|
6554
|
+
* returned value satisfies the OTel `Counter` interface, so call sites
|
|
6555
|
+
* are unchanged.
|
|
6556
|
+
*/
|
|
6557
|
+
function lazyCounter(name, description) {
|
|
6558
|
+
let inst;
|
|
6559
|
+
const get = () => inst ??= meter().createCounter(name, { description });
|
|
6560
|
+
return { add: (...args) => get().add(...args) };
|
|
6561
|
+
}
|
|
6562
|
+
/**
|
|
6563
|
+
* A histogram whose real OTel instrument is created on first `.record()` —
|
|
6564
|
+
* after `initTelemetry()`. Satisfies the OTel `Histogram` interface.
|
|
6565
|
+
*/
|
|
6566
|
+
function lazyHistogram(name, description, explicitBucketBoundaries) {
|
|
6567
|
+
let inst;
|
|
6568
|
+
const get = () => inst ??= meter().createHistogram(name, {
|
|
6569
|
+
description,
|
|
6570
|
+
advice: { explicitBucketBoundaries }
|
|
6571
|
+
});
|
|
6572
|
+
return { record: (...args) => get().record(...args) };
|
|
6573
|
+
}
|
|
6574
|
+
/**
|
|
6575
|
+
* An observable gauge backed by per-label state. The underlying OTel
|
|
6576
|
+
* instrument + scrape callback are created on first `.set()`, so the
|
|
6577
|
+
* gauge binds to the real MeterProvider rather than the no-op one. The
|
|
6578
|
+
* callback reads the latest per-label value on each Prometheus scrape.
|
|
6114
6579
|
*/
|
|
6115
6580
|
function createGaugeWithState(name, description) {
|
|
6116
6581
|
const state = /* @__PURE__ */ new Map();
|
|
6117
|
-
|
|
6118
|
-
|
|
6119
|
-
|
|
6582
|
+
let registered = false;
|
|
6583
|
+
function ensureRegistered() {
|
|
6584
|
+
if (registered) return;
|
|
6585
|
+
registered = true;
|
|
6586
|
+
meter().createObservableGauge(name, { description }).addCallback((result) => {
|
|
6587
|
+
for (const entry of state.values()) result.observe(entry.value, entry.attributes);
|
|
6588
|
+
});
|
|
6589
|
+
}
|
|
6120
6590
|
return {
|
|
6121
6591
|
set(labels, value) {
|
|
6592
|
+
ensureRegistered();
|
|
6122
6593
|
const key = JSON.stringify(labels);
|
|
6123
6594
|
state.set(key, {
|
|
6124
6595
|
attributes: labels,
|
|
@@ -6130,24 +6601,20 @@ function createGaugeWithState(name, description) {
|
|
|
6130
6601
|
}
|
|
6131
6602
|
};
|
|
6132
6603
|
}
|
|
6133
|
-
var
|
|
6604
|
+
var _meter, jobRunsTotal, jobDurationSeconds, jobLastSuccessTimestamp, jobLastFailureTimestamp, jobConsecutiveFailures;
|
|
6134
6605
|
var init_scheduled_jobs = __esmMin((() => {
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
120,
|
|
6148
|
-
600
|
|
6149
|
-
] }
|
|
6150
|
-
});
|
|
6606
|
+
jobRunsTotal = lazyCounter("kici_orch_job_runs_total", "Scheduled job run outcomes, by job and result (success/failure)");
|
|
6607
|
+
jobDurationSeconds = lazyHistogram("kici_orch_job_duration_seconds", "Scheduled job tick duration in seconds", [
|
|
6608
|
+
.01,
|
|
6609
|
+
.05,
|
|
6610
|
+
.1,
|
|
6611
|
+
.5,
|
|
6612
|
+
1,
|
|
6613
|
+
5,
|
|
6614
|
+
30,
|
|
6615
|
+
120,
|
|
6616
|
+
600
|
|
6617
|
+
]);
|
|
6151
6618
|
jobLastSuccessTimestamp = createGaugeWithState("kici_orch_job_last_success_timestamp_seconds", "Unix timestamp of the most recent successful scheduled job tick");
|
|
6152
6619
|
jobLastFailureTimestamp = createGaugeWithState("kici_orch_job_last_failure_timestamp_seconds", "Unix timestamp of the most recent failed scheduled job tick");
|
|
6153
6620
|
jobConsecutiveFailures = createGaugeWithState("kici_orch_job_consecutive_failures", "Consecutive failures for each scheduled job since its last success");
|
|
@@ -8428,7 +8895,8 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8428
8895
|
* 4. If queue full: return 'rejected'.
|
|
8429
8896
|
*/
|
|
8430
8897
|
async dispatch(job) {
|
|
8431
|
-
|
|
8898
|
+
if (job.pinnedAgentId) return this.dispatchPinned(job);
|
|
8899
|
+
const available = this.registry.findAvailable(job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? []);
|
|
8432
8900
|
if (available.length > 0) {
|
|
8433
8901
|
const agent = available[0];
|
|
8434
8902
|
this.registry.incrementActiveJobs(agent.agentId);
|
|
@@ -8461,6 +8929,8 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8461
8929
|
depsHash: job.depsHash,
|
|
8462
8930
|
requestId: job.requestId,
|
|
8463
8931
|
excludeLabels: job.excludeLabels ?? [],
|
|
8932
|
+
runsOnPatterns: job.runsOnPatterns ?? [],
|
|
8933
|
+
excludePatterns: job.excludePatterns ?? [],
|
|
8464
8934
|
routingKey: job.routingKey,
|
|
8465
8935
|
resources: job.resources
|
|
8466
8936
|
};
|
|
@@ -8490,7 +8960,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8490
8960
|
throw err;
|
|
8491
8961
|
}
|
|
8492
8962
|
if (this.onNoMatchingAgent) {
|
|
8493
|
-
if ((await this.onNoMatchingAgent(job.runsOnLabels, jobId, job.runId, job.excludeLabels ?? [], job.resources)).action === "no-backend" && !this.registry.hasMatchingAgent(job.runsOnLabels, job.excludeLabels ?? [])) return {
|
|
8963
|
+
if ((await this.onNoMatchingAgent(job.runsOnLabels, jobId, job.runId, job.excludeLabels ?? [], job.resources)).action === "no-backend" && !this.registry.hasMatchingAgent(job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? [])) return {
|
|
8494
8964
|
status: "queued-no-backend",
|
|
8495
8965
|
jobId
|
|
8496
8966
|
};
|
|
@@ -8501,6 +8971,84 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8501
8971
|
};
|
|
8502
8972
|
}
|
|
8503
8973
|
/**
|
|
8974
|
+
* Dispatch a host-fanout pinned child. The job targets exactly
|
|
8975
|
+
* `job.pinnedAgentId`: if that agent is locally connected, satisfies the
|
|
8976
|
+
* runsOn/exclude/mandatory gate, and has capacity, dispatch immediately;
|
|
8977
|
+
* otherwise queue it WITH the pin so the pin-aware drain delivers it when the
|
|
8978
|
+
* agent frees up or (re)connects. A pinned job never falls through to a
|
|
8979
|
+
* different agent.
|
|
8980
|
+
*/
|
|
8981
|
+
async dispatchPinned(job) {
|
|
8982
|
+
const agentId = job.pinnedAgentId;
|
|
8983
|
+
const agent = this.registry.get(agentId);
|
|
8984
|
+
if (agent && this.registry.agentSatisfies(agent, job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? []) && agent.activeJobs < agent.maxConcurrency) {
|
|
8985
|
+
this.registry.incrementActiveJobs(agentId);
|
|
8986
|
+
let jobId;
|
|
8987
|
+
try {
|
|
8988
|
+
jobId = await this.queue.insertDispatched(job);
|
|
8989
|
+
} catch (err) {
|
|
8990
|
+
this.registry.decrementActiveJobs(agentId);
|
|
8991
|
+
throw err;
|
|
8992
|
+
}
|
|
8993
|
+
const queuedJob = {
|
|
8994
|
+
id: jobId,
|
|
8995
|
+
runId: job.runId,
|
|
8996
|
+
workflowName: job.workflowName,
|
|
8997
|
+
jobName: job.jobName,
|
|
8998
|
+
runsOnLabels: job.runsOnLabels,
|
|
8999
|
+
jobConfig: job.jobConfig,
|
|
9000
|
+
repoUrl: job.repoUrl,
|
|
9001
|
+
ref: job.ref,
|
|
9002
|
+
sha: job.sha,
|
|
9003
|
+
status: "dispatched",
|
|
9004
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9005
|
+
expiresAt: null,
|
|
9006
|
+
deliveryId: job.deliveryId,
|
|
9007
|
+
provider: job.provider,
|
|
9008
|
+
providerContext: job.providerContext,
|
|
9009
|
+
sourceTarUrl: job.sourceTarUrl,
|
|
9010
|
+
sourceTarHash: job.sourceTarHash,
|
|
9011
|
+
depsUrl: job.depsUrl,
|
|
9012
|
+
depsHash: job.depsHash,
|
|
9013
|
+
requestId: job.requestId,
|
|
9014
|
+
excludeLabels: job.excludeLabels ?? [],
|
|
9015
|
+
runsOnPatterns: job.runsOnPatterns ?? [],
|
|
9016
|
+
excludePatterns: job.excludePatterns ?? [],
|
|
9017
|
+
routingKey: job.routingKey,
|
|
9018
|
+
resources: job.resources,
|
|
9019
|
+
pinnedAgentId: agentId
|
|
9020
|
+
};
|
|
9021
|
+
this.trackJobForAgent(agentId, jobId, queuedJob.runId);
|
|
9022
|
+
await this.onDispatch(agentId, queuedJob);
|
|
9023
|
+
await this.armAckDeadline(agentId, queuedJob);
|
|
9024
|
+
this.metrics.incJobsDispatched("dispatched");
|
|
9025
|
+
return {
|
|
9026
|
+
status: "dispatched",
|
|
9027
|
+
agentId,
|
|
9028
|
+
jobId
|
|
9029
|
+
};
|
|
9030
|
+
}
|
|
9031
|
+
let jobId;
|
|
9032
|
+
try {
|
|
9033
|
+
jobId = await this.queue.enqueue(job);
|
|
9034
|
+
this.metrics.incJobsDispatched("queued");
|
|
9035
|
+
await this.updateQueueDepthMetric();
|
|
9036
|
+
} catch (err) {
|
|
9037
|
+
if (toErrorMessage(err) === "queue full") {
|
|
9038
|
+
this.metrics.incJobsDispatched("rejected");
|
|
9039
|
+
return {
|
|
9040
|
+
status: "rejected",
|
|
9041
|
+
reason: "queue full"
|
|
9042
|
+
};
|
|
9043
|
+
}
|
|
9044
|
+
throw err;
|
|
9045
|
+
}
|
|
9046
|
+
return {
|
|
9047
|
+
status: "queued",
|
|
9048
|
+
jobId
|
|
9049
|
+
};
|
|
9050
|
+
}
|
|
9051
|
+
/**
|
|
8504
9052
|
* Eagerly dispatch a specific bound job to a freshly-registered
|
|
8505
9053
|
* scaler-managed agent.
|
|
8506
9054
|
*
|
|
@@ -8554,7 +9102,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8554
9102
|
this.registry.incrementActiveJobs(agentId);
|
|
8555
9103
|
let job = null;
|
|
8556
9104
|
try {
|
|
8557
|
-
job = await this.queue.dequeueForLabels(agentLabels, agentMandatoryLabels);
|
|
9105
|
+
job = await this.queue.dequeueByPinnedAgent(agentId, agentLabels) ?? await this.queue.dequeueForLabels(agentLabels, agentMandatoryLabels, agentId);
|
|
8558
9106
|
} finally {
|
|
8559
9107
|
if (!job) this.registry.decrementActiveJobs(agentId);
|
|
8560
9108
|
}
|
|
@@ -8770,7 +9318,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8770
9318
|
async redispatch(jobId) {
|
|
8771
9319
|
const job = await this.queue.getFullJobById(jobId);
|
|
8772
9320
|
if (!job || job.status !== "pending") return;
|
|
8773
|
-
const available = this.registry.findAvailable(job.runsOnLabels, job.excludeLabels ?? []);
|
|
9321
|
+
const available = this.registry.findAvailable(job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? []);
|
|
8774
9322
|
if (available.length > 0) {
|
|
8775
9323
|
if (await this.dispatchBoundJob(available[0].agentId, jobId)) return;
|
|
8776
9324
|
}
|
|
@@ -9220,6 +9768,37 @@ var init_dispatcher = __esmMin((() => {
|
|
|
9220
9768
|
};
|
|
9221
9769
|
}));
|
|
9222
9770
|
//#endregion
|
|
9771
|
+
//#region src/lockfile-redos-guard.ts
|
|
9772
|
+
/**
|
|
9773
|
+
* Lock-load ReDoS revalidation.
|
|
9774
|
+
*
|
|
9775
|
+
* Re-validates every regex matcher in a fetched lock file before it is cached
|
|
9776
|
+
* or dispatched. Defense-in-depth against a hand-edited or non-compiled lock
|
|
9777
|
+
* that smuggled a ReDoS-prone pattern past the compile-time gate: the compiler
|
|
9778
|
+
* runs the same `assertMatchersSafe` check when it emits the lock, but the
|
|
9779
|
+
* orchestrator does not trust that the lock it fetched was produced by our
|
|
9780
|
+
* compiler.
|
|
9781
|
+
*/
|
|
9782
|
+
/**
|
|
9783
|
+
* Walk every static job's `runsOn` / `excludeLabels` / `runsOnAll` matchers and
|
|
9784
|
+
* throw if any regex matcher is ReDoS-prone. Dynamic job generators carry no
|
|
9785
|
+
* static routing matchers (they materialize jobs at eval time, which re-runs the
|
|
9786
|
+
* compile-time gate), so only static jobs are checked.
|
|
9787
|
+
*/
|
|
9788
|
+
function assertLockFileRegexesSafe(lockFile) {
|
|
9789
|
+
for (const wf of lockFile.workflows ?? []) for (const job of wf.jobs ?? []) {
|
|
9790
|
+
if (!isLockStaticJob(job)) continue;
|
|
9791
|
+
const ctx = `lock workflow '${wf.name}' job '${job.name}'`;
|
|
9792
|
+
if (job.runsOn) assertMatchersSafe(job.runsOn, `${ctx} runsOn`);
|
|
9793
|
+
if (job.excludeLabels) assertMatchersSafe(job.excludeLabels, `${ctx} excludeLabels`);
|
|
9794
|
+
if (job.runsOnAll) {
|
|
9795
|
+
for (const grp of job.runsOnAll.include ?? []) assertMatchersSafe(grp, `${ctx} runsOnAll`);
|
|
9796
|
+
if (job.runsOnAll.exclude) assertMatchersSafe(job.runsOnAll.exclude, `${ctx} runsOnAll exclude`);
|
|
9797
|
+
}
|
|
9798
|
+
}
|
|
9799
|
+
}
|
|
9800
|
+
var init_lockfile_redos_guard = __esmMin((() => {}));
|
|
9801
|
+
//#endregion
|
|
9223
9802
|
//#region src/lockfile-cache.ts
|
|
9224
9803
|
/**
|
|
9225
9804
|
* Provider-agnostic LRU cache for lock files.
|
|
@@ -9232,6 +9811,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
9232
9811
|
*/
|
|
9233
9812
|
var logger$76, LockFileCache;
|
|
9234
9813
|
var init_lockfile_cache = __esmMin((() => {
|
|
9814
|
+
init_lockfile_redos_guard();
|
|
9235
9815
|
logger$76 = createLogger({ prefix: "lockfile-cache" });
|
|
9236
9816
|
LockFileCache = class {
|
|
9237
9817
|
cache;
|
|
@@ -9287,6 +9867,17 @@ var init_lockfile_cache = __esmMin((() => {
|
|
|
9287
9867
|
});
|
|
9288
9868
|
return null;
|
|
9289
9869
|
}
|
|
9870
|
+
try {
|
|
9871
|
+
assertLockFileRegexesSafe(lockFile);
|
|
9872
|
+
} catch (error) {
|
|
9873
|
+
const message = toErrorMessage(error);
|
|
9874
|
+
logger$76.warn("Lock file carries a ReDoS-prone regex matcher", {
|
|
9875
|
+
repoIdentifier,
|
|
9876
|
+
ref,
|
|
9877
|
+
error: message
|
|
9878
|
+
});
|
|
9879
|
+
throw new LockFileParseError(repoIdentifier, ref, message);
|
|
9880
|
+
}
|
|
9290
9881
|
this.cache.set(cacheKey, lockFile);
|
|
9291
9882
|
return lockFile;
|
|
9292
9883
|
}
|
|
@@ -11634,16 +12225,25 @@ async function cancelRunWithReason(deps, runId, reason, options = {}) {
|
|
|
11634
12225
|
const agentId = dispatcher.getAgentIdForJob(jobId);
|
|
11635
12226
|
if (!agentId) continue;
|
|
11636
12227
|
const entry = registry.get(agentId);
|
|
11637
|
-
if (
|
|
11638
|
-
|
|
11639
|
-
|
|
11640
|
-
|
|
11641
|
-
|
|
11642
|
-
|
|
11643
|
-
|
|
11644
|
-
|
|
11645
|
-
|
|
11646
|
-
|
|
12228
|
+
if (entry?.ws?.readyState !== 1) continue;
|
|
12229
|
+
try {
|
|
12230
|
+
entry.ws.send(JSON.stringify({
|
|
12231
|
+
type: "job.cancel",
|
|
12232
|
+
messageId: randomUUID(),
|
|
12233
|
+
runId,
|
|
12234
|
+
jobId,
|
|
12235
|
+
reason,
|
|
12236
|
+
...force && { force: true }
|
|
12237
|
+
}));
|
|
12238
|
+
agentsNotified++;
|
|
12239
|
+
} catch (err) {
|
|
12240
|
+
logger$71.warn("Failed to send job.cancel to agent; treating job as orphaned", {
|
|
12241
|
+
runId,
|
|
12242
|
+
jobId,
|
|
12243
|
+
agentId,
|
|
12244
|
+
error: err instanceof Error ? err.message : String(err)
|
|
12245
|
+
});
|
|
12246
|
+
}
|
|
11647
12247
|
}
|
|
11648
12248
|
const pendingResult = await db.updateTable("execution_jobs").set({
|
|
11649
12249
|
status: ExecutionJobStatus.enum.cancelled,
|
|
@@ -11697,6 +12297,15 @@ var init_agent_job_failed_error = __esmMin((() => {
|
|
|
11697
12297
|
//#endregion
|
|
11698
12298
|
//#region src/ws/agent-handler.ts
|
|
11699
12299
|
/**
|
|
12300
|
+
* Narrow the token's `agent_type` column (a free string at the DB layer) to
|
|
12301
|
+
* the host-roster lifecycle class. Anything other than the two known values
|
|
12302
|
+
* (including `undefined` under `agentAuth: 'none'`) maps to `null` so the
|
|
12303
|
+
* roster's reconcile hook treats it as the GC-able default class.
|
|
12304
|
+
*/
|
|
12305
|
+
function toLifecycleClass(agentType) {
|
|
12306
|
+
return agentType === "static" || agentType === "ephemeral" ? agentType : null;
|
|
12307
|
+
}
|
|
12308
|
+
/**
|
|
11700
12309
|
* Run the three token-bound authorization gates against a wire-supplied
|
|
11701
12310
|
* `agent.register` payload. The gates are identical at first register
|
|
11702
12311
|
* (Phase 2 / `pendingRegistration`) and on every subsequent re-register
|
|
@@ -12080,7 +12689,8 @@ function createAgentWsHandler(deps) {
|
|
|
12080
12689
|
runningAsUid: parsed.data.runningAsUid,
|
|
12081
12690
|
tokenId: regEntry.tokenId ?? null,
|
|
12082
12691
|
mandatoryLabels: scalerInfo?.mandatoryLabels,
|
|
12083
|
-
scalerManaged: scalerInfo !== null
|
|
12692
|
+
scalerManaged: scalerInfo !== null,
|
|
12693
|
+
tokenAgentType: toLifecycleClass(regEntry.tokenAgentType)
|
|
12084
12694
|
});
|
|
12085
12695
|
wsToAgentId.set(ws, agentId);
|
|
12086
12696
|
if (regEntry.tokenId !== void 0 && regEntry.tokenExpiresAt !== void 0 && regEntry.tokenExpiresAt !== null) registry.scheduleExpiryKick(regEntry.tokenId, regEntry.tokenExpiresAt);
|
|
@@ -12192,7 +12802,8 @@ function createAgentWsHandler(deps) {
|
|
|
12192
12802
|
runningAsUser: msg.runningAsUser,
|
|
12193
12803
|
runningAsUid: msg.runningAsUid,
|
|
12194
12804
|
mandatoryLabels: existingEntry ? [...existingEntry.mandatoryLabels] : void 0,
|
|
12195
|
-
scalerManaged: existingEntry?.scalerManaged ?? false
|
|
12805
|
+
scalerManaged: existingEntry?.scalerManaged ?? false,
|
|
12806
|
+
tokenAgentType: toLifecycleClass(reregisterAuthState?.tokenAgentType) ?? existingEntry?.tokenAgentType ?? null
|
|
12196
12807
|
});
|
|
12197
12808
|
wsToAgentId.set(ws, msg.agentId);
|
|
12198
12809
|
setAgentsActive(registry.getActiveCount());
|
|
@@ -13614,7 +14225,8 @@ var init_rbac = __esmMin((() => {
|
|
|
13614
14225
|
* Shared error handler for admin route files.
|
|
13615
14226
|
*
|
|
13616
14227
|
* Handles common error types: RBAC permission denied, Zod validation,
|
|
13617
|
-
* PostgreSQL unique constraint violations,
|
|
14228
|
+
* PostgreSQL unique constraint violations, PostgreSQL invalid-text-representation
|
|
14229
|
+
* (malformed typed input → 400), and generic errors.
|
|
13618
14230
|
*/
|
|
13619
14231
|
function handleAdminError(c, err, logger) {
|
|
13620
14232
|
if (err instanceof PermissionDeniedError) return c.json({ error: err.message }, 403);
|
|
@@ -13623,6 +14235,7 @@ function handleAdminError(c, err, logger) {
|
|
|
13623
14235
|
details: err.issues
|
|
13624
14236
|
}, 400);
|
|
13625
14237
|
if (err instanceof Error && "code" in err && err.code === "23505") return c.json({ error: "Conflict: resource already exists" }, 409);
|
|
14238
|
+
if (err instanceof Error && "code" in err && err.code === "22P02") return c.json({ error: "Invalid request: malformed value for a typed field" }, 400);
|
|
13626
14239
|
logger.error("Admin API error", {
|
|
13627
14240
|
error: toErrorMessage(err),
|
|
13628
14241
|
stack: err instanceof Error ? err.stack : void 0
|
|
@@ -13865,10 +14478,10 @@ var init_admin_sources = __esmMin((() => {
|
|
|
13865
14478
|
//#endregion
|
|
13866
14479
|
//#region src/db/migrations/001_initial.ts
|
|
13867
14480
|
var _001_initial_exports = /* @__PURE__ */ __exportAll({
|
|
13868
|
-
down: () => down$
|
|
13869
|
-
up: () => up$
|
|
14481
|
+
down: () => down$41,
|
|
14482
|
+
up: () => up$41
|
|
13870
14483
|
});
|
|
13871
|
-
async function up$
|
|
14484
|
+
async function up$41(db) {
|
|
13872
14485
|
for (const stmt of DDL_STATEMENTS) await sql.raw(stmt).execute(db);
|
|
13873
14486
|
await sql`
|
|
13874
14487
|
INSERT INTO cluster_meta (key, value)
|
|
@@ -13890,7 +14503,7 @@ async function up$37(db) {
|
|
|
13890
14503
|
* Rollback drops everything created above. Uses CASCADE on table drops to cut
|
|
13891
14504
|
* through the FK graph without relying on exact topological order.
|
|
13892
14505
|
*/
|
|
13893
|
-
async function down$
|
|
14506
|
+
async function down$41(db) {
|
|
13894
14507
|
for (const [trig, tbl] of [["source_secrets_change_trigger", "scoped_secrets"], ["sources_change_trigger", "sources"]]) await sql.raw(`DROP TRIGGER IF EXISTS ${trig} ON public.${tbl}`).execute(db);
|
|
13895
14508
|
for (const table of [
|
|
13896
14509
|
"workflow_registrations",
|
|
@@ -14610,8 +15223,8 @@ var init__001_initial = __esmMin((() => {
|
|
|
14610
15223
|
//#endregion
|
|
14611
15224
|
//#region src/db/migrations/002_config_versions_key_version.ts
|
|
14612
15225
|
var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
14613
|
-
down: () => down$
|
|
14614
|
-
up: () => up$
|
|
15226
|
+
down: () => down$40,
|
|
15227
|
+
up: () => up$40
|
|
14615
15228
|
});
|
|
14616
15229
|
/**
|
|
14617
15230
|
* Add key_version column to config_versions so that sensitive-field encryption
|
|
@@ -14625,13 +15238,13 @@ var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
|
14625
15238
|
* No index is needed: rotation does a full-table scan; reads are by
|
|
14626
15239
|
* `version` primary key and never filter on `key_version`.
|
|
14627
15240
|
*/
|
|
14628
|
-
async function up$
|
|
15241
|
+
async function up$40(db) {
|
|
14629
15242
|
await sql`
|
|
14630
15243
|
ALTER TABLE public.config_versions
|
|
14631
15244
|
ADD COLUMN key_version integer NOT NULL DEFAULT 1
|
|
14632
15245
|
`.execute(db);
|
|
14633
15246
|
}
|
|
14634
|
-
async function down$
|
|
15247
|
+
async function down$40(db) {
|
|
14635
15248
|
await sql`
|
|
14636
15249
|
ALTER TABLE public.config_versions
|
|
14637
15250
|
DROP COLUMN key_version
|
|
@@ -14641,8 +15254,8 @@ var init__002_config_versions_key_version = __esmMin((() => {}));
|
|
|
14641
15254
|
//#endregion
|
|
14642
15255
|
//#region src/db/migrations/003_access_log.ts
|
|
14643
15256
|
var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
14644
|
-
down: () => down$
|
|
14645
|
-
up: () => up$
|
|
15257
|
+
down: () => down$39,
|
|
15258
|
+
up: () => up$39
|
|
14646
15259
|
});
|
|
14647
15260
|
/**
|
|
14648
15261
|
* Access log: one row per read or orchestrator-admin mutation attributable
|
|
@@ -14666,7 +15279,7 @@ var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
|
14666
15279
|
* Retention is TTL-based via expires_at; packages/orchestrator/src/queue/
|
|
14667
15280
|
* cleanup.ts picks up the prune pass.
|
|
14668
15281
|
*/
|
|
14669
|
-
async function up$
|
|
15282
|
+
async function up$39(db) {
|
|
14670
15283
|
await sql`
|
|
14671
15284
|
CREATE TABLE public.access_log (
|
|
14672
15285
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
@@ -14704,21 +15317,21 @@ async function up$35(db) {
|
|
|
14704
15317
|
ON public.access_log (actor_type, actor_id, created_at DESC)
|
|
14705
15318
|
`.execute(db);
|
|
14706
15319
|
}
|
|
14707
|
-
async function down$
|
|
15320
|
+
async function down$39(db) {
|
|
14708
15321
|
await sql`DROP TABLE IF EXISTS public.access_log`.execute(db);
|
|
14709
15322
|
}
|
|
14710
15323
|
var init__003_access_log = __esmMin((() => {}));
|
|
14711
15324
|
//#endregion
|
|
14712
15325
|
//#region src/db/migrations/004_rename_bundle_to_source.ts
|
|
14713
15326
|
var _004_rename_bundle_to_source_exports = /* @__PURE__ */ __exportAll({
|
|
14714
|
-
down: () => down$
|
|
14715
|
-
up: () => up$
|
|
15327
|
+
down: () => down$38,
|
|
15328
|
+
up: () => up$38
|
|
14716
15329
|
});
|
|
14717
|
-
async function up$
|
|
15330
|
+
async function up$38(db) {
|
|
14718
15331
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_url", "source_tar_url").execute();
|
|
14719
15332
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_hash", "source_tar_hash").execute();
|
|
14720
15333
|
}
|
|
14721
|
-
async function down$
|
|
15334
|
+
async function down$38(db) {
|
|
14722
15335
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_url", "bundle_url").execute();
|
|
14723
15336
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_hash", "bundle_hash").execute();
|
|
14724
15337
|
}
|
|
@@ -14726,8 +15339,8 @@ var init__004_rename_bundle_to_source = __esmMin((() => {}));
|
|
|
14726
15339
|
//#endregion
|
|
14727
15340
|
//#region src/db/migrations/005_cold_store_chunk_counter.ts
|
|
14728
15341
|
var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
14729
|
-
down: () => down$
|
|
14730
|
-
up: () => up$
|
|
15342
|
+
down: () => down$37,
|
|
15343
|
+
up: () => up$37
|
|
14731
15344
|
});
|
|
14732
15345
|
/**
|
|
14733
15346
|
* Cold-store chunk counter table.
|
|
@@ -14744,7 +15357,7 @@ var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
|
14744
15357
|
*
|
|
14745
15358
|
* sections 5 and 8.
|
|
14746
15359
|
*/
|
|
14747
|
-
async function up$
|
|
15360
|
+
async function up$37(db) {
|
|
14748
15361
|
await sql`
|
|
14749
15362
|
CREATE TABLE public.cold_store_chunk_counts (
|
|
14750
15363
|
db TEXT NOT NULL,
|
|
@@ -14762,15 +15375,15 @@ async function up$33(db) {
|
|
|
14762
15375
|
ON public.cold_store_chunk_counts (db, table_name)
|
|
14763
15376
|
`.execute(db);
|
|
14764
15377
|
}
|
|
14765
|
-
async function down$
|
|
15378
|
+
async function down$37(db) {
|
|
14766
15379
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunk_counts`.execute(db);
|
|
14767
15380
|
}
|
|
14768
15381
|
var init__005_cold_store_chunk_counter = __esmMin((() => {}));
|
|
14769
15382
|
//#endregion
|
|
14770
15383
|
//#region src/db/migrations/006_runs_jobs_steps_archived_at.ts
|
|
14771
15384
|
var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14772
|
-
down: () => down$
|
|
14773
|
-
up: () => up$
|
|
15385
|
+
down: () => down$36,
|
|
15386
|
+
up: () => up$36
|
|
14774
15387
|
});
|
|
14775
15388
|
/**
|
|
14776
15389
|
* `execution_runs` / `execution_jobs` / `execution_steps` cold-store
|
|
@@ -14806,7 +15419,7 @@ var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
14806
15419
|
* - `idx_execution_jobs_routing_key_created (routing_key, created_at)`
|
|
14807
15420
|
* - `idx_execution_steps_routing_key_created (routing_key, created_at)`
|
|
14808
15421
|
*/
|
|
14809
|
-
async function up$
|
|
15422
|
+
async function up$36(db) {
|
|
14810
15423
|
await sql`
|
|
14811
15424
|
ALTER TABLE public.execution_runs
|
|
14812
15425
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -14851,7 +15464,7 @@ async function up$32(db) {
|
|
|
14851
15464
|
ON public.execution_steps (routing_key, created_at)
|
|
14852
15465
|
`.execute(db);
|
|
14853
15466
|
}
|
|
14854
|
-
async function down$
|
|
15467
|
+
async function down$36(db) {
|
|
14855
15468
|
await sql`DROP INDEX IF EXISTS public.idx_execution_steps_routing_key_created`.execute(db);
|
|
14856
15469
|
await sql`
|
|
14857
15470
|
ALTER TABLE public.execution_steps
|
|
@@ -14877,8 +15490,8 @@ var init__006_runs_jobs_steps_archived_at = __esmMin((() => {}));
|
|
|
14877
15490
|
//#endregion
|
|
14878
15491
|
//#region src/db/migrations/007_audit_logs_archived_at.ts
|
|
14879
15492
|
var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14880
|
-
down: () => down$
|
|
14881
|
-
up: () => up$
|
|
15493
|
+
down: () => down$35,
|
|
15494
|
+
up: () => up$35
|
|
14882
15495
|
});
|
|
14883
15496
|
/**
|
|
14884
15497
|
* `secret_audit_log` and `access_log` cold-store schema additions, plus
|
|
@@ -14918,7 +15531,7 @@ var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
14918
15531
|
* `down()` would not have meaningful retention bounds. Acceptable for
|
|
14919
15532
|
* staging.
|
|
14920
15533
|
*/
|
|
14921
|
-
async function up$
|
|
15534
|
+
async function up$35(db) {
|
|
14922
15535
|
await sql`
|
|
14923
15536
|
ALTER TABLE public.secret_audit_log
|
|
14924
15537
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -14939,7 +15552,7 @@ async function up$31(db) {
|
|
|
14939
15552
|
DROP COLUMN IF EXISTS expires_at
|
|
14940
15553
|
`.execute(db);
|
|
14941
15554
|
}
|
|
14942
|
-
async function down$
|
|
15555
|
+
async function down$35(db) {
|
|
14943
15556
|
await sql`
|
|
14944
15557
|
ALTER TABLE public.access_log
|
|
14945
15558
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '90 days')
|
|
@@ -14968,8 +15581,8 @@ var init__007_audit_logs_archived_at = __esmMin((() => {}));
|
|
|
14968
15581
|
//#endregion
|
|
14969
15582
|
//#region src/db/migrations/008_event_log_archived_at.ts
|
|
14970
15583
|
var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14971
|
-
down: () => down$
|
|
14972
|
-
up: () => up$
|
|
15584
|
+
down: () => down$34,
|
|
15585
|
+
up: () => up$34
|
|
14973
15586
|
});
|
|
14974
15587
|
/**
|
|
14975
15588
|
* `event_log` cold-store schema additions plus removal of the
|
|
@@ -15007,7 +15620,7 @@ var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
15007
15620
|
* — best effort; rows inserted between `up()` and a hypothetical
|
|
15008
15621
|
* `down()` would not have meaningful retention bounds.
|
|
15009
15622
|
*/
|
|
15010
|
-
async function up$
|
|
15623
|
+
async function up$34(db) {
|
|
15011
15624
|
await sql`
|
|
15012
15625
|
ALTER TABLE public.event_log
|
|
15013
15626
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -15023,7 +15636,7 @@ async function up$30(db) {
|
|
|
15023
15636
|
DROP COLUMN IF EXISTS expires_at
|
|
15024
15637
|
`.execute(db);
|
|
15025
15638
|
}
|
|
15026
|
-
async function down$
|
|
15639
|
+
async function down$34(db) {
|
|
15027
15640
|
await sql`
|
|
15028
15641
|
ALTER TABLE public.event_log
|
|
15029
15642
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '30 days')
|
|
@@ -15043,8 +15656,8 @@ var init__008_event_log_archived_at = __esmMin((() => {}));
|
|
|
15043
15656
|
//#endregion
|
|
15044
15657
|
//#region src/db/migrations/009_access_log_trigram.ts
|
|
15045
15658
|
var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
15046
|
-
down: () => down$
|
|
15047
|
-
up: () => up$
|
|
15659
|
+
down: () => down$33,
|
|
15660
|
+
up: () => up$33
|
|
15048
15661
|
});
|
|
15049
15662
|
/**
|
|
15050
15663
|
* Trigram (pg_trgm) index on access_log.error_message for the federated
|
|
@@ -15057,7 +15670,7 @@ var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
|
15057
15670
|
* EXISTS` are both safe to re-run. No CONCURRENTLY because Kysely runs
|
|
15058
15671
|
* migrations inside a transaction; the lock is brief on a sampled table.
|
|
15059
15672
|
*/
|
|
15060
|
-
async function up$
|
|
15673
|
+
async function up$33(db) {
|
|
15061
15674
|
await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
|
|
15062
15675
|
await sql`
|
|
15063
15676
|
CREATE INDEX IF NOT EXISTS access_log_error_message_trgm_idx
|
|
@@ -15066,15 +15679,15 @@ async function up$29(db) {
|
|
|
15066
15679
|
WHERE error_message IS NOT NULL
|
|
15067
15680
|
`.execute(db);
|
|
15068
15681
|
}
|
|
15069
|
-
async function down$
|
|
15682
|
+
async function down$33(db) {
|
|
15070
15683
|
await sql`DROP INDEX IF EXISTS public.access_log_error_message_trgm_idx`.execute(db);
|
|
15071
15684
|
}
|
|
15072
15685
|
var init__009_access_log_trigram = __esmMin((() => {}));
|
|
15073
15686
|
//#endregion
|
|
15074
15687
|
//#region src/db/migrations/010_cold_store_chunks.ts
|
|
15075
15688
|
var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
15076
|
-
down: () => down$
|
|
15077
|
-
up: () => up$
|
|
15689
|
+
down: () => down$32,
|
|
15690
|
+
up: () => up$32
|
|
15078
15691
|
});
|
|
15079
15692
|
/**
|
|
15080
15693
|
* Cold-store chunk index — Phase 2 (cold-store purge).
|
|
@@ -15106,7 +15719,7 @@ var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
|
15106
15719
|
* forever. Adapters that don't opt into per-bucket archival via
|
|
15107
15720
|
* `coldTtlDays` don't insert here either.
|
|
15108
15721
|
*/
|
|
15109
|
-
async function up$
|
|
15722
|
+
async function up$32(db) {
|
|
15110
15723
|
await sql`
|
|
15111
15724
|
CREATE TABLE public.cold_store_chunks (
|
|
15112
15725
|
db TEXT NOT NULL,
|
|
@@ -15133,15 +15746,15 @@ async function up$28(db) {
|
|
|
15133
15746
|
ON public.cold_store_chunks (db, table_name, tenant_id, archived_at DESC)
|
|
15134
15747
|
`.execute(db);
|
|
15135
15748
|
}
|
|
15136
|
-
async function down$
|
|
15749
|
+
async function down$32(db) {
|
|
15137
15750
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunks`.execute(db);
|
|
15138
15751
|
}
|
|
15139
15752
|
var init__010_cold_store_chunks = __esmMin((() => {}));
|
|
15140
15753
|
//#endregion
|
|
15141
15754
|
//#region src/db/migrations/011_drop_source_secrets_notify.ts
|
|
15142
15755
|
var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
15143
|
-
down: () => down$
|
|
15144
|
-
up: () => up$
|
|
15756
|
+
down: () => down$31,
|
|
15757
|
+
up: () => up$31
|
|
15145
15758
|
});
|
|
15146
15759
|
/**
|
|
15147
15760
|
* Drop the `source_secrets_change_trigger` and the
|
|
@@ -15160,11 +15773,11 @@ var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
15160
15773
|
* in `001_initial.ts`. They wake up no consumer until the `WebhookSecretManager`
|
|
15161
15774
|
* is restored, so this migration is safe to roll back.
|
|
15162
15775
|
*/
|
|
15163
|
-
async function up$
|
|
15776
|
+
async function up$31(db) {
|
|
15164
15777
|
await sql`DROP TRIGGER IF EXISTS source_secrets_change_trigger ON public.scoped_secrets`.execute(db);
|
|
15165
15778
|
await sql`DROP FUNCTION IF EXISTS public.notify_source_secrets_change() CASCADE`.execute(db);
|
|
15166
15779
|
}
|
|
15167
|
-
async function down$
|
|
15780
|
+
async function down$31(db) {
|
|
15168
15781
|
await sql`
|
|
15169
15782
|
CREATE OR REPLACE FUNCTION public.notify_source_secrets_change() RETURNS trigger
|
|
15170
15783
|
LANGUAGE plpgsql
|
|
@@ -15204,8 +15817,8 @@ var init__011_drop_source_secrets_notify = __esmMin((() => {}));
|
|
|
15204
15817
|
//#endregion
|
|
15205
15818
|
//#region src/db/migrations/012_peer_credentials_active_uniq.ts
|
|
15206
15819
|
var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
15207
|
-
down: () => down$
|
|
15208
|
-
up: () => up$
|
|
15820
|
+
down: () => down$30,
|
|
15821
|
+
up: () => up$30
|
|
15209
15822
|
});
|
|
15210
15823
|
/**
|
|
15211
15824
|
* Add a partial unique index on `peer_credentials (instance_id) WHERE
|
|
@@ -15231,7 +15844,7 @@ var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
|
15231
15844
|
* `down()` only drops the index; it does NOT undo the dedupe (there's no
|
|
15232
15845
|
* safe way to recreate revoked rows, and the dedupe is monotonic).
|
|
15233
15846
|
*/
|
|
15234
|
-
async function up$
|
|
15847
|
+
async function up$30(db) {
|
|
15235
15848
|
await sql`
|
|
15236
15849
|
UPDATE public.peer_credentials
|
|
15237
15850
|
SET revoked_at = NOW()
|
|
@@ -15249,15 +15862,15 @@ async function up$26(db) {
|
|
|
15249
15862
|
WHERE revoked_at IS NULL
|
|
15250
15863
|
`.execute(db);
|
|
15251
15864
|
}
|
|
15252
|
-
async function down$
|
|
15865
|
+
async function down$30(db) {
|
|
15253
15866
|
await sql`DROP INDEX IF EXISTS public.peer_credentials_active_uniq`.execute(db);
|
|
15254
15867
|
}
|
|
15255
15868
|
var init__012_peer_credentials_active_uniq = __esmMin((() => {}));
|
|
15256
15869
|
//#endregion
|
|
15257
15870
|
//#region src/db/migrations/013_execution_log_bytes.ts
|
|
15258
15871
|
var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
15259
|
-
down: () => down$
|
|
15260
|
-
up: () => up$
|
|
15872
|
+
down: () => down$29,
|
|
15873
|
+
up: () => up$29
|
|
15261
15874
|
});
|
|
15262
15875
|
/**
|
|
15263
15876
|
* Add `log_bytes BIGINT NOT NULL DEFAULT 0` columns to `execution_runs` and
|
|
@@ -15276,7 +15889,7 @@ var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
|
15276
15889
|
*
|
|
15277
15890
|
* Idempotent (`ADD COLUMN IF NOT EXISTS`).
|
|
15278
15891
|
*/
|
|
15279
|
-
async function up$
|
|
15892
|
+
async function up$29(db) {
|
|
15280
15893
|
await sql`
|
|
15281
15894
|
ALTER TABLE public.execution_runs
|
|
15282
15895
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
@@ -15286,7 +15899,7 @@ async function up$25(db) {
|
|
|
15286
15899
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
15287
15900
|
`.execute(db);
|
|
15288
15901
|
}
|
|
15289
|
-
async function down$
|
|
15902
|
+
async function down$29(db) {
|
|
15290
15903
|
await sql`ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
15291
15904
|
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
15292
15905
|
}
|
|
@@ -15294,8 +15907,8 @@ var init__013_execution_log_bytes = __esmMin((() => {}));
|
|
|
15294
15907
|
//#endregion
|
|
15295
15908
|
//#region src/db/migrations/014_kici_events_lease_retry.ts
|
|
15296
15909
|
var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
15297
|
-
down: () => down$
|
|
15298
|
-
up: () => up$
|
|
15910
|
+
down: () => down$28,
|
|
15911
|
+
up: () => up$28
|
|
15299
15912
|
});
|
|
15300
15913
|
/**
|
|
15301
15914
|
* Add lease + retry + DLQ columns to `kici_events` so the EventRouter can
|
|
@@ -15329,7 +15942,7 @@ var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
|
15329
15942
|
*
|
|
15330
15943
|
* Idempotent (`ADD COLUMN IF NOT EXISTS` + `CREATE INDEX IF NOT EXISTS`).
|
|
15331
15944
|
*/
|
|
15332
|
-
async function up$
|
|
15945
|
+
async function up$28(db) {
|
|
15333
15946
|
await sql`
|
|
15334
15947
|
ALTER TABLE public.kici_events
|
|
15335
15948
|
ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ,
|
|
@@ -15360,7 +15973,7 @@ async function up$24(db) {
|
|
|
15360
15973
|
WHERE dlq_at IS NOT NULL
|
|
15361
15974
|
`.execute(db);
|
|
15362
15975
|
}
|
|
15363
|
-
async function down$
|
|
15976
|
+
async function down$28(db) {
|
|
15364
15977
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_dlq`.execute(db);
|
|
15365
15978
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_lease_expired`.execute(db);
|
|
15366
15979
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_retry_due`.execute(db);
|
|
@@ -15379,8 +15992,8 @@ var init__014_kici_events_lease_retry = __esmMin((() => {}));
|
|
|
15379
15992
|
//#endregion
|
|
15380
15993
|
//#region src/db/migrations/015_org_settings_customer_scoped.ts
|
|
15381
15994
|
var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
15382
|
-
down: () => down$
|
|
15383
|
-
up: () => up$
|
|
15995
|
+
down: () => down$27,
|
|
15996
|
+
up: () => up$27
|
|
15384
15997
|
});
|
|
15385
15998
|
/**
|
|
15386
15999
|
* Org-scope `org_settings` and qualify each glob entry by source.
|
|
@@ -15408,7 +16021,7 @@ var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
|
15408
16021
|
* Idempotent: a re-run on an already-migrated DB sees `customer_id` exists
|
|
15409
16022
|
* and the list columns are already jsonb, so it is a no-op.
|
|
15410
16023
|
*/
|
|
15411
|
-
async function up$
|
|
16024
|
+
async function up$27(db) {
|
|
15412
16025
|
if ((await sql`
|
|
15413
16026
|
SELECT EXISTS (
|
|
15414
16027
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15533,7 +16146,7 @@ async function up$23(db) {
|
|
|
15533
16146
|
await sql`DROP TABLE _org_settings_merged`.execute(db);
|
|
15534
16147
|
await sql`DROP TABLE _org_settings_stage`.execute(db);
|
|
15535
16148
|
}
|
|
15536
|
-
async function down$
|
|
16149
|
+
async function down$27(db) {
|
|
15537
16150
|
if (!(await sql`
|
|
15538
16151
|
SELECT EXISTS (
|
|
15539
16152
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15559,8 +16172,8 @@ var init__015_org_settings_customer_scoped = __esmMin((() => {}));
|
|
|
15559
16172
|
//#endregion
|
|
15560
16173
|
//#region src/db/migrations/016_org_settings_allow_http_npm.ts
|
|
15561
16174
|
var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
15562
|
-
down: () => down$
|
|
15563
|
-
up: () => up$
|
|
16175
|
+
down: () => down$26,
|
|
16176
|
+
up: () => up$26
|
|
15564
16177
|
});
|
|
15565
16178
|
/**
|
|
15566
16179
|
* Add `org_settings.allow_http_npm_registries boolean NOT NULL DEFAULT false`.
|
|
@@ -15573,7 +16186,7 @@ var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
|
15573
16186
|
*
|
|
15574
16187
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
15575
16188
|
*/
|
|
15576
|
-
async function up$
|
|
16189
|
+
async function up$26(db) {
|
|
15577
16190
|
if ((await sql`
|
|
15578
16191
|
SELECT EXISTS (
|
|
15579
16192
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15587,7 +16200,7 @@ async function up$22(db) {
|
|
|
15587
16200
|
ADD COLUMN allow_http_npm_registries boolean NOT NULL DEFAULT false
|
|
15588
16201
|
`.execute(db);
|
|
15589
16202
|
}
|
|
15590
|
-
async function down$
|
|
16203
|
+
async function down$26(db) {
|
|
15591
16204
|
await sql`
|
|
15592
16205
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_http_npm_registries
|
|
15593
16206
|
`.execute(db);
|
|
@@ -15596,13 +16209,13 @@ var init__016_org_settings_allow_http_npm = __esmMin((() => {}));
|
|
|
15596
16209
|
//#endregion
|
|
15597
16210
|
//#region src/db/migrations/017_org_id_widen.ts
|
|
15598
16211
|
var _017_org_id_widen_exports = /* @__PURE__ */ __exportAll({
|
|
15599
|
-
down: () => down$
|
|
15600
|
-
up: () => up$
|
|
16212
|
+
down: () => down$25,
|
|
16213
|
+
up: () => up$25
|
|
15601
16214
|
});
|
|
15602
|
-
async function up$
|
|
16215
|
+
async function up$25(db) {
|
|
15603
16216
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(16)`).execute(db);
|
|
15604
16217
|
}
|
|
15605
|
-
async function down$
|
|
16218
|
+
async function down$25(db) {
|
|
15606
16219
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(12)`).execute(db);
|
|
15607
16220
|
}
|
|
15608
16221
|
var ORG_ID_TABLES$1;
|
|
@@ -15621,14 +16234,14 @@ var init__017_org_id_widen = __esmMin((() => {
|
|
|
15621
16234
|
//#endregion
|
|
15622
16235
|
//#region src/db/migrations/018_org_id_prefix_backfill.ts
|
|
15623
16236
|
var _018_org_id_prefix_backfill_exports = /* @__PURE__ */ __exportAll({
|
|
15624
|
-
down: () => down$
|
|
15625
|
-
up: () => up$
|
|
16237
|
+
down: () => down$24,
|
|
16238
|
+
up: () => up$24
|
|
15626
16239
|
});
|
|
15627
|
-
async function up$
|
|
16240
|
+
async function up$24(db) {
|
|
15628
16241
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = 'org_' || org_id WHERE org_id <> 'kici-admin' AND org_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15629
16242
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = 'org_' || customer_id WHERE customer_id <> 'kici-admin' AND customer_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15630
16243
|
}
|
|
15631
|
-
async function down$
|
|
16244
|
+
async function down$24(db) {
|
|
15632
16245
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = substring(customer_id from 5) WHERE customer_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15633
16246
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = substring(org_id from 5) WHERE org_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15634
16247
|
}
|
|
@@ -15654,8 +16267,8 @@ var init__018_org_id_prefix_backfill = __esmMin((() => {
|
|
|
15654
16267
|
//#endregion
|
|
15655
16268
|
//#region src/db/migrations/019_generic_sources_change_notify.ts
|
|
15656
16269
|
var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
15657
|
-
down: () => down$
|
|
15658
|
-
up: () => up$
|
|
16270
|
+
down: () => down$23,
|
|
16271
|
+
up: () => up$23
|
|
15659
16272
|
});
|
|
15660
16273
|
/**
|
|
15661
16274
|
* Add a Postgres trigger on `generic_webhook_sources` that emits
|
|
@@ -15678,7 +16291,7 @@ var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
15678
16291
|
* (`notify_sources_change()` + `sources_change_trigger`, defined in
|
|
15679
16292
|
* `001_initial.ts`).
|
|
15680
16293
|
*/
|
|
15681
|
-
async function up$
|
|
16294
|
+
async function up$23(db) {
|
|
15682
16295
|
await sql`
|
|
15683
16296
|
CREATE FUNCTION public.notify_generic_sources_change() RETURNS trigger
|
|
15684
16297
|
LANGUAGE plpgsql
|
|
@@ -15699,7 +16312,7 @@ async function up$19(db) {
|
|
|
15699
16312
|
FOR EACH ROW EXECUTE FUNCTION public.notify_generic_sources_change()
|
|
15700
16313
|
`.execute(db);
|
|
15701
16314
|
}
|
|
15702
|
-
async function down$
|
|
16315
|
+
async function down$23(db) {
|
|
15703
16316
|
await sql`DROP TRIGGER IF EXISTS generic_sources_change_trigger ON public.generic_webhook_sources`.execute(db);
|
|
15704
16317
|
await sql`DROP FUNCTION IF EXISTS public.notify_generic_sources_change()`.execute(db);
|
|
15705
16318
|
}
|
|
@@ -15707,8 +16320,8 @@ var init__019_generic_sources_change_notify = __esmMin((() => {}));
|
|
|
15707
16320
|
//#endregion
|
|
15708
16321
|
//#region src/db/migrations/020_org_settings_dashboard_write_policy.ts
|
|
15709
16322
|
var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportAll({
|
|
15710
|
-
down: () => down$
|
|
15711
|
-
up: () => up$
|
|
16323
|
+
down: () => down$22,
|
|
16324
|
+
up: () => up$22
|
|
15712
16325
|
});
|
|
15713
16326
|
/**
|
|
15714
16327
|
* Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
|
|
@@ -15723,7 +16336,7 @@ var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportA
|
|
|
15723
16336
|
*
|
|
15724
16337
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
15725
16338
|
*/
|
|
15726
|
-
async function up$
|
|
16339
|
+
async function up$22(db) {
|
|
15727
16340
|
if ((await sql`
|
|
15728
16341
|
SELECT EXISTS (
|
|
15729
16342
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15737,7 +16350,7 @@ async function up$18(db) {
|
|
|
15737
16350
|
ADD COLUMN dashboard_write_policy jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
15738
16351
|
`.execute(db);
|
|
15739
16352
|
}
|
|
15740
|
-
async function down$
|
|
16353
|
+
async function down$22(db) {
|
|
15741
16354
|
await sql`
|
|
15742
16355
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dashboard_write_policy
|
|
15743
16356
|
`.execute(db);
|
|
@@ -15746,8 +16359,8 @@ var init__020_org_settings_dashboard_write_policy = __esmMin((() => {}));
|
|
|
15746
16359
|
//#endregion
|
|
15747
16360
|
//#region src/db/migrations/021_check_run_tracking.ts
|
|
15748
16361
|
var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
15749
|
-
down: () => down$
|
|
15750
|
-
up: () => up$
|
|
16362
|
+
down: () => down$21,
|
|
16363
|
+
up: () => up$21
|
|
15751
16364
|
});
|
|
15752
16365
|
/**
|
|
15753
16366
|
* Add `check_run_tracking` table for HA-safe check-run state persistence.
|
|
@@ -15771,7 +16384,7 @@ var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
|
15771
16384
|
*
|
|
15772
16385
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
15773
16386
|
*/
|
|
15774
|
-
async function up$
|
|
16387
|
+
async function up$21(db) {
|
|
15775
16388
|
if ((await sql`
|
|
15776
16389
|
SELECT EXISTS (
|
|
15777
16390
|
SELECT 1 FROM information_schema.tables
|
|
@@ -15802,15 +16415,15 @@ async function up$17(db) {
|
|
|
15802
16415
|
WHERE run_id IS NOT NULL
|
|
15803
16416
|
`.execute(db);
|
|
15804
16417
|
}
|
|
15805
|
-
async function down$
|
|
16418
|
+
async function down$21(db) {
|
|
15806
16419
|
await sql`DROP TABLE IF EXISTS public.check_run_tracking`.execute(db);
|
|
15807
16420
|
}
|
|
15808
16421
|
var init__021_check_run_tracking = __esmMin((() => {}));
|
|
15809
16422
|
//#endregion
|
|
15810
16423
|
//#region src/db/migrations/022_scaler_manager_state.ts
|
|
15811
16424
|
var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
15812
|
-
down: () => down$
|
|
15813
|
-
up: () => up$
|
|
16425
|
+
down: () => down$20,
|
|
16426
|
+
up: () => up$20
|
|
15814
16427
|
});
|
|
15815
16428
|
/**
|
|
15816
16429
|
* Add three tables persisting `ScalerManager` per-coord state:
|
|
@@ -15837,7 +16450,7 @@ var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
|
15837
16450
|
* Idempotent: a re-run on a DB that already has any of these tables
|
|
15838
16451
|
* leaves the existing one alone.
|
|
15839
16452
|
*/
|
|
15840
|
-
async function up$
|
|
16453
|
+
async function up$20(db) {
|
|
15841
16454
|
const tableExists = async (name) => {
|
|
15842
16455
|
return (await sql`
|
|
15843
16456
|
SELECT EXISTS (
|
|
@@ -15887,7 +16500,7 @@ async function up$16(db) {
|
|
|
15887
16500
|
`.execute(db);
|
|
15888
16501
|
}
|
|
15889
16502
|
}
|
|
15890
|
-
async function down$
|
|
16503
|
+
async function down$20(db) {
|
|
15891
16504
|
await sql`DROP TABLE IF EXISTS public.scaler_reservations`.execute(db);
|
|
15892
16505
|
await sql`DROP TABLE IF EXISTS public.scaler_agent_jobs`.execute(db);
|
|
15893
16506
|
await sql`DROP TABLE IF EXISTS public.scaler_spawning_agents`.execute(db);
|
|
@@ -15896,8 +16509,8 @@ var init__022_scaler_manager_state = __esmMin((() => {}));
|
|
|
15896
16509
|
//#endregion
|
|
15897
16510
|
//#region src/db/migrations/023_dispatch_queue_recovery_deadline.ts
|
|
15898
16511
|
var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
15899
|
-
down: () => down$
|
|
15900
|
-
up: () => up$
|
|
16512
|
+
down: () => down$19,
|
|
16513
|
+
up: () => up$19
|
|
15901
16514
|
});
|
|
15902
16515
|
/**
|
|
15903
16516
|
* Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
|
|
@@ -15921,7 +16534,7 @@ var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll(
|
|
|
15921
16534
|
* Idempotent: re-running on a DB that already has either column is a
|
|
15922
16535
|
* no-op.
|
|
15923
16536
|
*/
|
|
15924
|
-
async function up$
|
|
16537
|
+
async function up$19(db) {
|
|
15925
16538
|
const colExists = async (name) => {
|
|
15926
16539
|
return (await sql`
|
|
15927
16540
|
SELECT EXISTS (
|
|
@@ -15946,7 +16559,7 @@ async function up$15(db) {
|
|
|
15946
16559
|
WHERE recovery_deadline IS NOT NULL
|
|
15947
16560
|
`.execute(db);
|
|
15948
16561
|
}
|
|
15949
|
-
async function down$
|
|
16562
|
+
async function down$19(db) {
|
|
15950
16563
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_recovery_deadline`.execute(db);
|
|
15951
16564
|
await sql`
|
|
15952
16565
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS recovery_agent_id
|
|
@@ -15959,8 +16572,8 @@ var init__023_dispatch_queue_recovery_deadline = __esmMin((() => {}));
|
|
|
15959
16572
|
//#endregion
|
|
15960
16573
|
//#region src/db/migrations/024_dispatch_queue_provisioning_error.ts
|
|
15961
16574
|
var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll({
|
|
15962
|
-
down: () => down$
|
|
15963
|
-
up: () => up$
|
|
16575
|
+
down: () => down$18,
|
|
16576
|
+
up: () => up$18
|
|
15964
16577
|
});
|
|
15965
16578
|
/**
|
|
15966
16579
|
* Add `dispatch_queue.last_provisioning_error TEXT` recording the most
|
|
@@ -15976,7 +16589,7 @@ var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll
|
|
|
15976
16589
|
*
|
|
15977
16590
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
15978
16591
|
*/
|
|
15979
|
-
async function up$
|
|
16592
|
+
async function up$18(db) {
|
|
15980
16593
|
const colExists = async (name) => {
|
|
15981
16594
|
return (await sql`
|
|
15982
16595
|
SELECT EXISTS (
|
|
@@ -15992,7 +16605,7 @@ async function up$14(db) {
|
|
|
15992
16605
|
ADD COLUMN last_provisioning_error TEXT
|
|
15993
16606
|
`.execute(db);
|
|
15994
16607
|
}
|
|
15995
|
-
async function down$
|
|
16608
|
+
async function down$18(db) {
|
|
15996
16609
|
await sql`
|
|
15997
16610
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS last_provisioning_error
|
|
15998
16611
|
`.execute(db);
|
|
@@ -16001,8 +16614,8 @@ var init__024_dispatch_queue_provisioning_error = __esmMin((() => {}));
|
|
|
16001
16614
|
//#endregion
|
|
16002
16615
|
//#region src/db/migrations/025_init_failure.ts
|
|
16003
16616
|
var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
16004
|
-
down: () => down$
|
|
16005
|
-
up: () => up$
|
|
16617
|
+
down: () => down$17,
|
|
16618
|
+
up: () => up$17
|
|
16006
16619
|
});
|
|
16007
16620
|
/**
|
|
16008
16621
|
* Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
|
|
@@ -16016,7 +16629,7 @@ var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
|
16016
16629
|
* Idempotent: re-running on a DB that already has either column is a no-op
|
|
16017
16630
|
* for that column.
|
|
16018
16631
|
*/
|
|
16019
|
-
async function up$
|
|
16632
|
+
async function up$17(db) {
|
|
16020
16633
|
const colExists = async (table, name) => {
|
|
16021
16634
|
return (await sql`
|
|
16022
16635
|
SELECT EXISTS (
|
|
@@ -16036,7 +16649,7 @@ async function up$13(db) {
|
|
|
16036
16649
|
ADD COLUMN init_failure JSONB DEFAULT NULL
|
|
16037
16650
|
`.execute(db);
|
|
16038
16651
|
}
|
|
16039
|
-
async function down$
|
|
16652
|
+
async function down$17(db) {
|
|
16040
16653
|
await sql`
|
|
16041
16654
|
ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS init_failure
|
|
16042
16655
|
`.execute(db);
|
|
@@ -16048,8 +16661,8 @@ var init__025_init_failure = __esmMin((() => {}));
|
|
|
16048
16661
|
//#endregion
|
|
16049
16662
|
//#region src/db/migrations/026_event_log_lockfile_corrupt.ts
|
|
16050
16663
|
var _026_event_log_lockfile_corrupt_exports = /* @__PURE__ */ __exportAll({
|
|
16051
|
-
down: () => down$
|
|
16052
|
-
up: () => up$
|
|
16664
|
+
down: () => down$16,
|
|
16665
|
+
up: () => up$16
|
|
16053
16666
|
});
|
|
16054
16667
|
/**
|
|
16055
16668
|
* Extend the event_log.status CHECK constraint with 'lockfile_corrupt' so the
|
|
@@ -16058,7 +16671,7 @@ var _026_event_log_lockfile_corrupt_exports = /* @__PURE__ */ __exportAll({
|
|
|
16058
16671
|
*
|
|
16059
16672
|
* Idempotent: the DROP ... IF EXISTS / re-ADD pair re-runs cleanly.
|
|
16060
16673
|
*/
|
|
16061
|
-
async function up$
|
|
16674
|
+
async function up$16(db) {
|
|
16062
16675
|
await sql`ALTER TABLE event_log DROP CONSTRAINT IF EXISTS event_log_status_check`.execute(db);
|
|
16063
16676
|
await sql`
|
|
16064
16677
|
ALTER TABLE event_log ADD CONSTRAINT event_log_status_check
|
|
@@ -16068,7 +16681,7 @@ async function up$12(db) {
|
|
|
16068
16681
|
])))
|
|
16069
16682
|
`.execute(db);
|
|
16070
16683
|
}
|
|
16071
|
-
async function down$
|
|
16684
|
+
async function down$16(db) {
|
|
16072
16685
|
await sql`ALTER TABLE event_log DROP CONSTRAINT IF EXISTS event_log_status_check`.execute(db);
|
|
16073
16686
|
await sql`
|
|
16074
16687
|
ALTER TABLE event_log ADD CONSTRAINT event_log_status_check
|
|
@@ -16082,8 +16695,8 @@ var init__026_event_log_lockfile_corrupt = __esmMin((() => {}));
|
|
|
16082
16695
|
//#endregion
|
|
16083
16696
|
//#region src/db/migrations/027_workflow_timeout.ts
|
|
16084
16697
|
var _027_workflow_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
16085
|
-
down: () => down$
|
|
16086
|
-
up: () => up$
|
|
16698
|
+
down: () => down$15,
|
|
16699
|
+
up: () => up$15
|
|
16087
16700
|
});
|
|
16088
16701
|
/**
|
|
16089
16702
|
* Add `workflow_timeout_ms integer` to `execution_runs`.
|
|
@@ -16101,13 +16714,13 @@ var _027_workflow_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
|
16101
16714
|
*
|
|
16102
16715
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
16103
16716
|
*/
|
|
16104
|
-
async function up$
|
|
16717
|
+
async function up$15(db) {
|
|
16105
16718
|
await sql`
|
|
16106
16719
|
ALTER TABLE public.execution_runs
|
|
16107
16720
|
ADD COLUMN IF NOT EXISTS workflow_timeout_ms INTEGER DEFAULT NULL
|
|
16108
16721
|
`.execute(db);
|
|
16109
16722
|
}
|
|
16110
|
-
async function down$
|
|
16723
|
+
async function down$15(db) {
|
|
16111
16724
|
await sql`
|
|
16112
16725
|
ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS workflow_timeout_ms
|
|
16113
16726
|
`.execute(db);
|
|
@@ -16116,8 +16729,8 @@ var init__027_workflow_timeout = __esmMin((() => {}));
|
|
|
16116
16729
|
//#endregion
|
|
16117
16730
|
//#region src/db/migrations/028_org_settings_user_cache.ts
|
|
16118
16731
|
var _028_org_settings_user_cache_exports = /* @__PURE__ */ __exportAll({
|
|
16119
|
-
down: () => down$
|
|
16120
|
-
up: () => up$
|
|
16732
|
+
down: () => down$14,
|
|
16733
|
+
up: () => up$14
|
|
16121
16734
|
});
|
|
16122
16735
|
/**
|
|
16123
16736
|
* Add `org_settings.user_cache_quota_bytes bigint` and
|
|
@@ -16146,7 +16759,7 @@ async function columnExists(db, column) {
|
|
|
16146
16759
|
) AS exists
|
|
16147
16760
|
`.execute(db)).rows[0]?.exists ?? false;
|
|
16148
16761
|
}
|
|
16149
|
-
async function up$
|
|
16762
|
+
async function up$14(db) {
|
|
16150
16763
|
if (!await columnExists(db, "user_cache_quota_bytes")) await sql`
|
|
16151
16764
|
ALTER TABLE public.org_settings
|
|
16152
16765
|
ADD COLUMN user_cache_quota_bytes bigint
|
|
@@ -16156,7 +16769,7 @@ async function up$10(db) {
|
|
|
16156
16769
|
ADD COLUMN user_cache_ttl_ms bigint
|
|
16157
16770
|
`.execute(db);
|
|
16158
16771
|
}
|
|
16159
|
-
async function down$
|
|
16772
|
+
async function down$14(db) {
|
|
16160
16773
|
await sql`
|
|
16161
16774
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS user_cache_quota_bytes
|
|
16162
16775
|
`.execute(db);
|
|
@@ -16168,8 +16781,8 @@ var init__028_org_settings_user_cache = __esmMin((() => {}));
|
|
|
16168
16781
|
//#endregion
|
|
16169
16782
|
//#region src/db/migrations/029_dispatch_queue_attempts.ts
|
|
16170
16783
|
var _029_dispatch_queue_attempts_exports = /* @__PURE__ */ __exportAll({
|
|
16171
|
-
down: () => down$
|
|
16172
|
-
up: () => up$
|
|
16784
|
+
down: () => down$13,
|
|
16785
|
+
up: () => up$13
|
|
16173
16786
|
});
|
|
16174
16787
|
/**
|
|
16175
16788
|
* Add `dispatch_queue.dispatch_attempts INT NOT NULL DEFAULT 0`.
|
|
@@ -16183,7 +16796,7 @@ var _029_dispatch_queue_attempts_exports = /* @__PURE__ */ __exportAll({
|
|
|
16183
16796
|
*
|
|
16184
16797
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
16185
16798
|
*/
|
|
16186
|
-
async function up$
|
|
16799
|
+
async function up$13(db) {
|
|
16187
16800
|
if (!((await sql`
|
|
16188
16801
|
SELECT EXISTS (
|
|
16189
16802
|
SELECT 1 FROM information_schema.columns
|
|
@@ -16196,7 +16809,7 @@ async function up$9(db) {
|
|
|
16196
16809
|
ADD COLUMN dispatch_attempts INT NOT NULL DEFAULT 0
|
|
16197
16810
|
`.execute(db);
|
|
16198
16811
|
}
|
|
16199
|
-
async function down$
|
|
16812
|
+
async function down$13(db) {
|
|
16200
16813
|
await sql`
|
|
16201
16814
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS dispatch_attempts
|
|
16202
16815
|
`.execute(db);
|
|
@@ -16205,8 +16818,8 @@ var init__029_dispatch_queue_attempts = __esmMin((() => {}));
|
|
|
16205
16818
|
//#endregion
|
|
16206
16819
|
//#region src/db/migrations/030_held_runs_env_set_null.ts
|
|
16207
16820
|
var _030_held_runs_env_set_null_exports = /* @__PURE__ */ __exportAll({
|
|
16208
|
-
down: () => down$
|
|
16209
|
-
up: () => up$
|
|
16821
|
+
down: () => down$12,
|
|
16822
|
+
up: () => up$12
|
|
16210
16823
|
});
|
|
16211
16824
|
/**
|
|
16212
16825
|
* held_runs.environment_id becomes nullable with ON DELETE SET NULL so
|
|
@@ -16217,14 +16830,14 @@ var _030_held_runs_env_set_null_exports = /* @__PURE__ */ __exportAll({
|
|
|
16217
16830
|
* Idempotent: dropping the NOT NULL and the constraint are both no-ops on a
|
|
16218
16831
|
* re-run, and the constraint is re-created with the SET NULL action.
|
|
16219
16832
|
*/
|
|
16220
|
-
async function up$
|
|
16833
|
+
async function up$12(db) {
|
|
16221
16834
|
await sql`ALTER TABLE public.held_runs ALTER COLUMN environment_id DROP NOT NULL`.execute(db);
|
|
16222
16835
|
await sql`ALTER TABLE public.held_runs DROP CONSTRAINT IF EXISTS held_runs_environment_id_fkey`.execute(db);
|
|
16223
16836
|
await sql`ALTER TABLE public.held_runs
|
|
16224
16837
|
ADD CONSTRAINT held_runs_environment_id_fkey
|
|
16225
16838
|
FOREIGN KEY (environment_id) REFERENCES public.environments(id) ON DELETE SET NULL`.execute(db);
|
|
16226
16839
|
}
|
|
16227
|
-
async function down$
|
|
16840
|
+
async function down$12(db) {
|
|
16228
16841
|
await sql`ALTER TABLE public.held_runs DROP CONSTRAINT IF EXISTS held_runs_environment_id_fkey`.execute(db);
|
|
16229
16842
|
await sql`ALTER TABLE public.held_runs
|
|
16230
16843
|
ADD CONSTRAINT held_runs_environment_id_fkey
|
|
@@ -16235,8 +16848,8 @@ var init__030_held_runs_env_set_null = __esmMin((() => {}));
|
|
|
16235
16848
|
//#endregion
|
|
16236
16849
|
//#region src/db/migrations/031_dispatch_queue_ack_deadline.ts
|
|
16237
16850
|
var _031_dispatch_queue_ack_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
16238
|
-
down: () => down$
|
|
16239
|
-
up: () => up$
|
|
16851
|
+
down: () => down$11,
|
|
16852
|
+
up: () => up$11
|
|
16240
16853
|
});
|
|
16241
16854
|
/**
|
|
16242
16855
|
* Add `dispatch_queue.ack_deadline TIMESTAMPTZ` and
|
|
@@ -16253,7 +16866,7 @@ var _031_dispatch_queue_ack_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
|
16253
16866
|
*
|
|
16254
16867
|
* Idempotent: re-running on a DB that already has either column is a no-op.
|
|
16255
16868
|
*/
|
|
16256
|
-
async function up$
|
|
16869
|
+
async function up$11(db) {
|
|
16257
16870
|
const colExists = async (name) => {
|
|
16258
16871
|
return (await sql`
|
|
16259
16872
|
SELECT EXISTS (
|
|
@@ -16278,7 +16891,7 @@ async function up$7(db) {
|
|
|
16278
16891
|
WHERE ack_deadline IS NOT NULL
|
|
16279
16892
|
`.execute(db);
|
|
16280
16893
|
}
|
|
16281
|
-
async function down$
|
|
16894
|
+
async function down$11(db) {
|
|
16282
16895
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_ack_deadline`.execute(db);
|
|
16283
16896
|
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS ack_agent_id`.execute(db);
|
|
16284
16897
|
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS ack_deadline`.execute(db);
|
|
@@ -16287,8 +16900,8 @@ var init__031_dispatch_queue_ack_deadline = __esmMin((() => {}));
|
|
|
16287
16900
|
//#endregion
|
|
16288
16901
|
//#region src/db/migrations/032_org_settings_dispatch_ack_timeout.ts
|
|
16289
16902
|
var _032_org_settings_dispatch_ack_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
16290
|
-
down: () => down$
|
|
16291
|
-
up: () => up$
|
|
16903
|
+
down: () => down$10,
|
|
16904
|
+
up: () => up$10
|
|
16292
16905
|
});
|
|
16293
16906
|
/**
|
|
16294
16907
|
* Add `org_settings.dispatch_ack_timeout_ms BIGINT` (nullable).
|
|
@@ -16300,7 +16913,7 @@ var _032_org_settings_dispatch_ack_timeout_exports = /* @__PURE__ */ __exportAll
|
|
|
16300
16913
|
*
|
|
16301
16914
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
16302
16915
|
*/
|
|
16303
|
-
async function up$
|
|
16916
|
+
async function up$10(db) {
|
|
16304
16917
|
if ((await sql`
|
|
16305
16918
|
SELECT EXISTS (
|
|
16306
16919
|
SELECT 1 FROM information_schema.columns
|
|
@@ -16314,7 +16927,7 @@ async function up$6(db) {
|
|
|
16314
16927
|
ADD COLUMN dispatch_ack_timeout_ms BIGINT
|
|
16315
16928
|
`.execute(db);
|
|
16316
16929
|
}
|
|
16317
|
-
async function down$
|
|
16930
|
+
async function down$10(db) {
|
|
16318
16931
|
await sql`
|
|
16319
16932
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dispatch_ack_timeout_ms
|
|
16320
16933
|
`.execute(db);
|
|
@@ -16323,8 +16936,8 @@ var init__032_org_settings_dispatch_ack_timeout = __esmMin((() => {}));
|
|
|
16323
16936
|
//#endregion
|
|
16324
16937
|
//#region src/db/migrations/033_org_settings_approval.ts
|
|
16325
16938
|
var _033_org_settings_approval_exports = /* @__PURE__ */ __exportAll({
|
|
16326
|
-
down: () => down$
|
|
16327
|
-
up: () => up$
|
|
16939
|
+
down: () => down$9,
|
|
16940
|
+
up: () => up$9
|
|
16328
16941
|
});
|
|
16329
16942
|
/**
|
|
16330
16943
|
* Add the two approval-policy columns to `org_settings`:
|
|
@@ -16341,7 +16954,7 @@ var _033_org_settings_approval_exports = /* @__PURE__ */ __exportAll({
|
|
|
16341
16954
|
* and the orchestrator admin route. Idempotent: a re-run on a DB that already
|
|
16342
16955
|
* has the columns is a no-op (each column is guarded independently).
|
|
16343
16956
|
*/
|
|
16344
|
-
async function up$
|
|
16957
|
+
async function up$9(db) {
|
|
16345
16958
|
const colExists = async (column) => {
|
|
16346
16959
|
return (await sql`
|
|
16347
16960
|
SELECT EXISTS (
|
|
@@ -16361,7 +16974,7 @@ async function up$5(db) {
|
|
|
16361
16974
|
ADD COLUMN allow_self_approval BOOLEAN NOT NULL DEFAULT true
|
|
16362
16975
|
`.execute(db);
|
|
16363
16976
|
}
|
|
16364
|
-
async function down$
|
|
16977
|
+
async function down$9(db) {
|
|
16365
16978
|
await sql`
|
|
16366
16979
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS approval_expiry_seconds
|
|
16367
16980
|
`.execute(db);
|
|
@@ -16373,8 +16986,8 @@ var init__033_org_settings_approval = __esmMin((() => {}));
|
|
|
16373
16986
|
//#endregion
|
|
16374
16987
|
//#region src/db/migrations/034_held_runs_generalize.ts
|
|
16375
16988
|
var _034_held_runs_generalize_exports = /* @__PURE__ */ __exportAll({
|
|
16376
|
-
down: () => down$
|
|
16377
|
-
up: () => up$
|
|
16989
|
+
down: () => down$8,
|
|
16990
|
+
up: () => up$8
|
|
16378
16991
|
});
|
|
16379
16992
|
/**
|
|
16380
16993
|
* Generalize `held_runs` from an environment-only hold into the unified
|
|
@@ -16396,7 +17009,7 @@ var _034_held_runs_generalize_exports = /* @__PURE__ */ __exportAll({
|
|
|
16396
17009
|
* New `held_run_approvals` table: one row per approver decision, FK to
|
|
16397
17010
|
* `held_runs.id` (uuid) with ON DELETE CASCADE.
|
|
16398
17011
|
*/
|
|
16399
|
-
async function up$
|
|
17012
|
+
async function up$8(db) {
|
|
16400
17013
|
const colExists = async (column) => {
|
|
16401
17014
|
return (await sql`
|
|
16402
17015
|
SELECT EXISTS (
|
|
@@ -16429,7 +17042,7 @@ async function up$4(db) {
|
|
|
16429
17042
|
ON public.held_run_approvals USING btree (held_run_id)
|
|
16430
17043
|
`.execute(db);
|
|
16431
17044
|
}
|
|
16432
|
-
async function down$
|
|
17045
|
+
async function down$8(db) {
|
|
16433
17046
|
await sql`DROP TABLE IF EXISTS public.held_run_approvals`.execute(db);
|
|
16434
17047
|
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS approval_requirement`.execute(db);
|
|
16435
17048
|
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS trigger_source`.execute(db);
|
|
@@ -16440,8 +17053,8 @@ var init__034_held_runs_generalize = __esmMin((() => {}));
|
|
|
16440
17053
|
//#endregion
|
|
16441
17054
|
//#region src/db/migrations/035_pending_workflow_contexts.ts
|
|
16442
17055
|
var _035_pending_workflow_contexts_exports = /* @__PURE__ */ __exportAll({
|
|
16443
|
-
down: () => down$
|
|
16444
|
-
up: () => up$
|
|
17056
|
+
down: () => down$7,
|
|
17057
|
+
up: () => up$7
|
|
16445
17058
|
});
|
|
16446
17059
|
/**
|
|
16447
17060
|
* Pending workflow dispatch context — backs resume of a workflow whose install
|
|
@@ -16450,7 +17063,7 @@ var _035_pending_workflow_contexts_exports = /* @__PURE__ */ __exportAll({
|
|
|
16450
17063
|
* wait-timer expiry, concurrency slot free). The row is deleted once the resume
|
|
16451
17064
|
* dispatch has been kicked off.
|
|
16452
17065
|
*/
|
|
16453
|
-
async function up$
|
|
17066
|
+
async function up$7(db) {
|
|
16454
17067
|
await sql`
|
|
16455
17068
|
CREATE TABLE IF NOT EXISTS public.pending_workflow_contexts (
|
|
16456
17069
|
run_id text PRIMARY KEY,
|
|
@@ -16460,15 +17073,15 @@ async function up$3(db) {
|
|
|
16460
17073
|
)
|
|
16461
17074
|
`.execute(db);
|
|
16462
17075
|
}
|
|
16463
|
-
async function down$
|
|
17076
|
+
async function down$7(db) {
|
|
16464
17077
|
await sql`DROP TABLE IF EXISTS public.pending_workflow_contexts`.execute(db);
|
|
16465
17078
|
}
|
|
16466
17079
|
var init__035_pending_workflow_contexts = __esmMin((() => {}));
|
|
16467
17080
|
//#endregion
|
|
16468
17081
|
//#region src/db/migrations/036_attestations.ts
|
|
16469
17082
|
var _036_attestations_exports = /* @__PURE__ */ __exportAll({
|
|
16470
|
-
down: () => down$
|
|
16471
|
-
up: () => up$
|
|
17083
|
+
down: () => down$6,
|
|
17084
|
+
up: () => up$6
|
|
16472
17085
|
});
|
|
16473
17086
|
/**
|
|
16474
17087
|
* Add the `attestations` table for build-provenance bundles.
|
|
@@ -16481,7 +17094,7 @@ var _036_attestations_exports = /* @__PURE__ */ __exportAll({
|
|
|
16481
17094
|
*
|
|
16482
17095
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
16483
17096
|
*/
|
|
16484
|
-
async function up$
|
|
17097
|
+
async function up$6(db) {
|
|
16485
17098
|
if ((await sql`
|
|
16486
17099
|
SELECT EXISTS (
|
|
16487
17100
|
SELECT 1 FROM information_schema.tables
|
|
@@ -16507,15 +17120,15 @@ async function up$2(db) {
|
|
|
16507
17120
|
ON public.attestations (run_id, job_id)
|
|
16508
17121
|
`.execute(db);
|
|
16509
17122
|
}
|
|
16510
|
-
async function down$
|
|
17123
|
+
async function down$6(db) {
|
|
16511
17124
|
await sql`DROP TABLE IF EXISTS public.attestations`.execute(db);
|
|
16512
17125
|
}
|
|
16513
17126
|
var init__036_attestations = __esmMin((() => {}));
|
|
16514
17127
|
//#endregion
|
|
16515
17128
|
//#region src/db/migrations/037_generic_sources_provider_type_local.ts
|
|
16516
17129
|
var _037_generic_sources_provider_type_local_exports = /* @__PURE__ */ __exportAll({
|
|
16517
|
-
down: () => down$
|
|
16518
|
-
up: () => up$
|
|
17130
|
+
down: () => down$5,
|
|
17131
|
+
up: () => up$5
|
|
16519
17132
|
});
|
|
16520
17133
|
/**
|
|
16521
17134
|
* Replace the `generic_webhook_sources.provider_type` CHECK constraint so it
|
|
@@ -16532,7 +17145,7 @@ var _037_generic_sources_provider_type_local_exports = /* @__PURE__ */ __exportA
|
|
|
16532
17145
|
* Idempotent: the constraint is dropped IF EXISTS and recreated; the data
|
|
16533
17146
|
* backfill is a plain UPDATE that is a no-op once no `'internal'` rows remain.
|
|
16534
17147
|
*/
|
|
16535
|
-
async function up$
|
|
17148
|
+
async function up$5(db) {
|
|
16536
17149
|
await sql`
|
|
16537
17150
|
ALTER TABLE public.generic_webhook_sources
|
|
16538
17151
|
DROP CONSTRAINT IF EXISTS generic_webhook_sources_provider_type_check
|
|
@@ -16548,7 +17161,7 @@ async function up$1(db) {
|
|
|
16548
17161
|
CHECK (provider_type = ANY (ARRAY['generic'::text, 'local'::text]))
|
|
16549
17162
|
`.execute(db);
|
|
16550
17163
|
}
|
|
16551
|
-
async function down$
|
|
17164
|
+
async function down$5(db) {
|
|
16552
17165
|
await sql`
|
|
16553
17166
|
ALTER TABLE public.generic_webhook_sources
|
|
16554
17167
|
DROP CONSTRAINT IF EXISTS generic_webhook_sources_provider_type_check
|
|
@@ -16568,8 +17181,8 @@ var init__037_generic_sources_provider_type_local = __esmMin((() => {}));
|
|
|
16568
17181
|
//#endregion
|
|
16569
17182
|
//#region src/db/migrations/038_remote_sources.ts
|
|
16570
17183
|
var _038_remote_sources_exports = /* @__PURE__ */ __exportAll({
|
|
16571
|
-
down: () => down,
|
|
16572
|
-
up: () => up
|
|
17184
|
+
down: () => down$4,
|
|
17185
|
+
up: () => up$4
|
|
16573
17186
|
});
|
|
16574
17187
|
/**
|
|
16575
17188
|
* `remote_sources` anchors a Platform-relayed `kici run remote` to its real
|
|
@@ -16581,7 +17194,7 @@ var _038_remote_sources_exports = /* @__PURE__ */ __exportAll({
|
|
|
16581
17194
|
*
|
|
16582
17195
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
16583
17196
|
*/
|
|
16584
|
-
async function up(db) {
|
|
17197
|
+
async function up$4(db) {
|
|
16585
17198
|
if ((await sql`
|
|
16586
17199
|
SELECT EXISTS (
|
|
16587
17200
|
SELECT 1 FROM information_schema.tables
|
|
@@ -16600,11 +17213,205 @@ async function up(db) {
|
|
|
16600
17213
|
)
|
|
16601
17214
|
`.execute(db);
|
|
16602
17215
|
}
|
|
16603
|
-
async function down(db) {
|
|
17216
|
+
async function down$4(db) {
|
|
16604
17217
|
await sql`DROP TABLE IF EXISTS public.remote_sources`.execute(db);
|
|
16605
17218
|
}
|
|
16606
17219
|
var init__038_remote_sources = __esmMin((() => {}));
|
|
16607
17220
|
//#endregion
|
|
17221
|
+
//#region src/db/migrations/039_host_roster.ts
|
|
17222
|
+
var _039_host_roster_exports = /* @__PURE__ */ __exportAll({
|
|
17223
|
+
down: () => down$3,
|
|
17224
|
+
up: () => up$3
|
|
17225
|
+
});
|
|
17226
|
+
/**
|
|
17227
|
+
* `host_roster` is KiCI's declared inventory: one durable row per agent the
|
|
17228
|
+
* cluster has ever enrolled, reconciled from the in-memory AgentRegistry on
|
|
17229
|
+
* every register/unregister. `lifecycle_class` (snapshot of the auth token's
|
|
17230
|
+
* agent_type) drives reaping — `ephemeral` rows are GC'd past their TTL,
|
|
17231
|
+
* `static` rows persist and read as `unreachable` when their heartbeat goes
|
|
17232
|
+
* stale. `connected_instance_id` records which orchestrator holds the live WS
|
|
17233
|
+
* (cluster liveness + the host-fanout reroute target); NULL = disconnected.
|
|
17234
|
+
*
|
|
17235
|
+
* The roster lives in the shared cluster DB (one table, all instances). Status
|
|
17236
|
+
* is derived at read from the shared `last_seen` + `connected_instance_id`, so
|
|
17237
|
+
* every instance agrees regardless of which one holds the agent's live WS.
|
|
17238
|
+
*
|
|
17239
|
+
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
17240
|
+
*/
|
|
17241
|
+
async function up$3(db) {
|
|
17242
|
+
if ((await sql`
|
|
17243
|
+
SELECT EXISTS (
|
|
17244
|
+
SELECT 1 FROM information_schema.tables
|
|
17245
|
+
WHERE table_schema = 'public' AND table_name = 'host_roster'
|
|
17246
|
+
) AS exists
|
|
17247
|
+
`.execute(db)).rows[0]?.exists) return;
|
|
17248
|
+
await sql`
|
|
17249
|
+
CREATE TABLE public.host_roster (
|
|
17250
|
+
id uuid DEFAULT gen_random_uuid() NOT NULL,
|
|
17251
|
+
agent_id text NOT NULL,
|
|
17252
|
+
token_id uuid,
|
|
17253
|
+
lifecycle_class text NOT NULL,
|
|
17254
|
+
labels text NOT NULL DEFAULT '[]',
|
|
17255
|
+
hostname text,
|
|
17256
|
+
platform text,
|
|
17257
|
+
arch text,
|
|
17258
|
+
connected_instance_id text,
|
|
17259
|
+
last_seen timestamptz NOT NULL DEFAULT now(),
|
|
17260
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
17261
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
17262
|
+
CONSTRAINT host_roster_pkey PRIMARY KEY (id),
|
|
17263
|
+
CONSTRAINT host_roster_agent_id_key UNIQUE (agent_id),
|
|
17264
|
+
CONSTRAINT host_roster_lifecycle_class_check
|
|
17265
|
+
CHECK (lifecycle_class = ANY (ARRAY['static'::text, 'ephemeral'::text]))
|
|
17266
|
+
)
|
|
17267
|
+
`.execute(db);
|
|
17268
|
+
await sql`CREATE INDEX idx_host_roster_reap
|
|
17269
|
+
ON public.host_roster (lifecycle_class, last_seen)`.execute(db);
|
|
17270
|
+
}
|
|
17271
|
+
async function down$3(db) {
|
|
17272
|
+
await sql`DROP TABLE IF EXISTS public.host_roster`.execute(db);
|
|
17273
|
+
}
|
|
17274
|
+
var init__039_host_roster = __esmMin((() => {}));
|
|
17275
|
+
//#endregion
|
|
17276
|
+
//#region src/db/migrations/040_runsonall_pin.ts
|
|
17277
|
+
var _040_runsonall_pin_exports = /* @__PURE__ */ __exportAll({
|
|
17278
|
+
down: () => down$2,
|
|
17279
|
+
up: () => up$2
|
|
17280
|
+
});
|
|
17281
|
+
/**
|
|
17282
|
+
* Add the `runsOnAll` host fan-out columns:
|
|
17283
|
+
*
|
|
17284
|
+
* - `dispatch_queue.pinned_agent_id TEXT` — when set, the queued/dispatched job
|
|
17285
|
+
* targets exactly that agent (a host-fanout child). The dispatcher routes it
|
|
17286
|
+
* only to that agent; the queue drain never hands it to a different one.
|
|
17287
|
+
* - `execution_jobs.base_job_name` / `variant_kind` / `variant_label` — generic
|
|
17288
|
+
* fan-out columns (matrix + host uniform). `variant_kind` is `'matrix'` or
|
|
17289
|
+
* `'host'`; `variant_label` is the matrix suffix or the hostname. They make the
|
|
17290
|
+
* logical fan-out job first-class server-side so the dashboard groups on real
|
|
17291
|
+
* fields instead of string-parsing the job name. (`matrix_values` / `group_name`
|
|
17292
|
+
* already exist; the matrix path now also backfills `variant_kind='matrix'`.)
|
|
17293
|
+
*
|
|
17294
|
+
* Idempotent: re-running on a DB that already has any column is a no-op.
|
|
17295
|
+
*/
|
|
17296
|
+
async function colExists$2(db, table, name) {
|
|
17297
|
+
return (await sql`
|
|
17298
|
+
SELECT EXISTS (
|
|
17299
|
+
SELECT 1 FROM information_schema.columns
|
|
17300
|
+
WHERE table_schema = 'public'
|
|
17301
|
+
AND table_name = ${table}
|
|
17302
|
+
AND column_name = ${name}
|
|
17303
|
+
) AS exists
|
|
17304
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
17305
|
+
}
|
|
17306
|
+
async function up$2(db) {
|
|
17307
|
+
if (!await colExists$2(db, "dispatch_queue", "pinned_agent_id")) await sql`ALTER TABLE public.dispatch_queue ADD COLUMN pinned_agent_id TEXT`.execute(db);
|
|
17308
|
+
if (!await colExists$2(db, "execution_jobs", "base_job_name")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN base_job_name TEXT`.execute(db);
|
|
17309
|
+
if (!await colExists$2(db, "execution_jobs", "variant_kind")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN variant_kind TEXT`.execute(db);
|
|
17310
|
+
if (!await colExists$2(db, "execution_jobs", "variant_label")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN variant_label TEXT`.execute(db);
|
|
17311
|
+
await sql`
|
|
17312
|
+
CREATE INDEX IF NOT EXISTS idx_dispatch_queue_pinned_agent
|
|
17313
|
+
ON public.dispatch_queue (pinned_agent_id)
|
|
17314
|
+
WHERE pinned_agent_id IS NOT NULL
|
|
17315
|
+
`.execute(db);
|
|
17316
|
+
}
|
|
17317
|
+
async function down$2(db) {
|
|
17318
|
+
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_pinned_agent`.execute(db);
|
|
17319
|
+
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS pinned_agent_id`.execute(db);
|
|
17320
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS variant_label`.execute(db);
|
|
17321
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS variant_kind`.execute(db);
|
|
17322
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS base_job_name`.execute(db);
|
|
17323
|
+
}
|
|
17324
|
+
var init__040_runsonall_pin = __esmMin((() => {}));
|
|
17325
|
+
//#endregion
|
|
17326
|
+
//#region src/db/migrations/041_wave_gated.ts
|
|
17327
|
+
var _041_wave_gated_exports = /* @__PURE__ */ __exportAll({
|
|
17328
|
+
down: () => down$1,
|
|
17329
|
+
up: () => up$1
|
|
17330
|
+
});
|
|
17331
|
+
/**
|
|
17332
|
+
* Add the rolling fan-out wave-gate columns:
|
|
17333
|
+
*
|
|
17334
|
+
* - `execution_jobs.wave_gated boolean NOT NULL DEFAULT false` — when a fan-out
|
|
17335
|
+
* job declares `maxParallel`, children beyond the sliding window are persisted
|
|
17336
|
+
* `wave_gated=true` (held, not enqueued). The dispatch loop skips them; the
|
|
17337
|
+
* wave-scheduler clears the flag one-per-terminal as siblings complete (or, on
|
|
17338
|
+
* `failFast`, skips the held remainder).
|
|
17339
|
+
* - `execution_jobs.wave_max_parallel int` / `wave_fail_fast boolean` — the
|
|
17340
|
+
* base's wave policy, stamped on every fan-out child so the wave-scheduler can
|
|
17341
|
+
* read it at terminal time without re-fetching the lock file (the tracker has
|
|
17342
|
+
* no lock access). NULL for any job not part of a bounded wave.
|
|
17343
|
+
*
|
|
17344
|
+
* A composite index on (run_id, base_job_name, wave_gated) supports the
|
|
17345
|
+
* wave-scheduler's "next held sibling of this base" lookups.
|
|
17346
|
+
*
|
|
17347
|
+
* Idempotent: re-running on a DB that already has the columns is a no-op.
|
|
17348
|
+
*/
|
|
17349
|
+
async function colExists$1(db, table, name) {
|
|
17350
|
+
return (await sql`
|
|
17351
|
+
SELECT EXISTS (
|
|
17352
|
+
SELECT 1 FROM information_schema.columns
|
|
17353
|
+
WHERE table_schema = 'public'
|
|
17354
|
+
AND table_name = ${table}
|
|
17355
|
+
AND column_name = ${name}
|
|
17356
|
+
) AS exists
|
|
17357
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
17358
|
+
}
|
|
17359
|
+
async function up$1(db) {
|
|
17360
|
+
if (!await colExists$1(db, "execution_jobs", "wave_gated")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN wave_gated boolean NOT NULL DEFAULT false`.execute(db);
|
|
17361
|
+
if (!await colExists$1(db, "execution_jobs", "wave_max_parallel")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN wave_max_parallel integer`.execute(db);
|
|
17362
|
+
if (!await colExists$1(db, "execution_jobs", "wave_fail_fast")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN wave_fail_fast boolean`.execute(db);
|
|
17363
|
+
await sql`
|
|
17364
|
+
CREATE INDEX IF NOT EXISTS idx_execution_jobs_wave
|
|
17365
|
+
ON public.execution_jobs (run_id, base_job_name, wave_gated)
|
|
17366
|
+
`.execute(db);
|
|
17367
|
+
}
|
|
17368
|
+
async function down$1(db) {
|
|
17369
|
+
await sql`DROP INDEX IF EXISTS public.idx_execution_jobs_wave`.execute(db);
|
|
17370
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS wave_fail_fast`.execute(db);
|
|
17371
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS wave_max_parallel`.execute(db);
|
|
17372
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS wave_gated`.execute(db);
|
|
17373
|
+
}
|
|
17374
|
+
var init__041_wave_gated = __esmMin((() => {}));
|
|
17375
|
+
//#endregion
|
|
17376
|
+
//#region src/db/migrations/042_dispatch_queue_patterns.ts
|
|
17377
|
+
var _042_dispatch_queue_patterns_exports = /* @__PURE__ */ __exportAll({
|
|
17378
|
+
down: () => down,
|
|
17379
|
+
up: () => up
|
|
17380
|
+
});
|
|
17381
|
+
/**
|
|
17382
|
+
* Add pattern columns to dispatch_queue. Exact labels stay in runs_on_labels /
|
|
17383
|
+
* exclude_labels (the SQL @> prefilter); regex matchers go here and are applied
|
|
17384
|
+
* as a JS post-filter at drain time, since Postgres `~` regex semantics differ
|
|
17385
|
+
* from JavaScript `RegExp` and the engine's `matcherSatisfiedBy` is the single
|
|
17386
|
+
* matching authority.
|
|
17387
|
+
*
|
|
17388
|
+
* - `runs_on_patterns jsonb NOT NULL DEFAULT '[]'` — regex matchers the agent's
|
|
17389
|
+
* labels must satisfy.
|
|
17390
|
+
* - `exclude_patterns jsonb NOT NULL DEFAULT '[]'` — regex matchers that
|
|
17391
|
+
* disqualify an agent.
|
|
17392
|
+
*
|
|
17393
|
+
* Idempotent: re-running on a DB that already has the columns is a no-op.
|
|
17394
|
+
*/
|
|
17395
|
+
async function colExists(db, table, name) {
|
|
17396
|
+
return (await sql`
|
|
17397
|
+
SELECT EXISTS (
|
|
17398
|
+
SELECT 1 FROM information_schema.columns
|
|
17399
|
+
WHERE table_schema = 'public'
|
|
17400
|
+
AND table_name = ${table}
|
|
17401
|
+
AND column_name = ${name}
|
|
17402
|
+
) AS exists
|
|
17403
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
17404
|
+
}
|
|
17405
|
+
async function up(db) {
|
|
17406
|
+
if (!await colExists(db, "dispatch_queue", "runs_on_patterns")) await sql`ALTER TABLE public.dispatch_queue ADD COLUMN runs_on_patterns jsonb NOT NULL DEFAULT '[]'::jsonb`.execute(db);
|
|
17407
|
+
if (!await colExists(db, "dispatch_queue", "exclude_patterns")) await sql`ALTER TABLE public.dispatch_queue ADD COLUMN exclude_patterns jsonb NOT NULL DEFAULT '[]'::jsonb`.execute(db);
|
|
17408
|
+
}
|
|
17409
|
+
async function down(db) {
|
|
17410
|
+
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS exclude_patterns`.execute(db);
|
|
17411
|
+
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS runs_on_patterns`.execute(db);
|
|
17412
|
+
}
|
|
17413
|
+
var init__042_dispatch_queue_patterns = __esmMin((() => {}));
|
|
17414
|
+
//#endregion
|
|
16608
17415
|
//#region src/db/migration-provider.ts
|
|
16609
17416
|
function createMigrationProvider() {
|
|
16610
17417
|
return { async getMigrations() {
|
|
@@ -16646,7 +17453,11 @@ function createMigrationProvider() {
|
|
|
16646
17453
|
"035_pending_workflow_contexts": _035_pending_workflow_contexts_exports,
|
|
16647
17454
|
"036_attestations": _036_attestations_exports,
|
|
16648
17455
|
"037_generic_sources_provider_type_local": _037_generic_sources_provider_type_local_exports,
|
|
16649
|
-
"038_remote_sources": _038_remote_sources_exports
|
|
17456
|
+
"038_remote_sources": _038_remote_sources_exports,
|
|
17457
|
+
"039_host_roster": _039_host_roster_exports,
|
|
17458
|
+
"040_runsonall_pin": _040_runsonall_pin_exports,
|
|
17459
|
+
"041_wave_gated": _041_wave_gated_exports,
|
|
17460
|
+
"042_dispatch_queue_patterns": _042_dispatch_queue_patterns_exports
|
|
16650
17461
|
};
|
|
16651
17462
|
} };
|
|
16652
17463
|
}
|
|
@@ -16689,6 +17500,10 @@ var init_migration_provider = __esmMin((() => {
|
|
|
16689
17500
|
init__036_attestations();
|
|
16690
17501
|
init__037_generic_sources_provider_type_local();
|
|
16691
17502
|
init__038_remote_sources();
|
|
17503
|
+
init__039_host_roster();
|
|
17504
|
+
init__040_runsonall_pin();
|
|
17505
|
+
init__041_wave_gated();
|
|
17506
|
+
init__042_dispatch_queue_patterns();
|
|
16692
17507
|
}));
|
|
16693
17508
|
//#endregion
|
|
16694
17509
|
//#region src/db/migrator.ts
|
|
@@ -21525,6 +22340,21 @@ var init_admin_runs = __esmMin((() => {
|
|
|
21525
22340
|
}));
|
|
21526
22341
|
//#endregion
|
|
21527
22342
|
//#region src/cold-store/load-event-log-range.ts
|
|
22343
|
+
/**
|
|
22344
|
+
* Read-through helper for `event_log` (Orchestrator) — Phase E.
|
|
22345
|
+
*
|
|
22346
|
+
* Mirrors the Platform-side `load-event-log-range.ts`. Tenant column is
|
|
22347
|
+
* `routing_key` (NOT NULL on this side) so cold-store partitioning
|
|
22348
|
+
* doesn't need a synthetic-tenant fallback. Partition column is
|
|
22349
|
+
* `received_at`.
|
|
22350
|
+
*
|
|
22351
|
+
* The orchestrator's `event_log` row carries the `payload_key` reference
|
|
22352
|
+
* (an S3 object that holds the gzipped webhook body). Cold-store
|
|
22353
|
+
* archives the row metadata and preserves `payload_key` verbatim — the
|
|
22354
|
+
* payload blob itself stays in object storage indefinitely so the
|
|
22355
|
+
* dashboard's payload-detail view continues to resolve archived
|
|
22356
|
+
* deliveries identically to hot ones.
|
|
22357
|
+
*/
|
|
21528
22358
|
async function loadEventLogRange(args) {
|
|
21529
22359
|
const { db, coldStore, routingKey, orgId, event, action, status, deliveryId, fromTs, toTs, limit, offset, includeArchived } = args;
|
|
21530
22360
|
const warmCutoff = /* @__PURE__ */ new Date(Date.now() - EVENT_LOG_WARM_TTL_DAYS * 864e5);
|
|
@@ -21542,6 +22372,16 @@ async function loadEventLogRange(args) {
|
|
|
21542
22372
|
if (coldFromTs >= warmCutoff) return hotRows;
|
|
21543
22373
|
const remaining = limit - hotRows.length;
|
|
21544
22374
|
if (remaining <= 0) return hotRows;
|
|
22375
|
+
let hotCountQuery = db.selectFrom("event_log").select(sql`count(*)::text`.as("n")).where("routing_key", "=", routingKey);
|
|
22376
|
+
if (orgId) hotCountQuery = hotCountQuery.where("org_id", "=", orgId);
|
|
22377
|
+
if (event) hotCountQuery = hotCountQuery.where("event", "=", event);
|
|
22378
|
+
if (action) hotCountQuery = hotCountQuery.where("action", "=", action);
|
|
22379
|
+
if (status) hotCountQuery = hotCountQuery.where("status", "=", status);
|
|
22380
|
+
if (deliveryId) hotCountQuery = hotCountQuery.where("delivery_id", "like", `%${deliveryId}%`);
|
|
22381
|
+
if (fromTs) hotCountQuery = hotCountQuery.where("received_at", ">=", fromTs);
|
|
22382
|
+
if (toTs) hotCountQuery = hotCountQuery.where("received_at", "<", toTs);
|
|
22383
|
+
const hotCount = Number((await hotCountQuery.executeTakeFirst())?.n ?? "0");
|
|
22384
|
+
const coldOffset = Math.max(0, offset - hotCount);
|
|
21545
22385
|
const coldToTs = toTs && toTs < warmCutoff ? toTs : warmCutoff;
|
|
21546
22386
|
const coldRows = [];
|
|
21547
22387
|
try {
|
|
@@ -21558,7 +22398,6 @@ async function loadEventLogRange(args) {
|
|
|
21558
22398
|
if (status !== void 0 && row.status !== status) continue;
|
|
21559
22399
|
if (deliveryId !== void 0 && !row.delivery_id.includes(deliveryId)) continue;
|
|
21560
22400
|
coldRows.push(row);
|
|
21561
|
-
if (coldRows.length >= remaining) break;
|
|
21562
22401
|
}
|
|
21563
22402
|
} catch (err) {
|
|
21564
22403
|
if (hotRows.length === 0) {
|
|
@@ -21578,7 +22417,7 @@ async function loadEventLogRange(args) {
|
|
|
21578
22417
|
const at = a.received_at instanceof Date ? a.received_at.getTime() : new Date(a.received_at).getTime();
|
|
21579
22418
|
return (b.received_at instanceof Date ? b.received_at.getTime() : new Date(b.received_at).getTime()) - at;
|
|
21580
22419
|
});
|
|
21581
|
-
return [...hotRows, ...coldRows.slice(
|
|
22420
|
+
return [...hotRows, ...coldRows.slice(coldOffset, coldOffset + remaining)];
|
|
21582
22421
|
}
|
|
21583
22422
|
/**
|
|
21584
22423
|
* Detail lookup by `(orgId, deliveryId)`. The dashboard handler joins
|
|
@@ -22739,15 +23578,15 @@ var init_admin_config = __esmMin((() => {
|
|
|
22739
23578
|
function createHealthRoutes$1(deps = {}) {
|
|
22740
23579
|
return createHealthRoutes({
|
|
22741
23580
|
livenessInfo: () => ({
|
|
22742
|
-
version: "0.1.
|
|
22743
|
-
buildDate: "2026-06-
|
|
22744
|
-
buildCommit: "
|
|
22745
|
-
sdkVersion: "0.1.
|
|
22746
|
-
sdkBundleHash: "
|
|
22747
|
-
sharedVersion: "0.1.
|
|
22748
|
-
sharedBundleHash: "
|
|
22749
|
-
engineVersion: "0.1.
|
|
22750
|
-
engineBundleHash: "
|
|
23581
|
+
version: "0.1.19",
|
|
23582
|
+
buildDate: "2026-06-19T04:32:44.899Z",
|
|
23583
|
+
buildCommit: "1590f5e99",
|
|
23584
|
+
sdkVersion: "0.1.19",
|
|
23585
|
+
sdkBundleHash: "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f",
|
|
23586
|
+
sharedVersion: "0.1.19",
|
|
23587
|
+
sharedBundleHash: "5f2c220f24d166b0f13d0620a2e80d19689ac8b683a12154daef44e938fd46a7",
|
|
23588
|
+
engineVersion: "0.1.19",
|
|
23589
|
+
engineBundleHash: "79ce14640d1798eaaa7cb3aa6c4bc325da3bff1e9f4716936e19614eabb1d858"
|
|
22751
23590
|
}),
|
|
22752
23591
|
readinessCheck: deps.db ? async () => {
|
|
22753
23592
|
const checks = {};
|
|
@@ -22781,7 +23620,7 @@ function createCapabilitiesRoutes() {
|
|
|
22781
23620
|
const app = new Hono();
|
|
22782
23621
|
app.get("/api/v1/capabilities", (c) => {
|
|
22783
23622
|
const manifest = {
|
|
22784
|
-
orchestratorVersion: "0.1.
|
|
23623
|
+
orchestratorVersion: "0.1.19",
|
|
22785
23624
|
protocolVersion: PROTOCOL_VERSION,
|
|
22786
23625
|
minProtocolVersion: MIN_PROTOCOL_VERSION
|
|
22787
23626
|
};
|
|
@@ -24987,8 +25826,9 @@ function chooseTargetPlatform(workflow, agentRegistry) {
|
|
|
24987
25826
|
targetPlatform,
|
|
24988
25827
|
targetArch
|
|
24989
25828
|
};
|
|
24990
|
-
const
|
|
24991
|
-
const
|
|
25829
|
+
const firstJob = workflow.jobs.filter(isLockStaticJob)[0];
|
|
25830
|
+
const firstExact = partitionMatchers(firstJob?.runsOn ?? []).exact;
|
|
25831
|
+
const representativeLabels = workflow.jobs.length > 0 ? firstExact.length > 0 ? firstExact : ["default"] : ["default"];
|
|
24992
25832
|
const candidates = agentRegistry.findAvailable(representativeLabels);
|
|
24993
25833
|
if (candidates.length > 0) {
|
|
24994
25834
|
targetPlatform = candidates[0].platform;
|
|
@@ -25279,12 +26119,112 @@ async function readPostBuildCacheUrls(args) {
|
|
|
25279
26119
|
* other jobs still proceed. Dynamic-matrix jobs pass through with a
|
|
25280
26120
|
* `pendingDynamicMatrix` marker for the eval flow.
|
|
25281
26121
|
*/
|
|
25282
|
-
|
|
26122
|
+
/**
|
|
26123
|
+
* Resolve a `runsOnAll` lock job against the declared host roster and partition
|
|
26124
|
+
* the matched hosts into the target set per the `onUnreachable` policy (R2):
|
|
26125
|
+
* `ready` hosts always run; unreachable durable (`static`) hosts hold / fail /
|
|
26126
|
+
* skip; stale ephemeral hosts are always skipped. Throws {@link FanoutError}
|
|
26127
|
+
* when the run can't proceed (fail policy with an absent host, or zero targets).
|
|
26128
|
+
*/
|
|
26129
|
+
async function resolveHostFanoutTargets(lockJob, deps) {
|
|
26130
|
+
if (!deps.hostRosterStore) throw new FanoutError(lockJob.name, `runsOnAll for job '${lockJob.name}': roster unavailable`);
|
|
26131
|
+
const predicate = lockJob.runsOnAll;
|
|
26132
|
+
const onUnreachable = lockJob.onUnreachable ?? "hold";
|
|
26133
|
+
const matched = await deps.hostRosterStore.findMatching(predicate.include, predicate.exclude, deps.rosterGraceMs ?? 3e5);
|
|
26134
|
+
const targets = [];
|
|
26135
|
+
const unreachableDurable = [];
|
|
26136
|
+
for (const h of matched) if (h.status === "ready") targets.push(h);
|
|
26137
|
+
else if (h.lifecycleClass === "ephemeral") continue;
|
|
26138
|
+
else unreachableDurable.push(h);
|
|
26139
|
+
if (unreachableDurable.length > 0) {
|
|
26140
|
+
if (onUnreachable === "fail") throw new FanoutError(lockJob.name, `runsOnAll '${lockJob.name}': ${unreachableDurable.length} expected host(s) unreachable`);
|
|
26141
|
+
if (onUnreachable === "hold") targets.push(...unreachableDurable);
|
|
26142
|
+
}
|
|
26143
|
+
if (targets.length === 0) throw new FanoutError(lockJob.name, `runsOnAll '${lockJob.name}' matched zero usable hosts`);
|
|
26144
|
+
return targets.map((h) => ({
|
|
26145
|
+
agentId: h.agentId,
|
|
26146
|
+
host: h.host,
|
|
26147
|
+
labels: h.labels,
|
|
26148
|
+
platform: h.platform ?? void 0,
|
|
26149
|
+
arch: h.arch ?? void 0,
|
|
26150
|
+
connectedInstanceId: h.connectedInstanceId
|
|
26151
|
+
}));
|
|
26152
|
+
}
|
|
26153
|
+
/**
|
|
26154
|
+
* Partition a lock job's runsOn / excludeLabels matchers into exact labels (SQL
|
|
26155
|
+
* `@>` prefilter + registry index) and regex patterns (JS post-filter). A
|
|
26156
|
+
* `runsOnAll` host-fanout job has no `runsOn`; its pinned children carry no
|
|
26157
|
+
* routing (the pin targets the resolved agent directly).
|
|
26158
|
+
*/
|
|
26159
|
+
function runsOnSelectorsForLockJob(lockJob) {
|
|
26160
|
+
const include = partitionMatchers(lockJob.runsOn ?? []);
|
|
26161
|
+
const exclude = partitionMatchers(lockJob.excludeLabels ?? []);
|
|
26162
|
+
return {
|
|
26163
|
+
runsOnLabels: include.exact,
|
|
26164
|
+
runsOnPatterns: include.regex,
|
|
26165
|
+
excludeLabels: exclude.exact,
|
|
26166
|
+
excludePatterns: exclude.regex
|
|
26167
|
+
};
|
|
26168
|
+
}
|
|
26169
|
+
/**
|
|
26170
|
+
* The generic fan-out tracking fields persisted on `execution_jobs` for a
|
|
26171
|
+
* materialized child: `baseJobName` + `variantKind` + `variantLabel`. Serves
|
|
26172
|
+
* matrix (label = combination suffix) and host (label = hostname) uniformly so
|
|
26173
|
+
* the dashboard groups on real columns instead of string-parsing the name.
|
|
26174
|
+
*/
|
|
26175
|
+
function variantTrackingFields(mat) {
|
|
26176
|
+
if (!mat.variantKind) return {};
|
|
26177
|
+
const variantLabel = mat.variantKind === VariantKind.host ? mat.host : mat.variantValues ? mat.expandedName.slice(mat.baseName.length + 2, -1) : void 0;
|
|
26178
|
+
return {
|
|
26179
|
+
baseJobName: mat.baseName,
|
|
26180
|
+
variantKind: mat.variantKind,
|
|
26181
|
+
...variantLabel && { variantLabel }
|
|
26182
|
+
};
|
|
26183
|
+
}
|
|
26184
|
+
/**
|
|
26185
|
+
* Compute the rolling-wave plan for a materialized job set.
|
|
26186
|
+
*
|
|
26187
|
+
* For each base job declaring `maxParallel` whose fan-out produced more than one
|
|
26188
|
+
* child, children are ordered deterministically by `variant_label` (the matrix
|
|
26189
|
+
* suffix / hostname, via `expandedName`) and every child at index `>=
|
|
26190
|
+
* maxParallel` is held (`wave_gated=true`). The first `maxParallel` dispatch
|
|
26191
|
+
* immediately; held children release one-per-terminal via the wave-scheduler.
|
|
26192
|
+
* Every child of a bounded-wave base — held or not — gets a `policy` entry so
|
|
26193
|
+
* the wave-scheduler can read the width/failFast at terminal time. A non-fan-out
|
|
26194
|
+
* job (single child) or one without `maxParallel` contributes nothing.
|
|
26195
|
+
*/
|
|
26196
|
+
function computeWavePlan(materializedJobs) {
|
|
26197
|
+
const byBase = /* @__PURE__ */ new Map();
|
|
26198
|
+
for (const mat of materializedJobs) {
|
|
26199
|
+
const list = byBase.get(mat.baseName);
|
|
26200
|
+
if (list) list.push(mat);
|
|
26201
|
+
else byBase.set(mat.baseName, [mat]);
|
|
26202
|
+
}
|
|
26203
|
+
const held = /* @__PURE__ */ new Set();
|
|
26204
|
+
const policy = /* @__PURE__ */ new Map();
|
|
26205
|
+
for (const children of byBase.values()) {
|
|
26206
|
+
const maxParallel = children[0]?.lockJob.maxParallel;
|
|
26207
|
+
if (maxParallel === void 0 || children.length <= 1) continue;
|
|
26208
|
+
const failFast = children[0]?.lockJob.failFast ?? false;
|
|
26209
|
+
[...children].sort((a, b) => a.expandedName.localeCompare(b.expandedName)).forEach((mat, i) => {
|
|
26210
|
+
policy.set(mat.expandedName, {
|
|
26211
|
+
maxParallel,
|
|
26212
|
+
failFast
|
|
26213
|
+
});
|
|
26214
|
+
if (i >= maxParallel) held.add(mat.expandedName);
|
|
26215
|
+
});
|
|
26216
|
+
}
|
|
26217
|
+
return {
|
|
26218
|
+
held,
|
|
26219
|
+
policy
|
|
26220
|
+
};
|
|
26221
|
+
}
|
|
26222
|
+
async function materializeStaticJobsSafe(staticJobs, deps) {
|
|
25283
26223
|
const materializedJobs = [];
|
|
25284
26224
|
const expansionMap = /* @__PURE__ */ new Map();
|
|
25285
26225
|
const matrixFailures = [];
|
|
25286
26226
|
for (const lockJob of staticJobs) try {
|
|
25287
|
-
const result = materializeFanout([lockJob]);
|
|
26227
|
+
const result = lockJob.runsOnAll ? materializeResolvedHosts(lockJob, await resolveHostFanoutTargets(lockJob, deps), deps.maxFanoutHosts ?? 1024) : materializeFanout([lockJob]);
|
|
25288
26228
|
materializedJobs.push(...result.jobs);
|
|
25289
26229
|
for (const [k, v] of result.expansionMap) expansionMap.set(k, v);
|
|
25290
26230
|
} catch (err) {
|
|
@@ -25428,7 +26368,7 @@ async function prepareCacheAndBuild(ctx, setup) {
|
|
|
25428
26368
|
}
|
|
25429
26369
|
}
|
|
25430
26370
|
if (!contentHash && deps.sourceCache) logger$38.debug("Workflow missing contentHash, agents will compile from source", { workflow: workflow.name });
|
|
25431
|
-
const { materializedJobs, expansionMap, matrixFailures } = materializeStaticJobsSafe(staticJobs);
|
|
26371
|
+
const { materializedJobs, expansionMap, matrixFailures } = await materializeStaticJobsSafe(staticJobs, deps);
|
|
25432
26372
|
return {
|
|
25433
26373
|
sourceTarUrl,
|
|
25434
26374
|
sourceTarHash,
|
|
@@ -25903,6 +26843,13 @@ function makeBuildJobConfig(args) {
|
|
|
25903
26843
|
name: mat.expandedName,
|
|
25904
26844
|
baseJobName: mat.baseName,
|
|
25905
26845
|
...mat.variantValues && { matrixValues: mat.variantValues },
|
|
26846
|
+
...mat.host && { host: mat.host },
|
|
26847
|
+
...mat.agent && { agent: {
|
|
26848
|
+
host: mat.agent.host,
|
|
26849
|
+
labels: [...mat.agent.labels],
|
|
26850
|
+
...mat.agent.platform && { platform: mat.agent.platform },
|
|
26851
|
+
...mat.agent.arch && { arch: mat.agent.arch }
|
|
26852
|
+
} },
|
|
25906
26853
|
steps: lockJob.steps,
|
|
25907
26854
|
needs: lockJob.needs,
|
|
25908
26855
|
rules: lockJob.rules,
|
|
@@ -25931,15 +26878,17 @@ function makeBuildJobConfig(args) {
|
|
|
25931
26878
|
* single-orch paths).
|
|
25932
26879
|
*/
|
|
25933
26880
|
function buildExecutionJobInput(args) {
|
|
25934
|
-
const { ctx, setup, buildPrep, buildJobConfig, mat,
|
|
26881
|
+
const { ctx, setup, buildPrep, buildJobConfig, mat, selectors } = args;
|
|
25935
26882
|
const lockJob = mat.lockJob;
|
|
25936
26883
|
const { workflow, bundle, repoIdentifier, credentials, event, ref, runId } = ctx;
|
|
25937
26884
|
return {
|
|
25938
26885
|
runId,
|
|
25939
26886
|
workflowName: workflow.name,
|
|
25940
26887
|
jobName: mat.expandedName,
|
|
25941
|
-
runsOnLabels,
|
|
25942
|
-
|
|
26888
|
+
runsOnLabels: selectors.runsOnLabels,
|
|
26889
|
+
runsOnPatterns: selectors.runsOnPatterns,
|
|
26890
|
+
excludeLabels: selectors.excludeLabels,
|
|
26891
|
+
excludePatterns: selectors.excludePatterns,
|
|
25943
26892
|
jobConfig: buildJobConfig(mat),
|
|
25944
26893
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
25945
26894
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -25953,7 +26902,9 @@ function buildExecutionJobInput(args) {
|
|
|
25953
26902
|
depsUrl: buildPrep.depsUrl,
|
|
25954
26903
|
depsHash: buildPrep.depsHash,
|
|
25955
26904
|
requestId: getRequestContext().requestId,
|
|
25956
|
-
...lockJob.resources && { resources: lockJob.resources }
|
|
26905
|
+
...lockJob.resources && { resources: lockJob.resources },
|
|
26906
|
+
...mat.pinnedAgentId && { pinnedAgentId: mat.pinnedAgentId },
|
|
26907
|
+
...mat.connectedInstanceId !== void 0 && { connectedInstanceId: mat.connectedInstanceId }
|
|
25957
26908
|
};
|
|
25958
26909
|
}
|
|
25959
26910
|
/**
|
|
@@ -25967,7 +26918,8 @@ async function holdJobForApproval(args) {
|
|
|
25967
26918
|
const { ctx, setup, buildPrep, buildJobConfig, mat, envData, dispatchedJobs } = args;
|
|
25968
26919
|
const lockJob = mat.lockJob;
|
|
25969
26920
|
const { deps, workflow, runId } = ctx;
|
|
25970
|
-
const
|
|
26921
|
+
const selectors = runsOnSelectorsForLockJob(lockJob);
|
|
26922
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
25971
26923
|
const hold = envData.approvalHold;
|
|
25972
26924
|
if (!hold || !deps.heldRunStore || !deps.db) {
|
|
25973
26925
|
logger$38.info("Job held by protection rules", {
|
|
@@ -25983,8 +26935,7 @@ async function holdJobForApproval(args) {
|
|
|
25983
26935
|
buildPrep,
|
|
25984
26936
|
buildJobConfig,
|
|
25985
26937
|
mat,
|
|
25986
|
-
|
|
25987
|
-
excludeLabels: lockJob._type === "static" && lockJob.excludeLabels ? [...lockJob.excludeLabels] : void 0
|
|
26938
|
+
selectors
|
|
25988
26939
|
});
|
|
25989
26940
|
const heldRow = await deps.heldRunStore.createHold(ctx.resolvedOrgId, {
|
|
25990
26941
|
runId,
|
|
@@ -26072,15 +27023,15 @@ async function preRegisterNonRootJobs(args) {
|
|
|
26072
27023
|
const { deps, workflow, runId } = ctx;
|
|
26073
27024
|
for (const gated of needsGatedJobs) {
|
|
26074
27025
|
const gatedJob = gated.lockJob;
|
|
26075
|
-
const
|
|
27026
|
+
const selectors = runsOnSelectorsForLockJob(gatedJob);
|
|
27027
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
26076
27028
|
const jobInput = buildExecutionJobInput({
|
|
26077
27029
|
ctx,
|
|
26078
27030
|
setup,
|
|
26079
27031
|
buildPrep,
|
|
26080
27032
|
buildJobConfig,
|
|
26081
27033
|
mat: gated,
|
|
26082
|
-
|
|
26083
|
-
excludeLabels: gatedJob._type === "static" && gatedJob.excludeLabels ? [...gatedJob.excludeLabels] : void 0
|
|
27034
|
+
selectors
|
|
26084
27035
|
});
|
|
26085
27036
|
await storePendingJobContext(deps.db, runId, gated.expandedName, {
|
|
26086
27037
|
jobInput,
|
|
@@ -26139,9 +27090,12 @@ async function clusterRouteRootJobs(args) {
|
|
|
26139
27090
|
};
|
|
26140
27091
|
const jobsToRoute = rootDispatchableJobs.map((mj) => {
|
|
26141
27092
|
const j = mj.lockJob;
|
|
27093
|
+
const sel = runsOnSelectorsForLockJob(j);
|
|
26142
27094
|
return {
|
|
26143
27095
|
jobName: mj.expandedName,
|
|
26144
|
-
runsOnLabels: [
|
|
27096
|
+
runsOnLabels: [sel.runsOnLabels],
|
|
27097
|
+
runsOnPatterns: sel.runsOnPatterns,
|
|
27098
|
+
excludePatterns: sel.excludePatterns,
|
|
26145
27099
|
jobConfig: buildJobConfig(mj),
|
|
26146
27100
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26147
27101
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26150,7 +27104,7 @@ async function clusterRouteRootJobs(args) {
|
|
|
26150
27104
|
sourceTarHash: buildPrep.sourceTarHash,
|
|
26151
27105
|
depsUrl: buildPrep.depsUrl,
|
|
26152
27106
|
depsHash: buildPrep.depsHash,
|
|
26153
|
-
excludeLabels:
|
|
27107
|
+
excludeLabels: sel.excludeLabels,
|
|
26154
27108
|
...j.resources ? { resources: j.resources } : {}
|
|
26155
27109
|
};
|
|
26156
27110
|
});
|
|
@@ -26174,6 +27128,8 @@ async function clusterRouteRootJobs(args) {
|
|
|
26174
27128
|
workflowName: workflow.name,
|
|
26175
27129
|
jobName: jtr.jobName,
|
|
26176
27130
|
runsOnLabels: flatLabels,
|
|
27131
|
+
runsOnPatterns: jtr.runsOnPatterns,
|
|
27132
|
+
excludePatterns: jtr.excludePatterns,
|
|
26177
27133
|
excludeLabels: jtr.excludeLabels,
|
|
26178
27134
|
jobConfig: jtr.jobConfig,
|
|
26179
27135
|
repoUrl: jtr.repoUrl,
|
|
@@ -26253,6 +27209,15 @@ async function clusterRouteRootJobs(args) {
|
|
|
26253
27209
|
async function dispatchSingleOrchPath(args) {
|
|
26254
27210
|
const { ctx, setup, buildPrep, buildJobConfig, jobEnvironmentData, dispatchedJobs, rejectedJobs } = args;
|
|
26255
27211
|
const { deps, workflow, runId } = ctx;
|
|
27212
|
+
const wavePlan = computeWavePlan(buildPrep.materializedJobs);
|
|
27213
|
+
/** The wave-policy fields persisted on a bounded-wave child's execution_jobs row. */
|
|
27214
|
+
const wavePolicyFields = (name) => {
|
|
27215
|
+
const p = wavePlan.policy.get(name);
|
|
27216
|
+
return p ? {
|
|
27217
|
+
waveMaxParallel: p.maxParallel,
|
|
27218
|
+
waveFailFast: p.failFast
|
|
27219
|
+
} : {};
|
|
27220
|
+
};
|
|
26256
27221
|
for (const mat of buildPrep.materializedJobs) {
|
|
26257
27222
|
const lockJob = mat.lockJob;
|
|
26258
27223
|
const matrixValues = mat.variantValues;
|
|
@@ -26279,16 +27244,16 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26279
27244
|
continue;
|
|
26280
27245
|
}
|
|
26281
27246
|
if (envData?.pendingInit) continue;
|
|
26282
|
-
const
|
|
26283
|
-
const
|
|
27247
|
+
const selectors = runsOnSelectorsForLockJob(lockJob);
|
|
27248
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
27249
|
+
const excludeLabels = selectors.excludeLabels;
|
|
26284
27250
|
const jobInput = buildExecutionJobInput({
|
|
26285
27251
|
ctx,
|
|
26286
27252
|
setup,
|
|
26287
27253
|
buildPrep,
|
|
26288
27254
|
buildJobConfig,
|
|
26289
27255
|
mat,
|
|
26290
|
-
|
|
26291
|
-
excludeLabels
|
|
27256
|
+
selectors
|
|
26292
27257
|
});
|
|
26293
27258
|
if (!isRootJob(lockJob)) {
|
|
26294
27259
|
await storePendingJobContext(deps.db, runId, mat.expandedName, {
|
|
@@ -26300,6 +27265,7 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26300
27265
|
jobId: syntheticId,
|
|
26301
27266
|
jobName: mat.expandedName,
|
|
26302
27267
|
...matrixValues && { matrixValues },
|
|
27268
|
+
...variantTrackingFields(mat),
|
|
26303
27269
|
runsOnLabels
|
|
26304
27270
|
});
|
|
26305
27271
|
logger$38.info("Job gated by needs scheduler (not dispatched yet)", {
|
|
@@ -26309,6 +27275,29 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26309
27275
|
});
|
|
26310
27276
|
continue;
|
|
26311
27277
|
}
|
|
27278
|
+
if (wavePlan.held.has(mat.expandedName)) {
|
|
27279
|
+
await storePendingJobContext(deps.db, runId, mat.expandedName, {
|
|
27280
|
+
jobInput,
|
|
27281
|
+
runsOnLabels
|
|
27282
|
+
});
|
|
27283
|
+
const syntheticId = `needs-pending-${mat.expandedName}-${randomUUID()}`;
|
|
27284
|
+
dispatchedJobs.push({
|
|
27285
|
+
jobId: syntheticId,
|
|
27286
|
+
jobName: mat.expandedName,
|
|
27287
|
+
...matrixValues && { matrixValues },
|
|
27288
|
+
...variantTrackingFields(mat),
|
|
27289
|
+
...wavePolicyFields(mat.expandedName),
|
|
27290
|
+
runsOnLabels,
|
|
27291
|
+
waveGated: true
|
|
27292
|
+
});
|
|
27293
|
+
logger$38.info("Fan-out child held by rolling wave (maxParallel)", {
|
|
27294
|
+
runId,
|
|
27295
|
+
workflow: workflow.name,
|
|
27296
|
+
job: mat.expandedName,
|
|
27297
|
+
maxParallel: lockJob.maxParallel
|
|
27298
|
+
});
|
|
27299
|
+
continue;
|
|
27300
|
+
}
|
|
26312
27301
|
const result = await setup.dispatcher.dispatch(jobInput);
|
|
26313
27302
|
if (result.status === "rejected") {
|
|
26314
27303
|
const syntheticId = `rejected-${randomUUID()}`;
|
|
@@ -26316,6 +27305,8 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26316
27305
|
jobId: syntheticId,
|
|
26317
27306
|
jobName: mat.expandedName,
|
|
26318
27307
|
...matrixValues && { matrixValues },
|
|
27308
|
+
...variantTrackingFields(mat),
|
|
27309
|
+
...wavePolicyFields(mat.expandedName),
|
|
26319
27310
|
runsOnLabels
|
|
26320
27311
|
});
|
|
26321
27312
|
rejectedJobs.push({
|
|
@@ -26334,6 +27325,8 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26334
27325
|
jobId: result.jobId,
|
|
26335
27326
|
jobName: mat.expandedName,
|
|
26336
27327
|
...matrixValues && { matrixValues },
|
|
27328
|
+
...variantTrackingFields(mat),
|
|
27329
|
+
...wavePolicyFields(mat.expandedName),
|
|
26337
27330
|
runsOnLabels
|
|
26338
27331
|
});
|
|
26339
27332
|
logger$38.info("Job dispatched", {
|
|
@@ -26505,22 +27498,24 @@ async function dispatchExecutionAfterInit(args) {
|
|
|
26505
27498
|
const lockJob = mat.lockJob;
|
|
26506
27499
|
const matrixValues = mat.variantValues;
|
|
26507
27500
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
26508
|
-
const
|
|
26509
|
-
const
|
|
27501
|
+
const selectors = runsOnSelectorsForLockJob(lockJob);
|
|
27502
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
27503
|
+
const excludeLabels = selectors.excludeLabels;
|
|
26510
27504
|
const jobInput = buildExecutionJobInput({
|
|
26511
27505
|
ctx,
|
|
26512
27506
|
setup,
|
|
26513
27507
|
buildPrep,
|
|
26514
27508
|
buildJobConfig,
|
|
26515
27509
|
mat,
|
|
26516
|
-
|
|
26517
|
-
excludeLabels
|
|
27510
|
+
selectors
|
|
26518
27511
|
});
|
|
26519
27512
|
let dispatchStatus;
|
|
26520
27513
|
if (deps.coordinator && deps.coordinator.hasConnectedPeers()) {
|
|
26521
27514
|
const jobToRoute = {
|
|
26522
27515
|
jobName: mat.expandedName,
|
|
26523
27516
|
runsOnLabels: [runsOnLabels],
|
|
27517
|
+
runsOnPatterns: selectors.runsOnPatterns,
|
|
27518
|
+
excludePatterns: selectors.excludePatterns,
|
|
26524
27519
|
jobConfig: buildJobConfig(mat),
|
|
26525
27520
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26526
27521
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26716,7 +27711,7 @@ function startDeferredInitDispatch(args) {
|
|
|
26716
27711
|
jobId,
|
|
26717
27712
|
jobName: mat.expandedName,
|
|
26718
27713
|
...mat.variantValues && { matrixValues: mat.variantValues },
|
|
26719
|
-
runsOnLabels:
|
|
27714
|
+
runsOnLabels: runsOnSelectorsForLockJob(lockJob).runsOnLabels
|
|
26720
27715
|
}]).catch(() => {});
|
|
26721
27716
|
const carried = err instanceof AgentJobFailedError ? err.initFailure : void 0;
|
|
26722
27717
|
await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
|
|
@@ -26734,14 +27729,15 @@ function startDeferredInitDispatch(args) {
|
|
|
26734
27729
|
}
|
|
26735
27730
|
}
|
|
26736
27731
|
async function dispatchEvalJob(args) {
|
|
26737
|
-
const { ctx, setup, buildPrep, dynamicEntry } = args;
|
|
27732
|
+
const { ctx, setup, buildPrep, dynamicEntry, upstreamSnapshot } = args;
|
|
26738
27733
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
26739
27734
|
const evalJobName = `__dynamic__${workflow.name}__${dynamicEntry.source.index}`;
|
|
26740
27735
|
logger$38.info("Dispatching dynamic eval job", {
|
|
26741
27736
|
runId,
|
|
26742
27737
|
workflow: workflow.name,
|
|
26743
27738
|
evalJob: evalJobName,
|
|
26744
|
-
sourceIndex: dynamicEntry.source.index
|
|
27739
|
+
sourceIndex: dynamicEntry.source.index,
|
|
27740
|
+
resultAware: !!upstreamSnapshot
|
|
26745
27741
|
});
|
|
26746
27742
|
const evalJobInput = {
|
|
26747
27743
|
runId,
|
|
@@ -26759,7 +27755,12 @@ async function dispatchEvalJob(args) {
|
|
|
26759
27755
|
event,
|
|
26760
27756
|
timeoutMs: 12e4,
|
|
26761
27757
|
...workflow.contentHash && { contentHash: workflow.contentHash },
|
|
26762
|
-
...workflow.resolvedHashFiles?.length && { resolvedHashFiles: workflow.resolvedHashFiles }
|
|
27758
|
+
...workflow.resolvedHashFiles?.length && { resolvedHashFiles: workflow.resolvedHashFiles },
|
|
27759
|
+
...upstreamSnapshot && {
|
|
27760
|
+
resultAware: true,
|
|
27761
|
+
declaredNeeds: dynamicEntry.needs ?? [],
|
|
27762
|
+
upstreamSnapshot
|
|
27763
|
+
}
|
|
26763
27764
|
},
|
|
26764
27765
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26765
27766
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26774,38 +27775,65 @@ async function dispatchEvalJob(args) {
|
|
|
26774
27775
|
depsUrl: buildPrep.depsUrl,
|
|
26775
27776
|
depsHash: buildPrep.depsHash
|
|
26776
27777
|
};
|
|
27778
|
+
const replaceSyntheticId = upstreamSnapshot && deps.executionTracker ? await deps.executionTracker.findDynamicEvalSyntheticId(runId, evalJobName) : void 0;
|
|
26777
27779
|
const evalResult = await setup.dispatcher.dispatch(evalJobInput);
|
|
26778
27780
|
if (evalResult.status !== "dispatched" && evalResult.status !== "queued") throw new Error(`Dynamic eval job dispatch rejected: ${evalResult.status}`);
|
|
26779
|
-
if (deps.executionTracker) await deps.executionTracker.addJobsToRun(runId, [{
|
|
26780
|
-
jobId: evalResult.jobId,
|
|
26781
|
-
jobName: evalJobName,
|
|
26782
|
-
runsOnLabels: evalJobInput.runsOnLabels
|
|
26783
|
-
}]);
|
|
26784
27781
|
return {
|
|
26785
27782
|
evalJobId: evalResult.jobId,
|
|
26786
|
-
evalJobLabels: evalJobInput.runsOnLabels
|
|
27783
|
+
evalJobLabels: evalJobInput.runsOnLabels,
|
|
27784
|
+
evalJobName,
|
|
27785
|
+
replaceSyntheticId,
|
|
27786
|
+
runsOnLabels: evalJobInput.runsOnLabels
|
|
26787
27787
|
};
|
|
26788
27788
|
}
|
|
26789
27789
|
/**
|
|
26790
27790
|
* Resolve env/secrets per generated job and build their job configs.
|
|
26791
27791
|
* Skips jobs that fail individual secret resolution.
|
|
26792
27792
|
*/
|
|
27793
|
+
/**
|
|
27794
|
+
* Records a dropped generated-job matrix as a `matrix_expansion` init failure so
|
|
27795
|
+
* the run's dashboard surfaces it, mirroring the static / top-level dynamic-matrix
|
|
27796
|
+
* paths. A no-op when the run has no execution tracker.
|
|
27797
|
+
*/
|
|
27798
|
+
async function recordGeneratedMatrixFailure(deps, runId, err) {
|
|
27799
|
+
if (!deps.executionTracker) return;
|
|
27800
|
+
const jobId = `matrix-failed-${randomUUID()}`;
|
|
27801
|
+
await deps.executionTracker.addJobsToRun(runId, [{
|
|
27802
|
+
jobId,
|
|
27803
|
+
jobName: err.jobName,
|
|
27804
|
+
runsOnLabels: []
|
|
27805
|
+
}]).catch(() => {});
|
|
27806
|
+
await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
|
|
27807
|
+
error: err.message,
|
|
27808
|
+
initFailure: {
|
|
27809
|
+
scope: "job",
|
|
27810
|
+
category: InitFailureCategory.enum.matrix_expansion,
|
|
27811
|
+
message: err.message,
|
|
27812
|
+
jobName: err.jobName
|
|
27813
|
+
}
|
|
27814
|
+
}).catch(() => {});
|
|
27815
|
+
}
|
|
26793
27816
|
async function resolveGeneratedJobConfigs(args) {
|
|
26794
|
-
const { ctx, workflow, fullLockFile, resolvedSecrets, resolvedNamespacedSecrets, runPublicKeyBase64, npmRegistries, installEnvSecrets, generatedJobs, dynamicEntry } = args;
|
|
27817
|
+
const { ctx, workflow, fullLockFile, resolvedSecrets, resolvedNamespacedSecrets, runPublicKeyBase64, npmRegistries, installEnvSecrets, generatedJobs, dynamicEntry, upstreamSnapshot } = args;
|
|
26795
27818
|
const { deps, runId, resolvedOrgId, event } = ctx;
|
|
26796
27819
|
const out = [];
|
|
26797
27820
|
let fanout;
|
|
26798
|
-
|
|
26799
|
-
|
|
27821
|
+
const remaining = [...generatedJobs];
|
|
27822
|
+
for (;;) try {
|
|
27823
|
+
fanout = materializeFanout(remaining);
|
|
27824
|
+
break;
|
|
26800
27825
|
} catch (err) {
|
|
26801
|
-
if (err instanceof FanoutError)
|
|
26802
|
-
|
|
26803
|
-
|
|
26804
|
-
|
|
26805
|
-
|
|
26806
|
-
|
|
26807
|
-
|
|
26808
|
-
|
|
27826
|
+
if (!(err instanceof FanoutError)) throw err;
|
|
27827
|
+
logger$38.error("Dynamic generated job matrix materialization failed", {
|
|
27828
|
+
runId,
|
|
27829
|
+
job: err.jobName,
|
|
27830
|
+
error: err.message
|
|
27831
|
+
});
|
|
27832
|
+
await recordGeneratedMatrixFailure(deps, runId, err);
|
|
27833
|
+
const before = remaining.length;
|
|
27834
|
+
const idx = remaining.findIndex((j) => j.name === err.jobName);
|
|
27835
|
+
if (idx >= 0) remaining.splice(idx, 1);
|
|
27836
|
+
if (remaining.length === before) throw err;
|
|
26809
27837
|
}
|
|
26810
27838
|
const expectedJobNames = [...new Set(fanout.jobs.map((m) => m.baseName))];
|
|
26811
27839
|
const expandNeeds = (needs) => {
|
|
@@ -26883,9 +27911,14 @@ async function resolveGeneratedJobConfigs(args) {
|
|
|
26883
27911
|
dynamicSource: {
|
|
26884
27912
|
index: dynamicEntry.source.index,
|
|
26885
27913
|
event,
|
|
26886
|
-
expectedJobNames
|
|
27914
|
+
expectedJobNames,
|
|
27915
|
+
...upstreamSnapshot && {
|
|
27916
|
+
upstreamSnapshot,
|
|
27917
|
+
declaredNeeds: dynamicEntry.needs ?? []
|
|
27918
|
+
}
|
|
26887
27919
|
}
|
|
26888
27920
|
};
|
|
27921
|
+
const genSel = runsOnSelectorsForLockJob(genJob);
|
|
26889
27922
|
out.push({
|
|
26890
27923
|
genJob: {
|
|
26891
27924
|
...genJob,
|
|
@@ -26893,7 +27926,10 @@ async function resolveGeneratedJobConfigs(args) {
|
|
|
26893
27926
|
needs: expandedNeeds
|
|
26894
27927
|
},
|
|
26895
27928
|
genJobConfig,
|
|
26896
|
-
runsOnLabels:
|
|
27929
|
+
runsOnLabels: genSel.runsOnLabels,
|
|
27930
|
+
runsOnPatterns: genSel.runsOnPatterns,
|
|
27931
|
+
excludeLabels: genSel.excludeLabels,
|
|
27932
|
+
excludePatterns: genSel.excludePatterns,
|
|
26897
27933
|
...envelope.matrixValues && { matrixValues: envelope.matrixValues }
|
|
26898
27934
|
});
|
|
26899
27935
|
} catch (err) {
|
|
@@ -26924,13 +27960,15 @@ async function gateAndStoreNonRootGeneratedJobs(args) {
|
|
|
26924
27960
|
if_failed: need.ifFailed ?? "skip"
|
|
26925
27961
|
});
|
|
26926
27962
|
if (gatedEdgeRows.length > 0) await deps.db.insertInto("execution_job_needs").values(gatedEdgeRows).onConflict((oc) => oc.doNothing()).execute();
|
|
26927
|
-
for (const { genJob, genJobConfig, runsOnLabels, matrixValues } of gatedGeneratedConfigs) {
|
|
27963
|
+
for (const { genJob, genJobConfig, runsOnLabels, runsOnPatterns, excludeLabels, excludePatterns, matrixValues } of gatedGeneratedConfigs) {
|
|
26928
27964
|
const gatedJobInput = {
|
|
26929
27965
|
runId,
|
|
26930
27966
|
workflowName: workflow.name,
|
|
26931
27967
|
jobName: genJob.name,
|
|
26932
27968
|
runsOnLabels,
|
|
26933
|
-
|
|
27969
|
+
runsOnPatterns,
|
|
27970
|
+
excludeLabels,
|
|
27971
|
+
excludePatterns,
|
|
26934
27972
|
jobConfig: genJobConfig,
|
|
26935
27973
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26936
27974
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26967,13 +28005,15 @@ async function gateAndStoreNonRootGeneratedJobs(args) {
|
|
|
26967
28005
|
async function directDispatchGeneratedJobs(args) {
|
|
26968
28006
|
const { ctx, setup, buildPrep, configs } = args;
|
|
26969
28007
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
26970
|
-
for (const { genJob, genJobConfig, runsOnLabels, matrixValues } of configs) try {
|
|
28008
|
+
for (const { genJob, genJobConfig, runsOnLabels, runsOnPatterns, excludeLabels, excludePatterns, matrixValues } of configs) try {
|
|
26971
28009
|
const genJobInput = {
|
|
26972
28010
|
runId,
|
|
26973
28011
|
workflowName: workflow.name,
|
|
26974
28012
|
jobName: genJob.name,
|
|
26975
28013
|
runsOnLabels,
|
|
26976
|
-
|
|
28014
|
+
runsOnPatterns,
|
|
28015
|
+
excludeLabels,
|
|
28016
|
+
excludePatterns,
|
|
26977
28017
|
jobConfig: genJobConfig,
|
|
26978
28018
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26979
28019
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -27012,9 +28052,11 @@ async function routeRootGeneratedJobs(args) {
|
|
|
27012
28052
|
const { ctx, setup, buildPrep, rootGeneratedConfigs } = args;
|
|
27013
28053
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
27014
28054
|
if (rootGeneratedConfigs.length === 0) return;
|
|
27015
|
-
const generatedJobsToRoute = rootGeneratedConfigs.map(({ genJob, genJobConfig, runsOnLabels }) => ({
|
|
28055
|
+
const generatedJobsToRoute = rootGeneratedConfigs.map(({ genJob, genJobConfig, runsOnLabels, runsOnPatterns, excludeLabels, excludePatterns }) => ({
|
|
27016
28056
|
jobName: genJob.name,
|
|
27017
28057
|
runsOnLabels: [runsOnLabels],
|
|
28058
|
+
runsOnPatterns,
|
|
28059
|
+
excludePatterns,
|
|
27018
28060
|
jobConfig: genJobConfig,
|
|
27019
28061
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
27020
28062
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -27023,7 +28065,7 @@ async function routeRootGeneratedJobs(args) {
|
|
|
27023
28065
|
sourceTarHash: buildPrep.contentHash || void 0,
|
|
27024
28066
|
depsUrl: buildPrep.depsUrl,
|
|
27025
28067
|
depsHash: buildPrep.depsHash,
|
|
27026
|
-
excludeLabels
|
|
28068
|
+
excludeLabels,
|
|
27027
28069
|
...genJob.resources ? { resources: genJob.resources } : {}
|
|
27028
28070
|
}));
|
|
27029
28071
|
if (!(deps.coordinator && deps.coordinator.hasConnectedPeers())) {
|
|
@@ -27216,18 +28258,140 @@ async function recomputeAndDispatchReady(args) {
|
|
|
27216
28258
|
if (skipJobRow) await deps.executionTracker.onJobStatus(runId, skipJobRow.job_id, ExecutionJobStatus.enum.skipped, Date.now(), void 0, { error: result.reason });
|
|
27217
28259
|
}
|
|
27218
28260
|
}
|
|
28261
|
+
/**
|
|
28262
|
+
* Split a result-aware generator's declared needs into static/named upstream job
|
|
28263
|
+
* names and dynamic-group names. Reuses the same normalized lock edge shapes the
|
|
28264
|
+
* static-job `needs` serializer produces.
|
|
28265
|
+
*/
|
|
28266
|
+
function splitDeclaredNeeds(needs) {
|
|
28267
|
+
const jobNames = [];
|
|
28268
|
+
const groupNames = [];
|
|
28269
|
+
for (const need of needs ?? []) if (typeof need === "string") jobNames.push(need);
|
|
28270
|
+
else if ("group" in need) groupNames.push(need.group);
|
|
28271
|
+
else if ("name" in need) jobNames.push(need.name);
|
|
28272
|
+
return {
|
|
28273
|
+
jobNames,
|
|
28274
|
+
groupNames
|
|
28275
|
+
};
|
|
28276
|
+
}
|
|
28277
|
+
/**
|
|
28278
|
+
* Register a result-aware generator's eval job as a deferred, needs-gated DAG
|
|
28279
|
+
* job: insert a synthetic pending execution_jobs row plus its execution_job_needs
|
|
28280
|
+
* edges, so the existing scheduler gates the eval exactly like any other job.
|
|
28281
|
+
* Group needs expand to their member job names (members already carry group_name
|
|
28282
|
+
* from setGroupNameAndResolveEdges on the group's own eval completion).
|
|
28283
|
+
*/
|
|
28284
|
+
async function registerDeferredEvalJob(args) {
|
|
28285
|
+
const { ctx, evalJobName, dynamicEntry } = args;
|
|
28286
|
+
const { deps, runId } = ctx;
|
|
28287
|
+
if (!deps.db) return;
|
|
28288
|
+
const { jobNames, groupNames } = splitDeclaredNeeds(dynamicEntry.needs);
|
|
28289
|
+
const groupMembers = [];
|
|
28290
|
+
for (const groupName of groupNames) {
|
|
28291
|
+
const members = await deps.db.selectFrom("execution_jobs").select("job_name").where("run_id", "=", runId).where("group_name", "=", groupName).execute();
|
|
28292
|
+
for (const m of members) groupMembers.push(m.job_name);
|
|
28293
|
+
}
|
|
28294
|
+
const upstreamNames = [...new Set([...jobNames, ...groupMembers])];
|
|
28295
|
+
const ifFailedByName = /* @__PURE__ */ new Map();
|
|
28296
|
+
for (const need of dynamicEntry.needs ?? []) if (typeof need === "object" && "name" in need) ifFailedByName.set(need.name, need.ifFailed ?? "skip");
|
|
28297
|
+
const groupIfFailed = /* @__PURE__ */ new Map();
|
|
28298
|
+
for (const need of dynamicEntry.needs ?? []) if (typeof need === "object" && "group" in need) groupIfFailed.set(need.group, need.ifFailed ?? "skip");
|
|
28299
|
+
const syntheticId = `dynamic-eval-pending-${evalJobName}-${randomUUID()}`;
|
|
28300
|
+
if (deps.executionTracker) await deps.executionTracker.addJobsToRun(runId, [{
|
|
28301
|
+
jobId: syntheticId,
|
|
28302
|
+
jobName: evalJobName,
|
|
28303
|
+
runsOnLabels: []
|
|
28304
|
+
}]);
|
|
28305
|
+
const edgeRows = upstreamNames.map((upstreamName) => ({
|
|
28306
|
+
run_id: runId,
|
|
28307
|
+
job_name: evalJobName,
|
|
28308
|
+
upstream_name: upstreamName,
|
|
28309
|
+
if_failed: ifFailedByName.get(upstreamName) ?? [...groupIfFailed.values()][0] ?? "skip"
|
|
28310
|
+
}));
|
|
28311
|
+
if (edgeRows.length > 0) await deps.db.insertInto("execution_job_needs").values(edgeRows).onConflict((oc) => oc.doNothing()).execute();
|
|
28312
|
+
logger$38.info("Registered deferred result-aware eval job", {
|
|
28313
|
+
runId,
|
|
28314
|
+
evalJob: evalJobName,
|
|
28315
|
+
upstreams: upstreamNames
|
|
28316
|
+
});
|
|
28317
|
+
const results = await recomputeNeedsSatisfied(deps.db, runId, [evalJobName]);
|
|
28318
|
+
for (const result of results) if (result.action === "dispatch" && deps.executionTracker?.onJobReadyCallback) await deps.executionTracker.onJobReadyCallback(runId, evalJobName);
|
|
28319
|
+
else if (result.action === "skip") {
|
|
28320
|
+
if (deps.executionTracker?.onJobReadyCallback) await deps.executionTracker.onJobReadyCallback(runId, evalJobName);
|
|
28321
|
+
}
|
|
28322
|
+
}
|
|
28323
|
+
/**
|
|
28324
|
+
* Gather the frozen upstream snapshot for a result-aware eval: each declared
|
|
28325
|
+
* job/group-member's stored outputs (the same plain outputs map that backs
|
|
28326
|
+
* jobRef.result), plus group → ordered member names. Captured once, at eval
|
|
28327
|
+
* dispatch, and replayed unchanged on agent-side re-eval.
|
|
28328
|
+
*/
|
|
28329
|
+
async function gatherUpstreamSnapshot(args) {
|
|
28330
|
+
const { ctx, dynamicEntry } = args;
|
|
28331
|
+
const { deps, runId } = ctx;
|
|
28332
|
+
const snapshot = {
|
|
28333
|
+
jobs: {},
|
|
28334
|
+
groups: {}
|
|
28335
|
+
};
|
|
28336
|
+
if (!deps.db) return snapshot;
|
|
28337
|
+
const { jobNames, groupNames } = splitDeclaredNeeds(dynamicEntry.needs);
|
|
28338
|
+
const groupMembers = [];
|
|
28339
|
+
for (const groupName of groupNames) {
|
|
28340
|
+
const memberNames = (await deps.db.selectFrom("execution_jobs").select(["job_name", "ready_at"]).where("run_id", "=", runId).where("group_name", "=", groupName).orderBy("ready_at", "asc").orderBy("job_name", "asc").execute()).map((m) => m.job_name);
|
|
28341
|
+
snapshot.groups[groupName] = memberNames;
|
|
28342
|
+
for (const n of memberNames) groupMembers.push(n);
|
|
28343
|
+
}
|
|
28344
|
+
const allJobNames = [...new Set([...jobNames, ...groupMembers])];
|
|
28345
|
+
if (allJobNames.length > 0) {
|
|
28346
|
+
const rows = await deps.db.selectFrom("execution_jobs").select(["job_name", "outputs"]).where("run_id", "=", runId).where("job_name", "in", allJobNames).execute();
|
|
28347
|
+
for (const row of rows) {
|
|
28348
|
+
const parsed = parseOutputsCell(row.outputs);
|
|
28349
|
+
if (parsed) snapshot.jobs[row.job_name] = parsed;
|
|
28350
|
+
}
|
|
28351
|
+
}
|
|
28352
|
+
return snapshot;
|
|
28353
|
+
}
|
|
27219
28354
|
async function processDynamicEntry(args) {
|
|
27220
28355
|
const { ctx, setup, buildPrep, secrets, dynamicEntry } = args;
|
|
27221
28356
|
const { deps, workflow, fullLockFile, runId } = ctx;
|
|
27222
28357
|
if (!deps.pendingDynamics) return;
|
|
28358
|
+
const evalJobName = `__dynamic__${workflow.name}__${dynamicEntry.source.index}`;
|
|
27223
28359
|
try {
|
|
27224
|
-
|
|
28360
|
+
let upstreamSnapshot;
|
|
28361
|
+
if (dynamicEntry.resultAware) {
|
|
28362
|
+
const gateOpened = trackEvalGate(runId, evalJobName);
|
|
28363
|
+
await registerDeferredEvalJob({
|
|
28364
|
+
ctx,
|
|
28365
|
+
evalJobName,
|
|
28366
|
+
dynamicEntry
|
|
28367
|
+
});
|
|
28368
|
+
await gateOpened;
|
|
28369
|
+
upstreamSnapshot = await gatherUpstreamSnapshot({
|
|
28370
|
+
ctx,
|
|
28371
|
+
dynamicEntry
|
|
28372
|
+
});
|
|
28373
|
+
logger$38.info("Result-aware eval gate opened, dispatching eval with snapshot", {
|
|
28374
|
+
runId,
|
|
28375
|
+
workflow: workflow.name,
|
|
28376
|
+
evalJob: evalJobName,
|
|
28377
|
+
snapshotJobs: Object.keys(upstreamSnapshot.jobs).length,
|
|
28378
|
+
snapshotGroups: Object.keys(upstreamSnapshot.groups).length
|
|
28379
|
+
});
|
|
28380
|
+
}
|
|
28381
|
+
const { evalJobId, replaceSyntheticId, runsOnLabels } = await dispatchEvalJob({
|
|
27225
28382
|
ctx,
|
|
27226
28383
|
setup,
|
|
27227
28384
|
buildPrep,
|
|
27228
|
-
dynamicEntry
|
|
28385
|
+
dynamicEntry,
|
|
28386
|
+
upstreamSnapshot
|
|
27229
28387
|
});
|
|
27230
|
-
const
|
|
28388
|
+
const generatedJobsPromise = deps.pendingDynamics.track(evalJobId);
|
|
28389
|
+
if (deps.executionTracker) await deps.executionTracker.addJobsToRun(runId, [{
|
|
28390
|
+
jobId: evalJobId,
|
|
28391
|
+
jobName: evalJobName,
|
|
28392
|
+
runsOnLabels
|
|
28393
|
+
}], void 0, replaceSyntheticId);
|
|
28394
|
+
const generatedJobs = await generatedJobsPromise;
|
|
27231
28395
|
logger$38.info("Dynamic eval completed, dispatching generated jobs", {
|
|
27232
28396
|
runId,
|
|
27233
28397
|
workflow: workflow.name,
|
|
@@ -27244,7 +28408,8 @@ async function processDynamicEntry(args) {
|
|
|
27244
28408
|
npmRegistries: secrets.npmRegistries,
|
|
27245
28409
|
installEnvSecrets: secrets.installEnvSecrets,
|
|
27246
28410
|
generatedJobs,
|
|
27247
|
-
dynamicEntry
|
|
28411
|
+
dynamicEntry,
|
|
28412
|
+
upstreamSnapshot
|
|
27248
28413
|
});
|
|
27249
28414
|
const rootGeneratedConfigs = generatedJobConfigs.filter((c) => isRootJob(c.genJob));
|
|
27250
28415
|
await gateAndStoreNonRootGeneratedJobs({
|
|
@@ -27562,6 +28727,8 @@ async function dispatchMatchedWorkflow(ctx, opts = {}) {
|
|
|
27562
28727
|
}
|
|
27563
28728
|
var logger$38;
|
|
27564
28729
|
var init_dispatch_matched_workflow = __esmMin((() => {
|
|
28730
|
+
init_host_roster();
|
|
28731
|
+
init_orchestrator_core();
|
|
27565
28732
|
init_agent_job_failed_error();
|
|
27566
28733
|
init_pipeline();
|
|
27567
28734
|
init_environment_store();
|
|
@@ -28217,7 +29384,9 @@ function buildGlobalWorkflowJobInputs(args) {
|
|
|
28217
29384
|
const materialized = materializeFanout(globalWorkflow.jobs.filter(isLockStaticJob)).jobs;
|
|
28218
29385
|
for (const mat of materialized) {
|
|
28219
29386
|
const lockJob = mat.lockJob;
|
|
28220
|
-
const
|
|
29387
|
+
const runsOnParts = partitionMatchers(lockJob.runsOn ?? []);
|
|
29388
|
+
const excludeParts = partitionMatchers(lockJob.excludeLabels ?? []);
|
|
29389
|
+
const flatLabels = runsOnParts.exact;
|
|
28221
29390
|
const jobConfig = {
|
|
28222
29391
|
source: globalWorkflow.source ?? reg.lockEntry.source,
|
|
28223
29392
|
workflowName: globalWorkflow.name,
|
|
@@ -28240,7 +29409,9 @@ function buildGlobalWorkflowJobInputs(args) {
|
|
|
28240
29409
|
workflowName: globalWorkflow.name,
|
|
28241
29410
|
jobName: mat.expandedName,
|
|
28242
29411
|
runsOnLabels: flatLabels,
|
|
28243
|
-
|
|
29412
|
+
runsOnPatterns: runsOnParts.regex,
|
|
29413
|
+
excludeLabels: excludeParts.exact,
|
|
29414
|
+
excludePatterns: excludeParts.regex,
|
|
28244
29415
|
jobConfig,
|
|
28245
29416
|
repoUrl: dispatchBundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
28246
29417
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -28845,6 +30016,36 @@ var init_process_webhook = __esmMin((() => {
|
|
|
28845
30016
|
* Decision traces are forwarded to Platform via platformClient.send() (which buffers
|
|
28846
30017
|
* internally when disconnected -- the caller does NOT check connection state).
|
|
28847
30018
|
*/
|
|
30019
|
+
function evalGateKey(runId, evalJobName) {
|
|
30020
|
+
return `${runId}:${evalJobName}`;
|
|
30021
|
+
}
|
|
30022
|
+
/**
|
|
30023
|
+
* Register an eval gate and return a promise that resolves when the scheduler
|
|
30024
|
+
* opens it (the eval job's upstream needs are all satisfied).
|
|
30025
|
+
*/
|
|
30026
|
+
function trackEvalGate(runId, evalJobName) {
|
|
30027
|
+
return new Promise((resolve) => {
|
|
30028
|
+
pendingEvalGates.set(evalGateKey(runId, evalJobName), resolve);
|
|
30029
|
+
});
|
|
30030
|
+
}
|
|
30031
|
+
/**
|
|
30032
|
+
* Open a registered eval gate, unblocking the deferred dispatch task. Returns
|
|
30033
|
+
* true if a gate was registered for this eval job (so the scheduler knows it
|
|
30034
|
+
* handled the ready signal itself and must not run the normal dispatch path).
|
|
30035
|
+
*/
|
|
30036
|
+
function openEvalGate(runId, evalJobName) {
|
|
30037
|
+
const key = evalGateKey(runId, evalJobName);
|
|
30038
|
+
const resolve = pendingEvalGates.get(key);
|
|
30039
|
+
if (!resolve) return false;
|
|
30040
|
+
pendingEvalGates.delete(key);
|
|
30041
|
+
resolve();
|
|
30042
|
+
return true;
|
|
30043
|
+
}
|
|
30044
|
+
/** Clear all eval gates for a run (called on run completion / cleanup). */
|
|
30045
|
+
function clearEvalGatesForRun(runId) {
|
|
30046
|
+
const prefix = `${runId}:`;
|
|
30047
|
+
for (const key of pendingEvalGates.keys()) if (key.startsWith(prefix)) pendingEvalGates.delete(key);
|
|
30048
|
+
}
|
|
28848
30049
|
/**
|
|
28849
30050
|
* Store a pending dispatch context for a job that will be dispatched later
|
|
28850
30051
|
* by the needs scheduler. The key is `${runId}:${jobName}`.
|
|
@@ -29294,11 +30495,12 @@ function summarizeDecision(decision) {
|
|
|
29294
30495
|
checksCount: decision.checks.length
|
|
29295
30496
|
};
|
|
29296
30497
|
}
|
|
29297
|
-
var logger$36, pendingJobContexts;
|
|
30498
|
+
var logger$36, pendingJobContexts, pendingEvalGates;
|
|
29298
30499
|
var init_processor = __esmMin((() => {
|
|
29299
30500
|
init_process_webhook();
|
|
29300
30501
|
logger$36 = createLogger({ prefix: "pipeline" });
|
|
29301
30502
|
pendingJobContexts = /* @__PURE__ */ new Map();
|
|
30503
|
+
pendingEvalGates = /* @__PURE__ */ new Map();
|
|
29302
30504
|
}));
|
|
29303
30505
|
//#endregion
|
|
29304
30506
|
//#region src/concurrency/waiters.ts
|
|
@@ -29712,6 +30914,7 @@ var init_admin_event_dlq = __esmMin((() => {
|
|
|
29712
30914
|
* @returns Object with Hono app instance and injectWebSocket function
|
|
29713
30915
|
*/
|
|
29714
30916
|
function createApp(deps) {
|
|
30917
|
+
registerOrchestratorMetrics();
|
|
29715
30918
|
const app = new Hono().basePath(deps.config.basePath);
|
|
29716
30919
|
const { injectWebSocket, upgradeWebSocket, wss } = createNodeWebSocket({ app });
|
|
29717
30920
|
configureSecureWsServer(wss);
|
|
@@ -30224,7 +31427,11 @@ function createApp(deps) {
|
|
|
30224
31427
|
eventLog: deps.eventLogWriter,
|
|
30225
31428
|
eventLogSource: "direct",
|
|
30226
31429
|
contributorCache: deps.contributorCache,
|
|
30227
|
-
accessLogWriter: deps.accessLogWriter
|
|
31430
|
+
accessLogWriter: deps.accessLogWriter,
|
|
31431
|
+
hostRosterStore: deps.hostRosterStore,
|
|
31432
|
+
instanceId: deps.config.instanceId,
|
|
31433
|
+
rosterGraceMs: deps.config.rosterGraceMs,
|
|
31434
|
+
maxFanoutHosts: deps.config.maxFanoutHosts
|
|
30228
31435
|
});
|
|
30229
31436
|
} catch (err) {
|
|
30230
31437
|
if (deps.eventLogWriter) try {
|
|
@@ -30556,7 +31763,7 @@ async function createDebugBundle(options) {
|
|
|
30556
31763
|
const dir = path$1.dirname(outputPath);
|
|
30557
31764
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
30558
31765
|
const output = fs.createWriteStream(outputPath);
|
|
30559
|
-
const archive =
|
|
31766
|
+
const archive = new ZipArchive({ zlib: { level: 6 } });
|
|
30560
31767
|
const finalized = new Promise((resolve, reject) => {
|
|
30561
31768
|
output.on("close", resolve);
|
|
30562
31769
|
archive.on("error", reject);
|
|
@@ -30632,7 +31839,7 @@ function classifyError(err) {
|
|
|
30632
31839
|
};
|
|
30633
31840
|
}
|
|
30634
31841
|
async function collectFleetSubtree(opts, deps) {
|
|
30635
|
-
const archive =
|
|
31842
|
+
const archive = new ZipArchive({ zlib: { level: 6 } });
|
|
30636
31843
|
const out = [];
|
|
30637
31844
|
archive.on("data", (d) => out.push(d));
|
|
30638
31845
|
const done = new Promise((res, rej) => {
|
|
@@ -37028,6 +38235,61 @@ var init_log_storage = __esmMin((() => {
|
|
|
37028
38235
|
init_s3_log_storage();
|
|
37029
38236
|
}));
|
|
37030
38237
|
//#endregion
|
|
38238
|
+
//#region src/pipeline/wave-scheduler.ts
|
|
38239
|
+
/**
|
|
38240
|
+
* Decide what happens after a fan-out child of `baseJobName` reaches terminal.
|
|
38241
|
+
*
|
|
38242
|
+
* The wave policy (`maxParallel` / `failFast`) is read from the base group's
|
|
38243
|
+
* own rows — every child of a bounded wave carries the same stamped
|
|
38244
|
+
* `wave_max_parallel` / `wave_fail_fast`, so the just-completed child's slot
|
|
38245
|
+
* being re-inserted without the policy on release does not break the chain
|
|
38246
|
+
* (the still-held siblings carry it). If no sibling carries a policy, this is
|
|
38247
|
+
* not a bounded wave → `noop`.
|
|
38248
|
+
*
|
|
38249
|
+
* - `failFast` + a child failure → `skip-remaining` every still-held sibling.
|
|
38250
|
+
* - in-flight count `< maxParallel` AND a held sibling exists → `release` the
|
|
38251
|
+
* next held sibling (lowest `variant_label`).
|
|
38252
|
+
* - otherwise → `noop`.
|
|
38253
|
+
*
|
|
38254
|
+
* "In-flight" = a non-terminal, non-`wave_gated` child (it has been dispatched
|
|
38255
|
+
* and not yet completed). The just-completed child is terminal, so it does not
|
|
38256
|
+
* count against the window — its slot is the one we are filling.
|
|
38257
|
+
*/
|
|
38258
|
+
async function evaluateWave(db, evaluation) {
|
|
38259
|
+
const { runId, baseJobName, completedStatus } = evaluation;
|
|
38260
|
+
const children = await db.selectFrom("execution_jobs").select([
|
|
38261
|
+
"job_name",
|
|
38262
|
+
"status",
|
|
38263
|
+
"wave_gated",
|
|
38264
|
+
"variant_label",
|
|
38265
|
+
"wave_max_parallel",
|
|
38266
|
+
"wave_fail_fast"
|
|
38267
|
+
]).where("run_id", "=", runId).where("base_job_name", "=", baseJobName).execute();
|
|
38268
|
+
const policyRow = children.find((c) => c.wave_max_parallel != null);
|
|
38269
|
+
if (!policyRow || policyRow.wave_max_parallel == null) return { action: "noop" };
|
|
38270
|
+
const maxParallel = policyRow.wave_max_parallel;
|
|
38271
|
+
const failFast = policyRow.wave_fail_fast ?? false;
|
|
38272
|
+
const heldSiblings = children.filter((c) => c.wave_gated).sort((a, b) => (a.variant_label ?? a.job_name).localeCompare(b.variant_label ?? b.job_name));
|
|
38273
|
+
const isFailure = completedStatus !== ExecutionJobStatus.enum.success;
|
|
38274
|
+
if (failFast && isFailure) {
|
|
38275
|
+
if (heldSiblings.length === 0) return { action: "noop" };
|
|
38276
|
+
return {
|
|
38277
|
+
action: "skip-remaining",
|
|
38278
|
+
jobNames: heldSiblings.map((c) => c.job_name)
|
|
38279
|
+
};
|
|
38280
|
+
}
|
|
38281
|
+
if (heldSiblings.length === 0) return { action: "noop" };
|
|
38282
|
+
if (children.filter((c) => !c.wave_gated && !TERMINAL_JOB_STATES.has(c.status)).length >= maxParallel) return { action: "noop" };
|
|
38283
|
+
return {
|
|
38284
|
+
action: "release",
|
|
38285
|
+
jobName: heldSiblings[0].job_name,
|
|
38286
|
+
baseJobName,
|
|
38287
|
+
maxParallel,
|
|
38288
|
+
failFast
|
|
38289
|
+
};
|
|
38290
|
+
}
|
|
38291
|
+
var init_wave_scheduler = __esmMin((() => {}));
|
|
38292
|
+
//#endregion
|
|
37031
38293
|
//#region src/reporting/execution-tracker.ts
|
|
37032
38294
|
/**
|
|
37033
38295
|
* Execution state tracker with write-through DB persistence.
|
|
@@ -37046,6 +38308,7 @@ var logger$24, PRUNE_DELAY_MS, ExecutionTracker;
|
|
|
37046
38308
|
var init_execution_tracker = __esmMin((() => {
|
|
37047
38309
|
init_prometheus();
|
|
37048
38310
|
init_needs_scheduler();
|
|
38311
|
+
init_wave_scheduler();
|
|
37049
38312
|
logger$24 = createLogger({ prefix: "execution-tracker" });
|
|
37050
38313
|
PRUNE_DELAY_MS = 300 * 1e3;
|
|
37051
38314
|
ExecutionTracker = class {
|
|
@@ -37177,10 +38440,22 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37177
38440
|
job_name: job.jobName,
|
|
37178
38441
|
routing_key: routingKey ?? null,
|
|
37179
38442
|
matrix_values: job.matrixValues ? JSON.stringify(job.matrixValues) : null,
|
|
38443
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38444
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38445
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
38446
|
+
...job.waveGated && { wave_gated: true },
|
|
38447
|
+
...job.waveMaxParallel !== void 0 && { wave_max_parallel: job.waveMaxParallel },
|
|
38448
|
+
...job.waveFailFast !== void 0 && { wave_fail_fast: job.waveFailFast },
|
|
37180
38449
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37181
38450
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37182
38451
|
}).onConflict((oc) => oc.columns(["run_id", "job_id"]).doUpdateSet({
|
|
37183
38452
|
job_name: job.jobName,
|
|
38453
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38454
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38455
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
38456
|
+
...job.waveGated && { wave_gated: true },
|
|
38457
|
+
...job.waveMaxParallel !== void 0 && { wave_max_parallel: job.waveMaxParallel },
|
|
38458
|
+
...job.waveFailFast !== void 0 && { wave_fail_fast: job.waveFailFast },
|
|
37184
38459
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37185
38460
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37186
38461
|
})).execute();
|
|
@@ -37235,6 +38510,20 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37235
38510
|
return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
|
|
37236
38511
|
}
|
|
37237
38512
|
/**
|
|
38513
|
+
* Find the synthetic deferred-eval placeholder job ID for a result-aware
|
|
38514
|
+
* dynamic generator's eval job. Mirrors {@link findSyntheticJobId} but keys on
|
|
38515
|
+
* the `dynamic-eval-pending-<evalJobName>-` prefix that registerDeferredEvalJob
|
|
38516
|
+
* uses, so dispatchEvalJob can swap it for the real eval job id.
|
|
38517
|
+
*/
|
|
38518
|
+
async findDynamicEvalSyntheticId(runId, evalJobName) {
|
|
38519
|
+
const run = this.runs.get(runId);
|
|
38520
|
+
const prefix = `dynamic-eval-pending-${evalJobName}-`;
|
|
38521
|
+
if (run) {
|
|
38522
|
+
for (const key of run.jobs.keys()) if (key.startsWith(prefix)) return key;
|
|
38523
|
+
}
|
|
38524
|
+
return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", evalJobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
|
|
38525
|
+
}
|
|
38526
|
+
/**
|
|
37238
38527
|
* Run `fn` while holding a per-run lock, serializing the run-mutating methods
|
|
37239
38528
|
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|
|
37240
38529
|
* synthetic→real job swap and wedge the run in `running`.
|
|
@@ -37317,10 +38606,16 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37317
38606
|
job_name: job.jobName,
|
|
37318
38607
|
routing_key: run.routingKey ?? null,
|
|
37319
38608
|
matrix_values: job.matrixValues ? JSON.stringify(job.matrixValues) : null,
|
|
38609
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38610
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38611
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
37320
38612
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37321
38613
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37322
38614
|
}).onConflict((oc) => oc.columns(["run_id", "job_id"]).doUpdateSet({
|
|
37323
38615
|
job_name: job.jobName,
|
|
38616
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38617
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38618
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
37324
38619
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37325
38620
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37326
38621
|
})).execute();
|
|
@@ -37421,7 +38716,10 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37421
38716
|
workflowName: run.workflowName
|
|
37422
38717
|
});
|
|
37423
38718
|
if (TERMINAL_JOB_STATES.has(state) && data?.droppedJobs && Array.isArray(data.droppedJobs)) await this.handleDriftDroppedJobs(runId, data.droppedJobs);
|
|
37424
|
-
if (TERMINAL_JOB_STATES.has(state) && job)
|
|
38719
|
+
if (TERMINAL_JOB_STATES.has(state) && job) {
|
|
38720
|
+
await this.runSchedulerHook(runId, jobId, job.name, state);
|
|
38721
|
+
await this.runWaveSchedulerHook(runId, jobId, state);
|
|
38722
|
+
}
|
|
37425
38723
|
if (TERMINAL_JOB_STATES.has(state) && run && this.isRunComplete(runId)) {
|
|
37426
38724
|
if (await this.enforceSchedulerInvariantOrFail(runId)) return;
|
|
37427
38725
|
}
|
|
@@ -37725,6 +39023,60 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37725
39023
|
}
|
|
37726
39024
|
}
|
|
37727
39025
|
/**
|
|
39026
|
+
* Rolling-wave hook: fires beside the needs-scheduler when a fan-out child of
|
|
39027
|
+
* a bounded wave (`maxParallel` set) reaches terminal. Reads the completed
|
|
39028
|
+
* child's row to recover the base + wave policy, asks {@link evaluateWave}
|
|
39029
|
+
* what to do, then performs it:
|
|
39030
|
+
*
|
|
39031
|
+
* - `release`: clear the next held sibling's `wave_gated` flag and fire the
|
|
39032
|
+
* onJobReady callback (the existing ready→dispatch path).
|
|
39033
|
+
* - `skip-remaining`: mark every still-held sibling `skipped` (failFast).
|
|
39034
|
+
* - `noop`: nothing — a later terminal will free the next slot.
|
|
39035
|
+
*/
|
|
39036
|
+
async runWaveSchedulerHook(runId, jobId, state) {
|
|
39037
|
+
try {
|
|
39038
|
+
const row = await this.db.selectFrom("execution_jobs").select(["base_job_name"]).where("run_id", "=", runId).where("job_id", "=", jobId).executeTakeFirst();
|
|
39039
|
+
if (!row?.base_job_name) return;
|
|
39040
|
+
const result = await evaluateWave(this.db, {
|
|
39041
|
+
runId,
|
|
39042
|
+
baseJobName: row.base_job_name,
|
|
39043
|
+
completedStatus: state
|
|
39044
|
+
});
|
|
39045
|
+
if (result.action === "release") {
|
|
39046
|
+
await this.db.updateTable("execution_jobs").set({ wave_gated: false }).where("run_id", "=", runId).where("job_name", "=", result.jobName).execute();
|
|
39047
|
+
if (this.onJobReadyCallback) await this.onJobReadyCallback(runId, result.jobName);
|
|
39048
|
+
await this.db.updateTable("execution_jobs").set({
|
|
39049
|
+
base_job_name: result.baseJobName,
|
|
39050
|
+
wave_max_parallel: result.maxParallel,
|
|
39051
|
+
wave_fail_fast: result.failFast
|
|
39052
|
+
}).where("run_id", "=", runId).where("job_name", "=", result.jobName).execute();
|
|
39053
|
+
logger$24.info("Rolling wave released next child", {
|
|
39054
|
+
runId,
|
|
39055
|
+
baseJobName: row.base_job_name,
|
|
39056
|
+
released: result.jobName
|
|
39057
|
+
});
|
|
39058
|
+
} else if (result.action === "skip-remaining") {
|
|
39059
|
+
logger$24.info("Rolling wave halting (failFast): skipping held remainder", {
|
|
39060
|
+
runId,
|
|
39061
|
+
baseJobName: row.base_job_name,
|
|
39062
|
+
skipped: result.jobNames
|
|
39063
|
+
});
|
|
39064
|
+
for (const jobName of result.jobNames) {
|
|
39065
|
+
const heldRow = await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).executeTakeFirst();
|
|
39066
|
+
if (!heldRow) continue;
|
|
39067
|
+
await this.db.updateTable("execution_jobs").set({ wave_gated: false }).where("run_id", "=", runId).where("job_id", "=", heldRow.job_id).execute();
|
|
39068
|
+
await this.onJobStatus(runId, heldRow.job_id, ExecutionJobStatus.enum.skipped, Date.now(), void 0, { error: "fan-out halted by failFast" });
|
|
39069
|
+
}
|
|
39070
|
+
}
|
|
39071
|
+
} catch (e) {
|
|
39072
|
+
logger$24.error("Wave scheduler hook failed", {
|
|
39073
|
+
runId,
|
|
39074
|
+
jobId,
|
|
39075
|
+
error: e
|
|
39076
|
+
});
|
|
39077
|
+
}
|
|
39078
|
+
}
|
|
39079
|
+
/**
|
|
37728
39080
|
* Phase 9: stuck-jobs invariant check ( Layer 3).
|
|
37729
39081
|
* Before declaring a run complete, verify no stuck jobs exist. If any are
|
|
37730
39082
|
* found, fail them via recursive onJobStatus calls and signal the caller to
|
|
@@ -40691,6 +42043,8 @@ var init_pg_secret_store = __esmMin((() => {
|
|
|
40691
42043
|
async renameScope(orgId, oldScope, newScope) {
|
|
40692
42044
|
await this.db.transaction().execute(async (trx) => {
|
|
40693
42045
|
const rows = await trx.selectFrom("scoped_secrets").selectAll().where("org_id", "=", orgId).where("scope", "=", oldScope).execute();
|
|
42046
|
+
const bindings = await trx.selectFrom("environment_bindings").select("id").where("org_id", "=", orgId).where("scope_pattern", "=", oldScope).execute();
|
|
42047
|
+
if (rows.length === 0 && bindings.length === 0) throw new Error(`Secret scope '${oldScope}' not found`);
|
|
40694
42048
|
for (const row of rows) {
|
|
40695
42049
|
const oldAad = `${orgId}:${oldScope}:${row.key}`;
|
|
40696
42050
|
const newAad = `${orgId}:${newScope}:${row.key}`;
|
|
@@ -44723,9 +46077,12 @@ var init_disk_guard = __esmMin((() => {
|
|
|
44723
46077
|
*/
|
|
44724
46078
|
var orchestrator_core_exports = /* @__PURE__ */ __exportAll({
|
|
44725
46079
|
bootstrapOrchestrator: () => bootstrapOrchestrator$1,
|
|
46080
|
+
buildHostOutputsEnvelope: () => buildHostOutputsEnvelope,
|
|
44726
46081
|
buildMatrixOutputsEnvelope: () => buildMatrixOutputsEnvelope,
|
|
44727
46082
|
buildUpstreamOutputsByBase: () => buildUpstreamOutputsByBase,
|
|
46083
|
+
internalJobRunsOnSelectors: () => internalJobRunsOnSelectors,
|
|
44728
46084
|
mergeUpstreamOutputs: () => mergeUpstreamOutputs,
|
|
46085
|
+
parseOutputsCell: () => parseOutputsCell,
|
|
44729
46086
|
upstreamBaseNamesFromNeeds: () => upstreamBaseNamesFromNeeds
|
|
44730
46087
|
});
|
|
44731
46088
|
async function initializeScaler(config, db, tokenStore, onScalerEvent) {
|
|
@@ -45054,6 +46411,23 @@ function upstreamBaseNamesFromNeeds(needs) {
|
|
|
45054
46411
|
}
|
|
45055
46412
|
return names;
|
|
45056
46413
|
}
|
|
46414
|
+
/**
|
|
46415
|
+
* Partition a lock job's `runsOn` / `excludeLabels` matchers into exact label
|
|
46416
|
+
* strings and regex patterns for internal-event (cron / `ctx.emit`) dispatch.
|
|
46417
|
+
* Lock jobs carry `runsOn` as `LabelMatcher[]`; the coordinator routing and the
|
|
46418
|
+
* direct dispatcher both need exact labels for the indexed/SQL fast path and
|
|
46419
|
+
* regex patterns as a separate JS post-filter — never the raw matcher objects.
|
|
46420
|
+
*/
|
|
46421
|
+
function internalJobRunsOnSelectors(job) {
|
|
46422
|
+
const include = partitionMatchers(job.runsOn ?? []);
|
|
46423
|
+
const exclude = partitionMatchers(job.excludeLabels ?? []);
|
|
46424
|
+
return {
|
|
46425
|
+
runsOnLabels: include.exact,
|
|
46426
|
+
runsOnPatterns: include.regex,
|
|
46427
|
+
excludeLabels: exclude.exact,
|
|
46428
|
+
excludePatterns: exclude.regex
|
|
46429
|
+
};
|
|
46430
|
+
}
|
|
45057
46431
|
/** Escape SQL LIKE wildcards (`%`, `_`) so a literal base name matches exactly. */
|
|
45058
46432
|
function escapeLikePattern(value) {
|
|
45059
46433
|
return value.replace(/[\\%_]/g, (c) => `\\${c}`);
|
|
@@ -45081,6 +46455,16 @@ function parseOutputsCell(outputs) {
|
|
|
45081
46455
|
function buildUpstreamOutputsByBase(baseNames, rows) {
|
|
45082
46456
|
let result;
|
|
45083
46457
|
for (const base of baseNames) {
|
|
46458
|
+
const hostChildren = rows.filter((r) => r.variant_kind === VariantKind.host && r.job_name.startsWith(`${base} (`)).map((r) => ({
|
|
46459
|
+
host: r.variant_label ?? r.job_name.slice(base.length + 2, -1),
|
|
46460
|
+
status: r.status ?? null,
|
|
46461
|
+
parsed: parseOutputsCell(r.outputs) ?? {}
|
|
46462
|
+
}));
|
|
46463
|
+
if (hostChildren.length > 0) {
|
|
46464
|
+
if (!result) result = {};
|
|
46465
|
+
result[base] = buildHostOutputsEnvelope(hostChildren);
|
|
46466
|
+
continue;
|
|
46467
|
+
}
|
|
45084
46468
|
const exact = rows.find((r) => r.job_name === base && !r.matrix_values);
|
|
45085
46469
|
const children = rows.filter((r) => r.matrix_values && r.job_name.startsWith(`${base} (`)).map((r) => ({
|
|
45086
46470
|
job_name: r.job_name,
|
|
@@ -45100,6 +46484,34 @@ function buildUpstreamOutputsByBase(baseNames, rows) {
|
|
|
45100
46484
|
return result;
|
|
45101
46485
|
}
|
|
45102
46486
|
/**
|
|
46487
|
+
* Fold a `runsOnAll` upstream's host children into the `byHost` envelope
|
|
46488
|
+
* `{ byHost: { '<host>': outputs }, summary: { succeededHosts, failedHosts, outputs } }`.
|
|
46489
|
+
* Unlike the matrix envelope, `summary.outputs[key]` is an array view across hosts
|
|
46490
|
+
* (host order), never a last-write-wins scalar; `succeededHosts`/`failedHosts`
|
|
46491
|
+
* record each host's terminal outcome.
|
|
46492
|
+
*/
|
|
46493
|
+
function buildHostOutputsEnvelope(children) {
|
|
46494
|
+
const byHost = {};
|
|
46495
|
+
const succeededHosts = [];
|
|
46496
|
+
const failedHosts = [];
|
|
46497
|
+
const outputs = {};
|
|
46498
|
+
const ordered = [...children].sort((a, b) => a.host.localeCompare(b.host));
|
|
46499
|
+
for (const child of ordered) {
|
|
46500
|
+
byHost[child.host] = child.parsed;
|
|
46501
|
+
if (child.status === ExecutionJobStatus.enum.success) succeededHosts.push(child.host);
|
|
46502
|
+
else if (child.status === ExecutionJobStatus.enum.failed) failedHosts.push(child.host);
|
|
46503
|
+
for (const [key, value] of Object.entries(child.parsed)) (outputs[key] ??= []).push(value);
|
|
46504
|
+
}
|
|
46505
|
+
return {
|
|
46506
|
+
byHost,
|
|
46507
|
+
summary: {
|
|
46508
|
+
succeededHosts,
|
|
46509
|
+
failedHosts,
|
|
46510
|
+
outputs
|
|
46511
|
+
}
|
|
46512
|
+
};
|
|
46513
|
+
}
|
|
46514
|
+
/**
|
|
45103
46515
|
* Group an upstream's child rows into the matrix outputs envelope
|
|
45104
46516
|
* `{ byMatrix: { '<suffix>': outputs }, merged: <last-write-wins> }`. The suffix
|
|
45105
46517
|
* is the text inside the `(...)` of each expanded child name; children are
|
|
@@ -45150,7 +46562,10 @@ async function mergeUpstreamOutputs(db, runId, jobName, needs, dispatchSecrets,
|
|
|
45150
46562
|
"job_id",
|
|
45151
46563
|
"job_name",
|
|
45152
46564
|
"outputs",
|
|
45153
|
-
"matrix_values"
|
|
46565
|
+
"matrix_values",
|
|
46566
|
+
"variant_kind",
|
|
46567
|
+
"variant_label",
|
|
46568
|
+
"status"
|
|
45154
46569
|
]).where("run_id", "=", runId);
|
|
45155
46570
|
query = query.where((eb) => eb.or(baseNames.flatMap((base) => [eb("job_name", "=", base), eb("job_name", "like", `${escapeLikePattern(base)} (%`)])));
|
|
45156
46571
|
const upstreamJobs = await query.execute();
|
|
@@ -45365,15 +46780,21 @@ function buildOnSecretOutputs(config, db) {
|
|
|
45365
46780
|
*/
|
|
45366
46781
|
async function routeInternalJobsViaCoordinator(coordinator, dispatcher, runId, workflow, staticJobs, ctx, buildInternalJobConfig) {
|
|
45367
46782
|
const dispatchedJobs = [];
|
|
45368
|
-
const jobsToRoute = staticJobs.map((job) =>
|
|
45369
|
-
|
|
45370
|
-
|
|
45371
|
-
|
|
45372
|
-
|
|
45373
|
-
|
|
45374
|
-
|
|
45375
|
-
|
|
45376
|
-
|
|
46783
|
+
const jobsToRoute = staticJobs.map((job) => {
|
|
46784
|
+
const sel = internalJobRunsOnSelectors(job);
|
|
46785
|
+
return {
|
|
46786
|
+
jobName: job.name,
|
|
46787
|
+
runsOnLabels: [sel.runsOnLabels],
|
|
46788
|
+
runsOnPatterns: sel.runsOnPatterns,
|
|
46789
|
+
excludeLabels: sel.excludeLabels,
|
|
46790
|
+
excludePatterns: sel.excludePatterns,
|
|
46791
|
+
jobConfig: buildInternalJobConfig(job),
|
|
46792
|
+
repoUrl: ctx.repoUrl,
|
|
46793
|
+
ref: "",
|
|
46794
|
+
sha: ctx.cronCommitSha,
|
|
46795
|
+
...job.resources && { resources: job.resources }
|
|
46796
|
+
};
|
|
46797
|
+
});
|
|
45377
46798
|
const runCtx = {
|
|
45378
46799
|
runId,
|
|
45379
46800
|
deliveryId: ctx.event.id,
|
|
@@ -45425,11 +46846,15 @@ async function dispatchInternalJobsDirect(dispatcher, runId, workflow, staticJob
|
|
|
45425
46846
|
const dispatchedJobs = [];
|
|
45426
46847
|
const buildJobConfig = (job) => buildInternalJobConfigForWorkflow(workflow, job);
|
|
45427
46848
|
for (const job of staticJobs) {
|
|
46849
|
+
const sel = internalJobRunsOnSelectors(job);
|
|
45428
46850
|
const result = await dispatcher.dispatch({
|
|
45429
46851
|
runId,
|
|
45430
46852
|
workflowName: workflow.name,
|
|
45431
46853
|
jobName: job.name,
|
|
45432
|
-
runsOnLabels:
|
|
46854
|
+
runsOnLabels: sel.runsOnLabels,
|
|
46855
|
+
runsOnPatterns: sel.runsOnPatterns,
|
|
46856
|
+
excludeLabels: sel.excludeLabels,
|
|
46857
|
+
excludePatterns: sel.excludePatterns,
|
|
45433
46858
|
jobConfig: buildJobConfig(job),
|
|
45434
46859
|
repoUrl: ctx.repoUrl,
|
|
45435
46860
|
ref: "",
|
|
@@ -45858,7 +47283,11 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
45858
47283
|
const tokenStore = new AgentTokenStore(db);
|
|
45859
47284
|
if (config.agentAuth === "none") logger$3.warn("Agent authentication disabled (KICI_AGENT_AUTH=none). All agents will be accepted without tokens.");
|
|
45860
47285
|
else logger$3.info("Agent authentication enabled (token mode)");
|
|
45861
|
-
const
|
|
47286
|
+
const hostRosterStore = new HostRosterStore(db);
|
|
47287
|
+
const agentRegistry = new AgentRegistry({
|
|
47288
|
+
rosterStore: hostRosterStore,
|
|
47289
|
+
instanceId: config.instanceId
|
|
47290
|
+
});
|
|
45862
47291
|
const fleetAgentCollector = new FleetAgentCollector({ timeoutMs: FLEET_NODE_TIMEOUT_MS });
|
|
45863
47292
|
const queue = new JobQueue(db, {
|
|
45864
47293
|
maxDepth: config.queueMaxDepth,
|
|
@@ -45918,6 +47347,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
45918
47347
|
error: toErrorMessage(err)
|
|
45919
47348
|
});
|
|
45920
47349
|
});
|
|
47350
|
+
clearEvalGatesForRun(runId);
|
|
45921
47351
|
const [owner, repo] = context.repoIdentifier.split("/");
|
|
45922
47352
|
checkRunReporter.updateWorkflowStatus({
|
|
45923
47353
|
provider: context.provider,
|
|
@@ -46005,6 +47435,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46005
47435
|
});
|
|
46006
47436
|
executionTrackerRef = executionTracker;
|
|
46007
47437
|
executionTracker.setOnJobReadyCallback(async (runId, jobName) => {
|
|
47438
|
+
if (openEvalGate(runId, jobName)) return;
|
|
46008
47439
|
await dispatchReadyJob(runId, jobName, dispatcher, executionTracker, cluster.coordinator, db);
|
|
46009
47440
|
});
|
|
46010
47441
|
logger$3.info("Execution reporting initialized", { logStorageType: config.storage?.type ?? "filesystem" });
|
|
@@ -46165,8 +47596,21 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46165
47596
|
eventStore,
|
|
46166
47597
|
config: eventRouterConfig
|
|
46167
47598
|
});
|
|
46168
|
-
|
|
46169
|
-
|
|
47599
|
+
const hostRosterReaper = new HostRosterReaper({
|
|
47600
|
+
store: hostRosterStore,
|
|
47601
|
+
ttlMs: config.rosterTtlMs,
|
|
47602
|
+
graceMs: config.rosterGraceMs,
|
|
47603
|
+
scanIntervalMs: 6e4,
|
|
47604
|
+
setUnreachableGauge: setDeclaredHostsUnreachable
|
|
47605
|
+
});
|
|
47606
|
+
eventRetryScannerRef.onBecomeLeader = () => {
|
|
47607
|
+
eventRetryScanner.onBecomeLeader();
|
|
47608
|
+
hostRosterReaper.onBecomeLeader();
|
|
47609
|
+
};
|
|
47610
|
+
eventRetryScannerRef.onLoseLeadership = () => {
|
|
47611
|
+
eventRetryScanner.onLoseLeadership();
|
|
47612
|
+
hostRosterReaper.onLoseLeadership();
|
|
47613
|
+
};
|
|
46170
47614
|
const cronStore = new CronStore(db);
|
|
46171
47615
|
const cronScheduler = new CronScheduler({
|
|
46172
47616
|
db,
|
|
@@ -46275,6 +47719,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46275
47719
|
pool,
|
|
46276
47720
|
providerRegistry,
|
|
46277
47721
|
agentRegistry,
|
|
47722
|
+
hostRosterStore,
|
|
46278
47723
|
dispatcher,
|
|
46279
47724
|
queue,
|
|
46280
47725
|
scalerManager,
|
|
@@ -46485,6 +47930,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46485
47930
|
db,
|
|
46486
47931
|
pool,
|
|
46487
47932
|
registry: agentRegistry,
|
|
47933
|
+
hostRosterStore,
|
|
46488
47934
|
dispatcher,
|
|
46489
47935
|
jobQueue: queue,
|
|
46490
47936
|
dedup,
|
|
@@ -46578,13 +48024,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46578
48024
|
scanIntervalMs: config.staleDetectorScanIntervalMs,
|
|
46579
48025
|
heldRunStore: modeResult.appDepsExtras?.heldRunStore,
|
|
46580
48026
|
stepApprovalBridge: modeResult.appDepsExtras?.stepApprovalBridge,
|
|
46581
|
-
failRun: (runId, reason) =>
|
|
46582
|
-
db,
|
|
46583
|
-
jobQueue: queue,
|
|
46584
|
-
dispatcher,
|
|
46585
|
-
registry: agentRegistry,
|
|
46586
|
-
executionTracker
|
|
46587
|
-
}, runId, reason).then(() => void 0),
|
|
48027
|
+
failRun: (runId, reason) => executionTracker.failRun(runId, reason).then(() => void 0),
|
|
46588
48028
|
onWorkflowRelease: modeResult.appDepsExtras?.onWorkflowRelease,
|
|
46589
48029
|
accessLogWriter
|
|
46590
48030
|
});
|
|
@@ -46755,6 +48195,10 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46755
48195
|
name: "Stopping event retry scanner",
|
|
46756
48196
|
fn: () => eventRetryScanner.stop()
|
|
46757
48197
|
},
|
|
48198
|
+
{
|
|
48199
|
+
name: "Stopping host roster reaper",
|
|
48200
|
+
fn: () => hostRosterReaper.stop()
|
|
48201
|
+
},
|
|
46758
48202
|
{
|
|
46759
48203
|
name: "Stopping timers, cleanup, and reloader",
|
|
46760
48204
|
fn: () => {
|
|
@@ -46794,6 +48238,8 @@ var init_orchestrator_core = __esmMin((() => {
|
|
|
46794
48238
|
init_resolver();
|
|
46795
48239
|
init_client();
|
|
46796
48240
|
init_registry();
|
|
48241
|
+
init_host_roster();
|
|
48242
|
+
init_host_roster_reaper();
|
|
46797
48243
|
init_job_queue();
|
|
46798
48244
|
init_cleanup$1();
|
|
46799
48245
|
init_bootstrap();
|
|
@@ -47003,6 +48449,9 @@ function matchesGate(job, agentLabels, agentMandatoryLabels) {
|
|
|
47003
48449
|
const runsOnSet = new Set(job.runsOnLabels);
|
|
47004
48450
|
if (!agentMandatoryLabels.every((m) => runsOnSet.has(m))) return false;
|
|
47005
48451
|
}
|
|
48452
|
+
const labelSet = new Set(agentLabels);
|
|
48453
|
+
if (!job.runsOnPatterns.every((p) => matcherSatisfiedBy(p, labelSet))) return false;
|
|
48454
|
+
if (job.excludePatterns.some((p) => matcherSatisfiedBy(p, labelSet))) return false;
|
|
47006
48455
|
return true;
|
|
47007
48456
|
}
|
|
47008
48457
|
var InMemoryJobQueue;
|
|
@@ -47039,6 +48488,8 @@ var init_in_memory_job_queue = __esmMin((() => {
|
|
|
47039
48488
|
depsHash: input.depsHash,
|
|
47040
48489
|
requestId: input.requestId,
|
|
47041
48490
|
excludeLabels: input.excludeLabels ?? [],
|
|
48491
|
+
runsOnPatterns: input.runsOnPatterns ?? [],
|
|
48492
|
+
excludePatterns: input.excludePatterns ?? [],
|
|
47042
48493
|
routingKey: input.routingKey
|
|
47043
48494
|
});
|
|
47044
48495
|
return id;
|
|
@@ -47786,6 +49237,8 @@ async function bootstrapWorker(config, _opts) {
|
|
|
47786
49237
|
jobName: msg.jobName,
|
|
47787
49238
|
runsOnLabels: flatLabels,
|
|
47788
49239
|
excludeLabels: msg.excludeLabels,
|
|
49240
|
+
runsOnPatterns: msg.runsOnPatterns,
|
|
49241
|
+
excludePatterns: msg.excludePatterns,
|
|
47789
49242
|
jobConfig,
|
|
47790
49243
|
repoUrl: msg.repoUrl ?? "",
|
|
47791
49244
|
ref: msg.ref ?? "",
|
|
@@ -48079,14 +49532,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
48079
49532
|
init_fleet_wiring();
|
|
48080
49533
|
init_worker_status();
|
|
48081
49534
|
init_agent_heartbeat();
|
|
48082
|
-
ORCHESTRATOR_VERSION$1 = "0.1.
|
|
48083
|
-
WORKER_BUILD_COMMIT = "
|
|
48084
|
-
WORKER_SDK_VERSION = "0.1.
|
|
48085
|
-
WORKER_SDK_BUNDLE_HASH = "
|
|
48086
|
-
WORKER_SHARED_VERSION = "0.1.
|
|
48087
|
-
WORKER_SHARED_BUNDLE_HASH = "
|
|
48088
|
-
WORKER_ENGINE_VERSION = "0.1.
|
|
48089
|
-
WORKER_ENGINE_BUNDLE_HASH = "
|
|
49535
|
+
ORCHESTRATOR_VERSION$1 = "0.1.19";
|
|
49536
|
+
WORKER_BUILD_COMMIT = "1590f5e99";
|
|
49537
|
+
WORKER_SDK_VERSION = "0.1.19";
|
|
49538
|
+
WORKER_SDK_BUNDLE_HASH = "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f";
|
|
49539
|
+
WORKER_SHARED_VERSION = "0.1.19";
|
|
49540
|
+
WORKER_SHARED_BUNDLE_HASH = "5f2c220f24d166b0f13d0620a2e80d19689ac8b683a12154daef44e938fd46a7";
|
|
49541
|
+
WORKER_ENGINE_VERSION = "0.1.19";
|
|
49542
|
+
WORKER_ENGINE_BUNDLE_HASH = "79ce14640d1798eaaa7cb3aa6c4bc325da3bff1e9f4716936e19614eabb1d858";
|
|
48090
49543
|
logger$2 = createLogger({ prefix: "worker" });
|
|
48091
49544
|
DRAIN_TIMEOUT_MS = 3e5;
|
|
48092
49545
|
}));
|
|
@@ -48110,14 +49563,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
48110
49563
|
* Graceful shutdown:
|
|
48111
49564
|
* agent WS -> heartbeat -> HTTP -> DB
|
|
48112
49565
|
*/
|
|
48113
|
-
const ORCHESTRATOR_VERSION = "0.1.
|
|
48114
|
-
const BUILD_COMMIT = "
|
|
48115
|
-
const SDK_VERSION = "0.1.
|
|
48116
|
-
const SDK_BUNDLE_HASH = "
|
|
48117
|
-
const SHARED_VERSION = "0.1.
|
|
48118
|
-
const SHARED_BUNDLE_HASH = "
|
|
48119
|
-
const ENGINE_VERSION = "0.1.
|
|
48120
|
-
const ENGINE_BUNDLE_HASH = "
|
|
49566
|
+
const ORCHESTRATOR_VERSION = "0.1.19";
|
|
49567
|
+
const BUILD_COMMIT = "1590f5e99";
|
|
49568
|
+
const SDK_VERSION = "0.1.19";
|
|
49569
|
+
const SDK_BUNDLE_HASH = "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f";
|
|
49570
|
+
const SHARED_VERSION = "0.1.19";
|
|
49571
|
+
const SHARED_BUNDLE_HASH = "5f2c220f24d166b0f13d0620a2e80d19689ac8b683a12154daef44e938fd46a7";
|
|
49572
|
+
const ENGINE_VERSION = "0.1.19";
|
|
49573
|
+
const ENGINE_BUNDLE_HASH = "79ce14640d1798eaaa7cb3aa6c4bc325da3bff1e9f4716936e19614eabb1d858";
|
|
48121
49574
|
const otelSdk = initTelemetry({
|
|
48122
49575
|
serviceName: "kici-orchestrator",
|
|
48123
49576
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|