@kici-dev/orchestrator 0.1.17 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/dispatcher.d.ts +9 -0
- package/dist/agent/host-roster-reaper.d.ts +42 -0
- package/dist/agent/host-roster.d.ts +110 -0
- package/dist/agent/registry.d.ts +65 -2
- package/dist/app.d.ts +3 -0
- package/dist/cli/commands/host.d.ts +13 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +23 -0
- package/dist/cli/service/compose.d.ts +2 -1
- package/dist/cli/service/index.d.ts +2 -2
- package/dist/cli/service/instance/manifest.d.ts +12 -0
- package/dist/cli/service/instance/resolve.d.ts +12 -2
- package/dist/cli/service/launchd.d.ts +4 -1
- package/dist/cli/service/platform-detect.d.ts +5 -0
- package/dist/cli/service/systemd.d.ts +2 -1
- package/dist/cli/service/types.d.ts +37 -0
- package/dist/cli/service/windows.d.ts +2 -1
- package/dist/cli.js +794 -201
- package/dist/cluster/coordinator.d.ts +5 -1
- package/dist/config/schema.d.ts +4 -0
- package/dist/config/types.d.ts +9 -0
- package/dist/config.d.ts +6 -0
- package/dist/db/migrations/039_host_roster.d.ts +19 -0
- package/dist/db/migrations/040_runsonall_pin.d.ts +4 -0
- package/dist/db/migrations/041_wave_gated.d.ts +4 -0
- package/dist/db/migrations/042_dispatch_queue_patterns.d.ts +4 -0
- package/dist/db/types.d.ts +61 -0
- package/dist/diagnostics/fleet-collector.d.ts +1 -1
- package/dist/environments/held-runs.d.ts +9 -0
- package/dist/lockfile-redos-guard.d.ts +19 -0
- package/dist/metrics/prometheus.d.ts +13 -0
- package/dist/metrics/scheduled-jobs.d.ts +2 -2
- package/dist/orchestrator-core.d.ts +46 -1
- package/dist/pipeline/dispatch-matched-workflow.d.ts +42 -1
- package/dist/pipeline/processor.d.ts +25 -0
- package/dist/pipeline/wave-scheduler.d.ts +60 -0
- package/dist/queue/job-queue.d.ts +61 -2
- package/dist/reporting/execution-tracker.d.ts +28 -0
- package/dist/server.js +52720 -51280
- package/dist/standalone.js +1908 -488
- package/installer-image-digests.json +3 -3
- package/package.json +22 -22
- package/sbom.spdx.json +3227 -6682
package/dist/standalone.js
CHANGED
|
@@ -8,7 +8,7 @@ import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/s
|
|
|
8
8
|
import WebSocket from "ws";
|
|
9
9
|
import * as fs from "node:fs";
|
|
10
10
|
import { chmodSync, closeSync, createReadStream, createWriteStream, mkdirSync, mkdtempSync, openSync, promises, readFileSync, rmSync, statSync, unlinkSync, unwatchFile, watchFile, writeFileSync } from "node:fs";
|
|
11
|
-
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CacheRefScope, CheckRunConclusion, EnvDeleteErrorCode, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, FanoutError, HoldScope, IfFailedPolicy, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, LockFileParseError, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, TimeoutReason, TriggerSource, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_DISPATCH_ACK_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, materializeFanout, materializeResolvedMatrix, matrixEnvelopeFields, minAccessLogWarmDays, minSecretAuditLogWarmDays, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve } from "@kici-dev/engine";
|
|
11
|
+
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CacheRefScope, CheckRunConclusion, EnvDeleteErrorCode, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, FanoutError, HoldScope, IfFailedPolicy, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, LockFileParseError, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, TimeoutReason, TriggerSource, VariantKind, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_DISPATCH_ACK_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, matcherSatisfiedBy, materializeFanout, materializeResolvedHosts, materializeResolvedMatrix, matrixEnvelopeFields, minAccessLogWarmDays, minSecretAuditLogWarmDays, partitionMatchers, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve } from "@kici-dev/engine";
|
|
12
12
|
import { access, appendFile, chmod, constants, copyFile, link, mkdir, mkdtemp, open, readFile, readdir, rename, rm, stat, statfs, writeFile } from "node:fs/promises";
|
|
13
13
|
import * as path$1 from "node:path";
|
|
14
14
|
import path, { dirname, join, relative, resolve } from "node:path";
|
|
@@ -19,6 +19,7 @@ import { serve } from "@hono/node-server";
|
|
|
19
19
|
import { parse } from "yaml";
|
|
20
20
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
21
21
|
import { LRUCache } from "lru-cache";
|
|
22
|
+
import { assertMatchersSafe } from "@kici-dev/engine/labels/compile";
|
|
22
23
|
import { bodyLimit } from "hono/body-limit";
|
|
23
24
|
import { createNodeWebSocket } from "@hono/node-ws";
|
|
24
25
|
import { getConnInfo } from "@hono/node-server/conninfo";
|
|
@@ -41,7 +42,7 @@ import { JSONPath } from "jsonpath-plus";
|
|
|
41
42
|
import { execFile, spawn } from "node:child_process";
|
|
42
43
|
import { promisify } from "node:util";
|
|
43
44
|
import vm from "node:vm";
|
|
44
|
-
import
|
|
45
|
+
import { ZipArchive } from "archiver";
|
|
45
46
|
import { createInterface } from "node:readline";
|
|
46
47
|
import Docker from "dockerode";
|
|
47
48
|
import http from "node:http";
|
|
@@ -206,6 +207,9 @@ var init_config$5 = __esmMin((() => {
|
|
|
206
207
|
pgCustomerSecrets: z.enum(["true", "false"]).default("true").transform((v) => v === "true"),
|
|
207
208
|
agentAuth: z.enum(["token", "none"]).default("token"),
|
|
208
209
|
agentTokenTtlMs: z.coerce.number().default(36e5),
|
|
210
|
+
rosterGraceMs: z.coerce.number().int().min(1e3).default(3e5),
|
|
211
|
+
rosterTtlMs: z.coerce.number().int().min(1e3).default(18e5),
|
|
212
|
+
maxFanoutHosts: z.coerce.number().int().min(1).default(1024),
|
|
209
213
|
eventRouterMaxChainDepth: z.coerce.number().default(10),
|
|
210
214
|
eventRouterRateLimitPerWorkflowPerMinute: z.coerce.number().default(100),
|
|
211
215
|
eventRouterEventTtlSeconds: z.coerce.number().default(604800),
|
|
@@ -393,6 +397,9 @@ var init_config$5 = __esmMin((() => {
|
|
|
393
397
|
pgCustomerSecrets: "KICI_PG_CUSTOMER_SECRETS",
|
|
394
398
|
agentAuth: "KICI_AGENT_AUTH",
|
|
395
399
|
agentTokenTtlMs: "KICI_AGENT_TOKEN_TTL_MS",
|
|
400
|
+
rosterGraceMs: "KICI_ROSTER_GRACE_MS",
|
|
401
|
+
rosterTtlMs: "KICI_ROSTER_TTL_MS",
|
|
402
|
+
maxFanoutHosts: "KICI_MAX_FANOUT_HOSTS",
|
|
396
403
|
eventRouterMaxChainDepth: "KICI_EVENT_ROUTER_MAX_CHAIN_DEPTH",
|
|
397
404
|
eventRouterRateLimitPerWorkflowPerMinute: "KICI_EVENT_ROUTER_RATE_LIMIT_PER_WORKFLOW_PER_MINUTE",
|
|
398
405
|
eventRouterEventTtlSeconds: "KICI_EVENT_ROUTER_EVENT_TTL_SECONDS",
|
|
@@ -1053,12 +1060,12 @@ var init_peer_credentials = __esmMin((() => {
|
|
|
1053
1060
|
* Authentication uses ECDH key exchange followed by join token (first connect)
|
|
1054
1061
|
* or HMAC credential proof (reconnection).
|
|
1055
1062
|
*/
|
|
1056
|
-
var logger$
|
|
1063
|
+
var logger$89, SOFTWARE_VERSION$1, PeerClient$1;
|
|
1057
1064
|
var init_peer_client = __esmMin((() => {
|
|
1058
1065
|
init_peer_crypto();
|
|
1059
1066
|
init_peer_credentials();
|
|
1060
|
-
logger$
|
|
1061
|
-
SOFTWARE_VERSION$1 = "0.1.
|
|
1067
|
+
logger$89 = createLogger({ prefix: "peer-client" });
|
|
1068
|
+
SOFTWARE_VERSION$1 = "0.1.18";
|
|
1062
1069
|
PeerClient$1 = class {
|
|
1063
1070
|
ws = null;
|
|
1064
1071
|
_state = "disconnected";
|
|
@@ -1132,7 +1139,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1132
1139
|
*/
|
|
1133
1140
|
connect() {
|
|
1134
1141
|
if (this._state !== "disconnected") {
|
|
1135
|
-
logger$
|
|
1142
|
+
logger$89.warn("connect() called while not disconnected", { state: this._state });
|
|
1136
1143
|
return;
|
|
1137
1144
|
}
|
|
1138
1145
|
this.intentionalDisconnect = false;
|
|
@@ -1266,20 +1273,20 @@ var init_peer_client = __esmMin((() => {
|
|
|
1266
1273
|
}
|
|
1267
1274
|
});
|
|
1268
1275
|
} catch (err) {
|
|
1269
|
-
logger$
|
|
1276
|
+
logger$89.error("Failed to create peer WebSocket", { error: toErrorMessage(err) });
|
|
1270
1277
|
this._state = "disconnected";
|
|
1271
1278
|
this.scheduleReconnect();
|
|
1272
1279
|
return;
|
|
1273
1280
|
}
|
|
1274
1281
|
this.ws.on("open", () => {
|
|
1275
1282
|
this._state = "handshaking";
|
|
1276
|
-
logger$
|
|
1283
|
+
logger$89.info("Connected to peer, waiting for ECDH handshake", { url: this.url });
|
|
1277
1284
|
});
|
|
1278
1285
|
this.ws.on("message", (data) => {
|
|
1279
1286
|
this.handleMessage(data);
|
|
1280
1287
|
});
|
|
1281
1288
|
this.ws.on("close", (code, reason) => {
|
|
1282
|
-
logger$
|
|
1289
|
+
logger$89.info("Peer connection closed", {
|
|
1283
1290
|
code,
|
|
1284
1291
|
reason: reason.toString(),
|
|
1285
1292
|
targetInstanceId: this._targetInstanceId
|
|
@@ -1292,7 +1299,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1292
1299
|
if (!this.intentionalDisconnect) this.scheduleReconnect();
|
|
1293
1300
|
});
|
|
1294
1301
|
this.ws.on("error", (err) => {
|
|
1295
|
-
logger$
|
|
1302
|
+
logger$89.error("Peer WebSocket error", { error: err.message });
|
|
1296
1303
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
|
|
1297
1304
|
});
|
|
1298
1305
|
}
|
|
@@ -1303,12 +1310,12 @@ var init_peer_client = __esmMin((() => {
|
|
|
1303
1310
|
try {
|
|
1304
1311
|
parsed = JSON.parse(raw);
|
|
1305
1312
|
} catch {
|
|
1306
|
-
logger$
|
|
1313
|
+
logger$89.warn("Malformed JSON during handshake");
|
|
1307
1314
|
return;
|
|
1308
1315
|
}
|
|
1309
1316
|
const hello = peerHelloSchema.safeParse(parsed);
|
|
1310
1317
|
if (!hello.success) {
|
|
1311
|
-
logger$
|
|
1318
|
+
logger$89.warn("Expected peer.hello, got invalid message");
|
|
1312
1319
|
return;
|
|
1313
1320
|
}
|
|
1314
1321
|
const ecdh = generateEcdhKeyPair();
|
|
@@ -1317,7 +1324,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1317
1324
|
try {
|
|
1318
1325
|
this.sessionKey = deriveSessionKey(ecdh.privateKey, serverPubKey, nonce);
|
|
1319
1326
|
} catch (err) {
|
|
1320
|
-
logger$
|
|
1327
|
+
logger$89.error("ECDH key derivation failed", { error: toErrorMessage(err) });
|
|
1321
1328
|
if (this.ws) this.ws.close(1e3, "Key derivation failed");
|
|
1322
1329
|
return;
|
|
1323
1330
|
}
|
|
@@ -1327,48 +1334,48 @@ var init_peer_client = __esmMin((() => {
|
|
|
1327
1334
|
}));
|
|
1328
1335
|
this._state = "authenticating";
|
|
1329
1336
|
this.sendAuthRequest(nonce).catch((err) => {
|
|
1330
|
-
logger$
|
|
1337
|
+
logger$89.error("Failed to send auth request", { error: toErrorMessage(err) });
|
|
1331
1338
|
if (this.ws) this.ws.close(1e3, "Auth request failed");
|
|
1332
1339
|
});
|
|
1333
1340
|
return;
|
|
1334
1341
|
}
|
|
1335
1342
|
if (this._state === "authenticating") {
|
|
1336
1343
|
if (!this.sessionKey) {
|
|
1337
|
-
logger$
|
|
1344
|
+
logger$89.warn("No session key for auth response decryption");
|
|
1338
1345
|
return;
|
|
1339
1346
|
}
|
|
1340
1347
|
let decrypted;
|
|
1341
1348
|
try {
|
|
1342
1349
|
decrypted = decryptMessage(raw, this.sessionKey);
|
|
1343
1350
|
} catch {
|
|
1344
|
-
logger$
|
|
1351
|
+
logger$89.warn("Failed to decrypt auth response");
|
|
1345
1352
|
return;
|
|
1346
1353
|
}
|
|
1347
1354
|
let parsed;
|
|
1348
1355
|
try {
|
|
1349
1356
|
parsed = JSON.parse(decrypted);
|
|
1350
1357
|
} catch {
|
|
1351
|
-
logger$
|
|
1358
|
+
logger$89.warn("Malformed JSON in decrypted auth response");
|
|
1352
1359
|
return;
|
|
1353
1360
|
}
|
|
1354
1361
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
1355
1362
|
if (!msgResult.success) {
|
|
1356
|
-
logger$
|
|
1363
|
+
logger$89.warn("Invalid auth response", { errors: msgResult.error.issues });
|
|
1357
1364
|
return;
|
|
1358
1365
|
}
|
|
1359
1366
|
if (msgResult.data.type !== "peer.auth.response") {
|
|
1360
|
-
logger$
|
|
1367
|
+
logger$89.warn("Expected peer.auth.response, got", { type: msgResult.data.type });
|
|
1361
1368
|
return;
|
|
1362
1369
|
}
|
|
1363
1370
|
const msg = msgResult.data;
|
|
1364
1371
|
if (msg.accepted) {
|
|
1365
|
-
if (msg.softwareVersion) logger$
|
|
1372
|
+
if (msg.softwareVersion) logger$89.info("Coordinator software version", {
|
|
1366
1373
|
localVersion: SOFTWARE_VERSION$1,
|
|
1367
1374
|
coordinatorVersion: msg.softwareVersion
|
|
1368
1375
|
});
|
|
1369
1376
|
this._targetInstanceId = msg.instanceId ?? null;
|
|
1370
1377
|
if (msg.instanceId) this.onAuthenticated?.(msg.instanceId);
|
|
1371
|
-
logger$
|
|
1378
|
+
logger$89.info("Peer auth accepted", {
|
|
1372
1379
|
targetInstanceId: msg.instanceId,
|
|
1373
1380
|
agentCount: msg.agents?.length ?? 0,
|
|
1374
1381
|
scalerBackends: msg.scalerCapacity?.length ?? 0
|
|
@@ -1381,7 +1388,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1381
1388
|
role: msg.role ?? "coordinator",
|
|
1382
1389
|
issuedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1383
1390
|
}).catch((err) => {
|
|
1384
|
-
logger$
|
|
1391
|
+
logger$89.error("Failed to persist credential file", { error: toErrorMessage(err) });
|
|
1385
1392
|
});
|
|
1386
1393
|
this.peerRegistry.addPeer({
|
|
1387
1394
|
instanceId: msg.instanceId,
|
|
@@ -1409,12 +1416,12 @@ var init_peer_client = __esmMin((() => {
|
|
|
1409
1416
|
}
|
|
1410
1417
|
this.startHeartbeat();
|
|
1411
1418
|
} else {
|
|
1412
|
-
logger$
|
|
1419
|
+
logger$89.error("Peer auth rejected", { reason: msg.reason });
|
|
1413
1420
|
if (msg.reason === "Invalid proof" || msg.reason === "Unknown credential" || msg.reason === "Credential revoked") try {
|
|
1414
1421
|
unlinkSync(this.credentialFile);
|
|
1415
|
-
logger$
|
|
1422
|
+
logger$89.warn("Deleted stale credential file after server rejection", { reason: msg.reason });
|
|
1416
1423
|
} catch (err) {
|
|
1417
|
-
if (err.code !== "ENOENT") logger$
|
|
1424
|
+
if (err.code !== "ENOENT") logger$89.warn("Failed to delete stale credential file", {
|
|
1418
1425
|
error: toErrorMessage(err),
|
|
1419
1426
|
path: this.credentialFile
|
|
1420
1427
|
});
|
|
@@ -1428,19 +1435,19 @@ var init_peer_client = __esmMin((() => {
|
|
|
1428
1435
|
try {
|
|
1429
1436
|
decrypted = decryptMessage(raw, this.sessionKey);
|
|
1430
1437
|
} catch {
|
|
1431
|
-
logger$
|
|
1438
|
+
logger$89.warn("Failed to decrypt message from peer");
|
|
1432
1439
|
return;
|
|
1433
1440
|
}
|
|
1434
1441
|
let parsed;
|
|
1435
1442
|
try {
|
|
1436
1443
|
parsed = JSON.parse(decrypted);
|
|
1437
1444
|
} catch {
|
|
1438
|
-
logger$
|
|
1445
|
+
logger$89.warn("Malformed JSON from peer");
|
|
1439
1446
|
return;
|
|
1440
1447
|
}
|
|
1441
1448
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
1442
1449
|
if (!msgResult.success) {
|
|
1443
|
-
logger$
|
|
1450
|
+
logger$89.warn("Invalid message from peer", { errors: msgResult.error.issues });
|
|
1444
1451
|
return;
|
|
1445
1452
|
}
|
|
1446
1453
|
this.routeMessage(msgResult.data);
|
|
@@ -1482,7 +1489,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1482
1489
|
softwareVersion: SOFTWARE_VERSION$1,
|
|
1483
1490
|
role: this.role
|
|
1484
1491
|
};
|
|
1485
|
-
logger$
|
|
1492
|
+
logger$89.info("Sending credential-based auth request", {
|
|
1486
1493
|
targetUrl: this.url,
|
|
1487
1494
|
credentialInstanceId: cred.instanceId
|
|
1488
1495
|
});
|
|
@@ -1496,10 +1503,10 @@ var init_peer_client = __esmMin((() => {
|
|
|
1496
1503
|
softwareVersion: SOFTWARE_VERSION$1,
|
|
1497
1504
|
role: this.role
|
|
1498
1505
|
};
|
|
1499
|
-
logger$
|
|
1506
|
+
logger$89.info("Sending token-based auth request");
|
|
1500
1507
|
this.ws.send(encryptMessage(JSON.stringify(authRequest), this.sessionKey));
|
|
1501
1508
|
} else {
|
|
1502
|
-
logger$
|
|
1509
|
+
logger$89.error("No auth method available: no credential file and no join token");
|
|
1503
1510
|
if (this.ws.readyState === WebSocket.OPEN) this.ws.close(1e3, "No auth method");
|
|
1504
1511
|
}
|
|
1505
1512
|
}
|
|
@@ -1510,7 +1517,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1510
1517
|
break;
|
|
1511
1518
|
case "job.reroute":
|
|
1512
1519
|
this.onJobReroute(msg).catch((err) => {
|
|
1513
|
-
logger$
|
|
1520
|
+
logger$89.error("Error handling job reroute", { error: toErrorMessage(err) });
|
|
1514
1521
|
});
|
|
1515
1522
|
break;
|
|
1516
1523
|
case "job.reroute.ack": {
|
|
@@ -1518,7 +1525,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1518
1525
|
if (waiter) {
|
|
1519
1526
|
clearTimeout(waiter.timer);
|
|
1520
1527
|
this.ackWaiters.delete(msg.messageId);
|
|
1521
|
-
if (!msg.accepted) logger$
|
|
1528
|
+
if (!msg.accepted) logger$89.info("Reroute ACK rejected by peer", {
|
|
1522
1529
|
targetInstanceId: this._targetInstanceId,
|
|
1523
1530
|
reason: msg.reason
|
|
1524
1531
|
});
|
|
@@ -1548,7 +1555,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1548
1555
|
if (this.onPeerCacheUploadRequest) this.onPeerCacheUploadRequest(msg).then((response) => {
|
|
1549
1556
|
this.send(response);
|
|
1550
1557
|
}).catch((err) => {
|
|
1551
|
-
logger$
|
|
1558
|
+
logger$89.error("Error handling cache upload request", { error: toErrorMessage(err) });
|
|
1552
1559
|
});
|
|
1553
1560
|
break;
|
|
1554
1561
|
case "peer.cache.upload.response": {
|
|
@@ -1561,7 +1568,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1561
1568
|
break;
|
|
1562
1569
|
}
|
|
1563
1570
|
case "peer.auth.request":
|
|
1564
|
-
logger$
|
|
1571
|
+
logger$89.warn("Unexpected peer.auth.request on outgoing connection");
|
|
1565
1572
|
break;
|
|
1566
1573
|
case "raft.vote.response":
|
|
1567
1574
|
this.onRaftVoteResponse?.(msg);
|
|
@@ -1593,7 +1600,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1593
1600
|
handler(msg).then((result) => {
|
|
1594
1601
|
sendReply(result);
|
|
1595
1602
|
}).catch((err) => {
|
|
1596
|
-
logger$
|
|
1603
|
+
logger$89.error("Error executing peer config reload", { error: toErrorMessage(err) });
|
|
1597
1604
|
sendReply({
|
|
1598
1605
|
success: false,
|
|
1599
1606
|
errors: [toErrorMessage(err)]
|
|
@@ -1643,7 +1650,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1643
1650
|
this.cancelReconnect();
|
|
1644
1651
|
const delay = this.getReconnectDelay();
|
|
1645
1652
|
this.reconnectAttempts++;
|
|
1646
|
-
logger$
|
|
1653
|
+
logger$89.info("Scheduling peer reconnect", {
|
|
1647
1654
|
attempt: this.reconnectAttempts,
|
|
1648
1655
|
delayMs: Math.round(delay)
|
|
1649
1656
|
});
|
|
@@ -1950,7 +1957,7 @@ function createPeerHandler(deps) {
|
|
|
1950
1957
|
break;
|
|
1951
1958
|
case "job.reroute":
|
|
1952
1959
|
onJobReroute(msg).catch((err) => {
|
|
1953
|
-
logger$
|
|
1960
|
+
logger$88.error("Error handling job reroute from peer", { error: toErrorMessage(err) });
|
|
1954
1961
|
});
|
|
1955
1962
|
break;
|
|
1956
1963
|
case "job.reroute.ack": {
|
|
@@ -1987,7 +1994,7 @@ function createPeerHandler(deps) {
|
|
|
1987
1994
|
if (onPeerCacheUploadRequest) onPeerCacheUploadRequest(msg, conn.peerInstanceId).then((response) => {
|
|
1988
1995
|
sendEncryptedMessage(conn.ws, conn.sessionKey, response);
|
|
1989
1996
|
}).catch((err) => {
|
|
1990
|
-
logger$
|
|
1997
|
+
logger$88.error("Error handling cache upload request from peer", {
|
|
1991
1998
|
peerId: conn.peerInstanceId,
|
|
1992
1999
|
error: toErrorMessage(err)
|
|
1993
2000
|
});
|
|
@@ -2008,13 +2015,13 @@ function createPeerHandler(deps) {
|
|
|
2008
2015
|
});
|
|
2009
2016
|
break;
|
|
2010
2017
|
case "peer.cache.upload.response":
|
|
2011
|
-
logger$
|
|
2018
|
+
logger$88.warn("Unexpected peer.cache.upload.response on incoming connection");
|
|
2012
2019
|
break;
|
|
2013
2020
|
case "peer.auth.request":
|
|
2014
|
-
logger$
|
|
2021
|
+
logger$88.warn("Unexpected peer.auth.request after authentication");
|
|
2015
2022
|
break;
|
|
2016
2023
|
case "peer.auth.response":
|
|
2017
|
-
logger$
|
|
2024
|
+
logger$88.warn("Unexpected peer.auth.response on incoming connection");
|
|
2018
2025
|
break;
|
|
2019
2026
|
case "raft.vote.response":
|
|
2020
2027
|
onRaftVoteResponse?.(msg);
|
|
@@ -2046,7 +2053,7 @@ function createPeerHandler(deps) {
|
|
|
2046
2053
|
handler(msg).then((result) => {
|
|
2047
2054
|
sendResponse(result);
|
|
2048
2055
|
}).catch((err) => {
|
|
2049
|
-
logger$
|
|
2056
|
+
logger$88.error("Error executing peer config reload", {
|
|
2050
2057
|
peerId: conn.peerInstanceId,
|
|
2051
2058
|
error: toErrorMessage(err)
|
|
2052
2059
|
});
|
|
@@ -2088,7 +2095,7 @@ function createPeerHandler(deps) {
|
|
|
2088
2095
|
function handleConnection(ws, remoteIp) {
|
|
2089
2096
|
const ip = remoteIp ?? "unknown";
|
|
2090
2097
|
if (isRateLimited(ip)) {
|
|
2091
|
-
logger$
|
|
2098
|
+
logger$88.warn("Rate limited peer connection attempt", { ip });
|
|
2092
2099
|
ws.close(WS_CLOSE_UNAUTHORIZED, "Rate limited");
|
|
2093
2100
|
return;
|
|
2094
2101
|
}
|
|
@@ -2098,7 +2105,7 @@ function createPeerHandler(deps) {
|
|
|
2098
2105
|
let handshakeNonce = null;
|
|
2099
2106
|
const authTimer = setTimeout(() => {
|
|
2100
2107
|
if (!authenticated) {
|
|
2101
|
-
logger$
|
|
2108
|
+
logger$88.warn("Peer auth timeout, closing connection");
|
|
2102
2109
|
ws.close(WS_CLOSE_AUTH_TIMEOUT, "Auth timeout");
|
|
2103
2110
|
}
|
|
2104
2111
|
}, authTimeoutMs);
|
|
@@ -2118,12 +2125,12 @@ function createPeerHandler(deps) {
|
|
|
2118
2125
|
try {
|
|
2119
2126
|
parsed = JSON.parse(raw);
|
|
2120
2127
|
} catch {
|
|
2121
|
-
logger$
|
|
2128
|
+
logger$88.warn("Malformed JSON during ECDH handshake");
|
|
2122
2129
|
return;
|
|
2123
2130
|
}
|
|
2124
2131
|
const helloResp = peerHelloResponseSchema.safeParse(parsed);
|
|
2125
2132
|
if (!helloResp.success) {
|
|
2126
|
-
logger$
|
|
2133
|
+
logger$88.warn("Expected peer.hello.response, got invalid message");
|
|
2127
2134
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Expected hello response");
|
|
2128
2135
|
clearTimeout(authTimer);
|
|
2129
2136
|
return;
|
|
@@ -2131,7 +2138,7 @@ function createPeerHandler(deps) {
|
|
|
2131
2138
|
try {
|
|
2132
2139
|
sessionKey = deriveSessionKey(ecdh.privateKey, Buffer.from(helloResp.data.ephemeralPublicKey, "base64"), nonce);
|
|
2133
2140
|
} catch (err) {
|
|
2134
|
-
logger$
|
|
2141
|
+
logger$88.warn("ECDH key derivation failed", { error: toErrorMessage(err) });
|
|
2135
2142
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Key derivation failed");
|
|
2136
2143
|
clearTimeout(authTimer);
|
|
2137
2144
|
return;
|
|
@@ -2141,7 +2148,7 @@ function createPeerHandler(deps) {
|
|
|
2141
2148
|
}
|
|
2142
2149
|
if (!authenticated) {
|
|
2143
2150
|
if (!sessionKey) {
|
|
2144
|
-
logger$
|
|
2151
|
+
logger$88.warn("No session key for auth decryption");
|
|
2145
2152
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "No session key");
|
|
2146
2153
|
clearTimeout(authTimer);
|
|
2147
2154
|
return;
|
|
@@ -2150,7 +2157,7 @@ function createPeerHandler(deps) {
|
|
|
2150
2157
|
try {
|
|
2151
2158
|
decrypted = decryptMessage(raw, sessionKey);
|
|
2152
2159
|
} catch (err) {
|
|
2153
|
-
logger$
|
|
2160
|
+
logger$88.warn("Failed to decrypt auth request", { error: toErrorMessage(err) });
|
|
2154
2161
|
recordFailedAuth(ip);
|
|
2155
2162
|
ws.close(WS_CLOSE_UNAUTHORIZED, "Decryption failed");
|
|
2156
2163
|
clearTimeout(authTimer);
|
|
@@ -2160,14 +2167,14 @@ function createPeerHandler(deps) {
|
|
|
2160
2167
|
try {
|
|
2161
2168
|
authParsed = JSON.parse(decrypted);
|
|
2162
2169
|
} catch {
|
|
2163
|
-
logger$
|
|
2170
|
+
logger$88.warn("Malformed JSON in decrypted auth request");
|
|
2164
2171
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Invalid auth format");
|
|
2165
2172
|
clearTimeout(authTimer);
|
|
2166
2173
|
return;
|
|
2167
2174
|
}
|
|
2168
2175
|
const authMsg = peerAuthRequestSchema.safeParse(authParsed);
|
|
2169
2176
|
if (!authMsg.success) {
|
|
2170
|
-
logger$
|
|
2177
|
+
logger$88.warn("Invalid peer.auth.request format", { errors: authMsg.error.issues });
|
|
2171
2178
|
ws.close(WS_CLOSE_INVALID_MESSAGE, "Invalid auth request");
|
|
2172
2179
|
clearTimeout(authTimer);
|
|
2173
2180
|
return;
|
|
@@ -2177,7 +2184,7 @@ function createPeerHandler(deps) {
|
|
|
2177
2184
|
authenticated = true;
|
|
2178
2185
|
peerInstanceId = authMsg.data.instanceId;
|
|
2179
2186
|
clearTimeout(authTimer);
|
|
2180
|
-
logger$
|
|
2187
|
+
logger$88.info("Peer authenticated", { peerInstanceId });
|
|
2181
2188
|
peerRegistry.addPeer({
|
|
2182
2189
|
instanceId: peerInstanceId,
|
|
2183
2190
|
connectionId: randomUUID(),
|
|
@@ -2199,7 +2206,7 @@ function createPeerHandler(deps) {
|
|
|
2199
2206
|
connections.set(peerInstanceId, conn);
|
|
2200
2207
|
startHeartbeat(conn);
|
|
2201
2208
|
}).catch((err) => {
|
|
2202
|
-
logger$
|
|
2209
|
+
logger$88.error("Unexpected error during auth handling", { error: toErrorMessage(err) });
|
|
2203
2210
|
ws.close(WS_CLOSE_UNAUTHORIZED, "Auth error");
|
|
2204
2211
|
clearTimeout(authTimer);
|
|
2205
2212
|
});
|
|
@@ -2210,19 +2217,19 @@ function createPeerHandler(deps) {
|
|
|
2210
2217
|
try {
|
|
2211
2218
|
decrypted = decryptMessage(raw, sessionKey);
|
|
2212
2219
|
} catch {
|
|
2213
|
-
logger$
|
|
2220
|
+
logger$88.warn("Failed to decrypt message from authenticated peer");
|
|
2214
2221
|
return;
|
|
2215
2222
|
}
|
|
2216
2223
|
let parsed;
|
|
2217
2224
|
try {
|
|
2218
2225
|
parsed = JSON.parse(decrypted);
|
|
2219
2226
|
} catch {
|
|
2220
|
-
logger$
|
|
2227
|
+
logger$88.warn("Malformed JSON from peer connection");
|
|
2221
2228
|
return;
|
|
2222
2229
|
}
|
|
2223
2230
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
2224
2231
|
if (!msgResult.success) {
|
|
2225
|
-
logger$
|
|
2232
|
+
logger$88.warn("Invalid message from peer", { errors: msgResult.error.issues });
|
|
2226
2233
|
return;
|
|
2227
2234
|
}
|
|
2228
2235
|
const conn = connections.get(peerInstanceId);
|
|
@@ -2231,14 +2238,14 @@ function createPeerHandler(deps) {
|
|
|
2231
2238
|
ws.on("close", () => {
|
|
2232
2239
|
clearTimeout(authTimer);
|
|
2233
2240
|
if (peerInstanceId) {
|
|
2234
|
-
logger$
|
|
2241
|
+
logger$88.info("Peer disconnected", { peerInstanceId });
|
|
2235
2242
|
rejectLogsCollectForPeer(peerInstanceId);
|
|
2236
2243
|
const conn = connections.get(peerInstanceId);
|
|
2237
2244
|
if (conn) cleanupConnection(conn);
|
|
2238
2245
|
}
|
|
2239
2246
|
});
|
|
2240
2247
|
ws.on("error", (err) => {
|
|
2241
|
-
logger$
|
|
2248
|
+
logger$88.error("Peer connection error", { error: toErrorMessage(err) });
|
|
2242
2249
|
});
|
|
2243
2250
|
}
|
|
2244
2251
|
/**
|
|
@@ -2247,7 +2254,7 @@ function createPeerHandler(deps) {
|
|
|
2247
2254
|
*/
|
|
2248
2255
|
async function handleAuth(ws, sessionKey, nonce, authMsg, ip) {
|
|
2249
2256
|
if (authMsg.protocolVersion < MIN_PROTOCOL_VERSION) {
|
|
2250
|
-
logger$
|
|
2257
|
+
logger$88.warn("Peer protocol version below minimum", {
|
|
2251
2258
|
peerInstanceId: authMsg.instanceId,
|
|
2252
2259
|
received: authMsg.protocolVersion,
|
|
2253
2260
|
minimum: MIN_PROTOCOL_VERSION
|
|
@@ -2263,7 +2270,7 @@ function createPeerHandler(deps) {
|
|
|
2263
2270
|
ws.close(WS_CLOSE_PROTOCOL_ERROR, "Unsupported protocol version");
|
|
2264
2271
|
return false;
|
|
2265
2272
|
}
|
|
2266
|
-
if (authMsg.softwareVersion) logger$
|
|
2273
|
+
if (authMsg.softwareVersion) logger$88.info("Peer software version", {
|
|
2267
2274
|
peerInstanceId: authMsg.instanceId,
|
|
2268
2275
|
localVersion: SOFTWARE_VERSION,
|
|
2269
2276
|
remoteVersion: authMsg.softwareVersion
|
|
@@ -2271,7 +2278,7 @@ function createPeerHandler(deps) {
|
|
|
2271
2278
|
if (authMsg.token) try {
|
|
2272
2279
|
const result = await tokenManager.validateAndConsumeToken(authMsg.token, instanceId);
|
|
2273
2280
|
if (!acceptedRoles.includes(result.routing.role)) {
|
|
2274
|
-
logger$
|
|
2281
|
+
logger$88.warn("Peer role mismatch", {
|
|
2275
2282
|
peerInstanceId: authMsg.instanceId,
|
|
2276
2283
|
accepted: acceptedRoles,
|
|
2277
2284
|
actual: result.routing.role
|
|
@@ -2315,7 +2322,7 @@ function createPeerHandler(deps) {
|
|
|
2315
2322
|
const existing = await credentialStore.findByInstanceId(authMsg.instanceId);
|
|
2316
2323
|
if (existing && !existing.revokedAt && existing.sourceTokenHash === presentedHash) {
|
|
2317
2324
|
if (!acceptedRoles.includes(parsed.routing.role)) {
|
|
2318
|
-
logger$
|
|
2325
|
+
logger$88.warn("Peer role mismatch on token retry", {
|
|
2319
2326
|
peerInstanceId: authMsg.instanceId,
|
|
2320
2327
|
accepted: acceptedRoles,
|
|
2321
2328
|
actual: parsed.routing.role
|
|
@@ -2339,7 +2346,7 @@ function createPeerHandler(deps) {
|
|
|
2339
2346
|
routingKeys: [parsed.routing.routingKey],
|
|
2340
2347
|
sourceTokenHash: presentedHash
|
|
2341
2348
|
});
|
|
2342
|
-
logger$
|
|
2349
|
+
logger$88.info("Peer idempotent token retry accepted", {
|
|
2343
2350
|
peerInstanceId: authMsg.instanceId,
|
|
2344
2351
|
sourceTokenHash: presentedHash
|
|
2345
2352
|
});
|
|
@@ -2358,12 +2365,12 @@ function createPeerHandler(deps) {
|
|
|
2358
2365
|
return true;
|
|
2359
2366
|
}
|
|
2360
2367
|
} catch (recoveryErr) {
|
|
2361
|
-
logger$
|
|
2368
|
+
logger$88.warn("Peer token idempotent recovery failed", {
|
|
2362
2369
|
peerInstanceId: authMsg.instanceId,
|
|
2363
2370
|
error: toErrorMessage(recoveryErr)
|
|
2364
2371
|
});
|
|
2365
2372
|
}
|
|
2366
|
-
logger$
|
|
2373
|
+
logger$88.warn("Peer token validation failed", {
|
|
2367
2374
|
peerInstanceId: authMsg.instanceId,
|
|
2368
2375
|
error: toErrorMessage(err)
|
|
2369
2376
|
});
|
|
@@ -2380,7 +2387,7 @@ function createPeerHandler(deps) {
|
|
|
2380
2387
|
else if (authMsg.proof) {
|
|
2381
2388
|
const stored = await credentialStore.findByInstanceId(authMsg.instanceId);
|
|
2382
2389
|
if (!stored) {
|
|
2383
|
-
logger$
|
|
2390
|
+
logger$88.warn("Peer credential not found", { peerInstanceId: authMsg.instanceId });
|
|
2384
2391
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2385
2392
|
type: "peer.auth.response",
|
|
2386
2393
|
accepted: false,
|
|
@@ -2392,7 +2399,7 @@ function createPeerHandler(deps) {
|
|
|
2392
2399
|
return false;
|
|
2393
2400
|
}
|
|
2394
2401
|
if (stored.revokedAt) {
|
|
2395
|
-
logger$
|
|
2402
|
+
logger$88.warn("Peer credential revoked", { peerInstanceId: authMsg.instanceId });
|
|
2396
2403
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2397
2404
|
type: "peer.auth.response",
|
|
2398
2405
|
accepted: false,
|
|
@@ -2407,7 +2414,7 @@ function createPeerHandler(deps) {
|
|
|
2407
2414
|
const expectedProof = createHmac("sha256", Buffer.from(stored.credentialHash, "hex")).update(nonceB64 + ":" + authMsg.instanceId).digest();
|
|
2408
2415
|
const proofBuffer = Buffer.from(authMsg.proof, "hex");
|
|
2409
2416
|
if (proofBuffer.length !== expectedProof.length || !timingSafeEqual(proofBuffer, expectedProof)) {
|
|
2410
|
-
logger$
|
|
2417
|
+
logger$88.warn("Peer HMAC proof invalid", { peerInstanceId: authMsg.instanceId });
|
|
2411
2418
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2412
2419
|
type: "peer.auth.response",
|
|
2413
2420
|
accepted: false,
|
|
@@ -2431,7 +2438,7 @@ function createPeerHandler(deps) {
|
|
|
2431
2438
|
});
|
|
2432
2439
|
return true;
|
|
2433
2440
|
} else {
|
|
2434
|
-
logger$
|
|
2441
|
+
logger$88.warn("Peer auth request missing token and proof", { peerInstanceId: authMsg.instanceId });
|
|
2435
2442
|
sendEncryptedMessage(ws, sessionKey, {
|
|
2436
2443
|
type: "peer.auth.response",
|
|
2437
2444
|
accepted: false,
|
|
@@ -2579,12 +2586,12 @@ function createPeerHandler(deps) {
|
|
|
2579
2586
|
closeAllInbound
|
|
2580
2587
|
};
|
|
2581
2588
|
}
|
|
2582
|
-
var logger$
|
|
2589
|
+
var logger$88, SOFTWARE_VERSION, RATE_LIMIT_MAX, RATE_LIMIT_WINDOW_MS;
|
|
2583
2590
|
var init_peer_handler = __esmMin((() => {
|
|
2584
2591
|
init_peer_crypto();
|
|
2585
2592
|
init_join_token();
|
|
2586
|
-
logger$
|
|
2587
|
-
SOFTWARE_VERSION = "0.1.
|
|
2593
|
+
logger$88 = createLogger({ prefix: "peer-handler" });
|
|
2594
|
+
SOFTWARE_VERSION = "0.1.18";
|
|
2588
2595
|
RATE_LIMIT_MAX = 5;
|
|
2589
2596
|
RATE_LIMIT_WINDOW_MS = 6e4;
|
|
2590
2597
|
}));
|
|
@@ -2606,9 +2613,9 @@ var init_peer_handler = __esmMin((() => {
|
|
|
2606
2613
|
* heartbeat (peer.heartbeat every 30s). Raft heartbeat is purely for leader
|
|
2607
2614
|
* liveness detection.
|
|
2608
2615
|
*/
|
|
2609
|
-
var logger$
|
|
2616
|
+
var logger$87, RaftNode;
|
|
2610
2617
|
var init_raft = __esmMin((() => {
|
|
2611
|
-
logger$
|
|
2618
|
+
logger$87 = createLogger({ prefix: "raft" });
|
|
2612
2619
|
RaftNode = class {
|
|
2613
2620
|
instanceId;
|
|
2614
2621
|
stateStore;
|
|
@@ -2659,7 +2666,7 @@ var init_raft = __esmMin((() => {
|
|
|
2659
2666
|
this.leaderId = state.leaderId;
|
|
2660
2667
|
this.role = "follower";
|
|
2661
2668
|
this.startedAt = Date.now();
|
|
2662
|
-
logger$
|
|
2669
|
+
logger$87.info("Raft node started", {
|
|
2663
2670
|
instanceId: this.instanceId,
|
|
2664
2671
|
currentTerm: this.currentTerm,
|
|
2665
2672
|
leaderId: this.leaderId
|
|
@@ -2677,7 +2684,7 @@ var init_raft = __esmMin((() => {
|
|
|
2677
2684
|
votedFor: this.votedFor,
|
|
2678
2685
|
leaderId: this.leaderId
|
|
2679
2686
|
});
|
|
2680
|
-
logger$
|
|
2687
|
+
logger$87.info("Raft node stopped", { instanceId: this.instanceId });
|
|
2681
2688
|
}
|
|
2682
2689
|
/** Whether this node is the current leader. */
|
|
2683
2690
|
isLeader() {
|
|
@@ -2703,11 +2710,11 @@ var init_raft = __esmMin((() => {
|
|
|
2703
2710
|
*/
|
|
2704
2711
|
handlePeerLeaving(instanceId) {
|
|
2705
2712
|
if (this.role === "leader") {
|
|
2706
|
-
logger$
|
|
2713
|
+
logger$87.debug("Ignoring peer.leaving (we are leader)", { leavingPeer: instanceId });
|
|
2707
2714
|
return;
|
|
2708
2715
|
}
|
|
2709
2716
|
if (instanceId === this.leaderId) {
|
|
2710
|
-
logger$
|
|
2717
|
+
logger$87.info("Current leader is leaving, starting immediate election", {
|
|
2711
2718
|
leavingPeer: instanceId,
|
|
2712
2719
|
term: this.currentTerm
|
|
2713
2720
|
});
|
|
@@ -2715,7 +2722,7 @@ var init_raft = __esmMin((() => {
|
|
|
2715
2722
|
this.consecutiveUnansweredElections = 0;
|
|
2716
2723
|
this.clearElectionTimer();
|
|
2717
2724
|
this.startElection();
|
|
2718
|
-
} else logger$
|
|
2725
|
+
} else logger$87.debug("Non-leader peer leaving", { leavingPeer: instanceId });
|
|
2719
2726
|
}
|
|
2720
2727
|
/**
|
|
2721
2728
|
* Reset the election timer with a randomized timeout.
|
|
@@ -2763,7 +2770,7 @@ var init_raft = __esmMin((() => {
|
|
|
2763
2770
|
this.votedFor = this.instanceId;
|
|
2764
2771
|
this.role = "candidate";
|
|
2765
2772
|
this.votesReceived = new Set([this.instanceId]);
|
|
2766
|
-
logger$
|
|
2773
|
+
logger$87.info("Starting election", {
|
|
2767
2774
|
instanceId: this.instanceId,
|
|
2768
2775
|
term: this.currentTerm
|
|
2769
2776
|
});
|
|
@@ -2772,12 +2779,12 @@ var init_raft = __esmMin((() => {
|
|
|
2772
2779
|
votedFor: this.votedFor,
|
|
2773
2780
|
leaderId: this.leaderId
|
|
2774
2781
|
}).catch((err) => {
|
|
2775
|
-
logger$
|
|
2782
|
+
logger$87.error("Failed to persist election state", { error: toErrorMessage(err) });
|
|
2776
2783
|
});
|
|
2777
2784
|
const connectedPeerCount = this.peerRegistry.getConnectedCoordinatorPeerCount();
|
|
2778
2785
|
if (connectedPeerCount === 0) {
|
|
2779
2786
|
if (this.gracePeriodMs > 0 && Date.now() - this.startedAt < this.gracePeriodMs) {
|
|
2780
|
-
logger$
|
|
2787
|
+
logger$87.info("Dormant mode: deferring self-election (grace period active)", {
|
|
2781
2788
|
instanceId: this.instanceId,
|
|
2782
2789
|
elapsedMs: Date.now() - this.startedAt,
|
|
2783
2790
|
gracePeriodMs: this.gracePeriodMs
|
|
@@ -2785,7 +2792,7 @@ var init_raft = __esmMin((() => {
|
|
|
2785
2792
|
this.resetElectionTimer();
|
|
2786
2793
|
return;
|
|
2787
2794
|
}
|
|
2788
|
-
logger$
|
|
2795
|
+
logger$87.info("No peers connected, self-electing as leader", {
|
|
2789
2796
|
instanceId: this.instanceId,
|
|
2790
2797
|
term: this.currentTerm
|
|
2791
2798
|
});
|
|
@@ -2793,7 +2800,7 @@ var init_raft = __esmMin((() => {
|
|
|
2793
2800
|
return;
|
|
2794
2801
|
}
|
|
2795
2802
|
if (this.consecutiveUnansweredElections >= 2) {
|
|
2796
|
-
logger$
|
|
2803
|
+
logger$87.warn("No vote responses after consecutive elections, forcing self-election", {
|
|
2797
2804
|
instanceId: this.instanceId,
|
|
2798
2805
|
term: this.currentTerm,
|
|
2799
2806
|
consecutiveUnanswered: this.consecutiveUnansweredElections,
|
|
@@ -2828,14 +2835,14 @@ var init_raft = __esmMin((() => {
|
|
|
2828
2835
|
votedFor: this.votedFor,
|
|
2829
2836
|
leaderId: this.leaderId
|
|
2830
2837
|
}).catch((err) => {
|
|
2831
|
-
logger$
|
|
2838
|
+
logger$87.error("Failed to persist vote", { error: toErrorMessage(err) });
|
|
2832
2839
|
});
|
|
2833
2840
|
this.resetElectionTimer();
|
|
2834
|
-
logger$
|
|
2841
|
+
logger$87.info("Vote granted", {
|
|
2835
2842
|
candidateId: msg.candidateId,
|
|
2836
2843
|
term: msg.term
|
|
2837
2844
|
});
|
|
2838
|
-
} else logger$
|
|
2845
|
+
} else logger$87.info("Vote denied", {
|
|
2839
2846
|
candidateId: msg.candidateId,
|
|
2840
2847
|
term: msg.term,
|
|
2841
2848
|
currentTerm: this.currentTerm,
|
|
@@ -2860,7 +2867,7 @@ var init_raft = __esmMin((() => {
|
|
|
2860
2867
|
this.consecutiveUnansweredElections = 0;
|
|
2861
2868
|
if (msg.voteGranted) {
|
|
2862
2869
|
this.votesReceived.add(msg.voterId);
|
|
2863
|
-
logger$
|
|
2870
|
+
logger$87.info("Vote received", {
|
|
2864
2871
|
from: msg.voterId,
|
|
2865
2872
|
term: msg.term,
|
|
2866
2873
|
totalVotes: this.votesReceived.size
|
|
@@ -2887,7 +2894,7 @@ var init_raft = __esmMin((() => {
|
|
|
2887
2894
|
this.leaderId = this.instanceId;
|
|
2888
2895
|
this.consecutiveUnansweredElections = 0;
|
|
2889
2896
|
this.clearElectionTimer();
|
|
2890
|
-
logger$
|
|
2897
|
+
logger$87.info("Became leader", {
|
|
2891
2898
|
instanceId: this.instanceId,
|
|
2892
2899
|
term: this.currentTerm
|
|
2893
2900
|
});
|
|
@@ -2897,7 +2904,7 @@ var init_raft = __esmMin((() => {
|
|
|
2897
2904
|
votedFor: this.votedFor,
|
|
2898
2905
|
leaderId: this.leaderId
|
|
2899
2906
|
}).catch((err) => {
|
|
2900
|
-
logger$
|
|
2907
|
+
logger$87.error("Failed to persist leader state", { error: toErrorMessage(err) });
|
|
2901
2908
|
});
|
|
2902
2909
|
this.onBecomeLeader();
|
|
2903
2910
|
}
|
|
@@ -2918,16 +2925,16 @@ var init_raft = __esmMin((() => {
|
|
|
2918
2925
|
votedFor: this.votedFor,
|
|
2919
2926
|
leaderId: this.leaderId
|
|
2920
2927
|
}).catch((err) => {
|
|
2921
|
-
logger$
|
|
2928
|
+
logger$87.error("Failed to persist step-down state", { error: toErrorMessage(err) });
|
|
2922
2929
|
});
|
|
2923
2930
|
if (wasLeader) {
|
|
2924
|
-
logger$
|
|
2931
|
+
logger$87.info("Lost leadership", {
|
|
2925
2932
|
instanceId: this.instanceId,
|
|
2926
2933
|
newTerm,
|
|
2927
2934
|
newLeaderId
|
|
2928
2935
|
});
|
|
2929
2936
|
this.onLoseLeadership();
|
|
2930
|
-
} else logger$
|
|
2937
|
+
} else logger$87.info("Stepped down", {
|
|
2931
2938
|
instanceId: this.instanceId,
|
|
2932
2939
|
newTerm,
|
|
2933
2940
|
newLeaderId
|
|
@@ -3035,9 +3042,9 @@ var init_raft_state = __esmMin((() => {
|
|
|
3035
3042
|
}));
|
|
3036
3043
|
//#endregion
|
|
3037
3044
|
//#region src/cluster/orphan-recovery.ts
|
|
3038
|
-
var logger$
|
|
3045
|
+
var logger$86, DEFAULT_STALE_THRESHOLD_MS, DEFAULT_JOB_STUCK_THRESHOLD_MS, OrphanRecovery;
|
|
3039
3046
|
var init_orphan_recovery = __esmMin((() => {
|
|
3040
|
-
logger$
|
|
3047
|
+
logger$86 = createLogger({ prefix: "orphan-recovery" });
|
|
3041
3048
|
DEFAULT_STALE_THRESHOLD_MS = 300 * 1e3;
|
|
3042
3049
|
DEFAULT_JOB_STUCK_THRESHOLD_MS = 180 * 1e3;
|
|
3043
3050
|
OrphanRecovery = class {
|
|
@@ -3066,10 +3073,10 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3066
3073
|
if (this.scanTimer) return;
|
|
3067
3074
|
this.scanTimer = setInterval(() => {
|
|
3068
3075
|
this.scanForOrphans().catch((err) => {
|
|
3069
|
-
logger$
|
|
3076
|
+
logger$86.error("Orphan recovery scan failed", { error: toErrorMessage(err) });
|
|
3070
3077
|
});
|
|
3071
3078
|
}, this.scanIntervalMs);
|
|
3072
|
-
logger$
|
|
3079
|
+
logger$86.info("Orphan recovery started", {
|
|
3073
3080
|
scanIntervalMs: this.scanIntervalMs,
|
|
3074
3081
|
staleThresholdMs: this.staleThresholdMs
|
|
3075
3082
|
});
|
|
@@ -3082,7 +3089,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3082
3089
|
clearInterval(this.scanTimer);
|
|
3083
3090
|
this.scanTimer = null;
|
|
3084
3091
|
}
|
|
3085
|
-
logger$
|
|
3092
|
+
logger$86.info("Orphan recovery stopped");
|
|
3086
3093
|
}
|
|
3087
3094
|
/**
|
|
3088
3095
|
* Scan for orphaned runs and recover them.
|
|
@@ -3100,7 +3107,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3100
3107
|
"sha"
|
|
3101
3108
|
]).where("status", "=", ExecutionRunStatus.enum.running).where("started_at", "<", staleThreshold).execute();
|
|
3102
3109
|
if (staleRuns.length === 0) return;
|
|
3103
|
-
logger$
|
|
3110
|
+
logger$86.info("Found potentially orphaned runs", { count: staleRuns.length });
|
|
3104
3111
|
for (const run of staleRuns) await this.recoverRun(run);
|
|
3105
3112
|
}
|
|
3106
3113
|
/**
|
|
@@ -3108,7 +3115,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3108
3115
|
*/
|
|
3109
3116
|
async recoverRun(run) {
|
|
3110
3117
|
if (this.isCoordinatorStillConnected(run.routing_key)) return;
|
|
3111
|
-
logger$
|
|
3118
|
+
logger$86.info("Recovering orphan run", {
|
|
3112
3119
|
runId: run.run_id,
|
|
3113
3120
|
routingKey: run.routing_key,
|
|
3114
3121
|
workflowName: run.workflow_name
|
|
@@ -3125,7 +3132,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3125
3132
|
completed_at: /* @__PURE__ */ new Date(),
|
|
3126
3133
|
failure_reason: "No jobs found for orphaned run"
|
|
3127
3134
|
}).where("run_id", "=", run.run_id).where("status", "=", ExecutionRunStatus.enum.running).execute();
|
|
3128
|
-
logger$
|
|
3135
|
+
logger$86.info("Finalized orphan run with no jobs", { runId: run.run_id });
|
|
3129
3136
|
this.executionTracker.emitInfraEvent(run.run_id, "orchestrator.run.orphan_recovered", { metadata: {
|
|
3130
3137
|
routingKey: run.routing_key,
|
|
3131
3138
|
reason: "No jobs found"
|
|
@@ -3152,7 +3159,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
3152
3159
|
reason: errorMsg
|
|
3153
3160
|
}
|
|
3154
3161
|
});
|
|
3155
|
-
logger$
|
|
3162
|
+
logger$86.info("Marked stuck job as failed", {
|
|
3156
3163
|
runId: run.run_id,
|
|
3157
3164
|
jobId: job.job_id,
|
|
3158
3165
|
jobName: job.job_name
|
|
@@ -3206,9 +3213,9 @@ function agentSatisfiesMandatoryLabels(agent, requiredLabels) {
|
|
|
3206
3213
|
const required = new Set(requiredLabels);
|
|
3207
3214
|
return mandatory.every((m) => required.has(m));
|
|
3208
3215
|
}
|
|
3209
|
-
var logger$
|
|
3216
|
+
var logger$85, DEFAULT_ACK_TIMEOUT_MS, DEFAULT_MAX_HOPS, NAK_BACKOFF_BASE_MS, NAK_BACKOFF_MAX_MS, RunCoordinator;
|
|
3210
3217
|
var init_coordinator = __esmMin((() => {
|
|
3211
|
-
logger$
|
|
3218
|
+
logger$85 = createLogger({ prefix: "coordinator" });
|
|
3212
3219
|
DEFAULT_ACK_TIMEOUT_MS = 15e3;
|
|
3213
3220
|
DEFAULT_MAX_HOPS = 3;
|
|
3214
3221
|
NAK_BACKOFF_BASE_MS = 1e3;
|
|
@@ -3266,6 +3273,8 @@ var init_coordinator = __esmMin((() => {
|
|
|
3266
3273
|
workflowName: runContext.workflowName,
|
|
3267
3274
|
jobName: job.jobName,
|
|
3268
3275
|
runsOnLabels: flatLabels,
|
|
3276
|
+
runsOnPatterns: job.runsOnPatterns,
|
|
3277
|
+
excludePatterns: job.excludePatterns,
|
|
3269
3278
|
excludeLabels: job.excludeLabels,
|
|
3270
3279
|
jobConfig: job.jobConfig,
|
|
3271
3280
|
repoUrl: job.repoUrl,
|
|
@@ -3317,7 +3326,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3317
3326
|
}));
|
|
3318
3327
|
await Promise.all(reroutePromises);
|
|
3319
3328
|
}
|
|
3320
|
-
logger$
|
|
3329
|
+
logger$85.info("Jobs routed", {
|
|
3321
3330
|
runId: runContext.runId,
|
|
3322
3331
|
local: result.localJobs.length,
|
|
3323
3332
|
rerouted: result.reroutedJobs.length,
|
|
@@ -3346,6 +3355,8 @@ var init_coordinator = __esmMin((() => {
|
|
|
3346
3355
|
jobName: msg.jobName,
|
|
3347
3356
|
runsOnLabels: flatLabels,
|
|
3348
3357
|
excludeLabels: msg.excludeLabels,
|
|
3358
|
+
runsOnPatterns: msg.runsOnPatterns,
|
|
3359
|
+
excludePatterns: msg.excludePatterns,
|
|
3349
3360
|
jobConfig: msg.jobConfig ?? msg.payload,
|
|
3350
3361
|
repoUrl: msg.repoUrl ?? "",
|
|
3351
3362
|
ref: msg.ref ?? "",
|
|
@@ -3378,7 +3389,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3378
3389
|
runId: msg.runId
|
|
3379
3390
|
});
|
|
3380
3391
|
} catch (err) {
|
|
3381
|
-
logger$
|
|
3392
|
+
logger$85.warn("Failed to create check runs for rerouted job", {
|
|
3382
3393
|
runId: msg.runId,
|
|
3383
3394
|
error: toErrorMessage(err)
|
|
3384
3395
|
});
|
|
@@ -3395,7 +3406,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3395
3406
|
jobName: msg.jobName
|
|
3396
3407
|
}], msg.routingKey);
|
|
3397
3408
|
} catch (err) {
|
|
3398
|
-
logger$
|
|
3409
|
+
logger$85.warn("Failed to register rerouted execution for tracking", {
|
|
3399
3410
|
runId: msg.runId,
|
|
3400
3411
|
error: toErrorMessage(err)
|
|
3401
3412
|
});
|
|
@@ -3414,7 +3425,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3414
3425
|
*/
|
|
3415
3426
|
onPeerJobProgress(msg) {
|
|
3416
3427
|
if (this.executionTracker) (msg.kind === "job" ? this.executionTracker.onJobStatus(msg.runId, msg.jobId, msg.state, msg.timestamp, void 0, msg.data) : this.executionTracker.onStepStatus(msg.runId, msg.jobId, msg.stepIndex, msg.stepName, msg.state, msg.timestamp, msg.data)).catch((err) => {
|
|
3417
|
-
logger$
|
|
3428
|
+
logger$85.error("Failed to track peer job progress", {
|
|
3418
3429
|
error: toErrorMessage(err),
|
|
3419
3430
|
runId: msg.runId,
|
|
3420
3431
|
jobId: msg.jobId,
|
|
@@ -3428,7 +3439,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3428
3439
|
if (runJobs.size === 0) this.reroutedJobs.delete(msg.runId);
|
|
3429
3440
|
}
|
|
3430
3441
|
}
|
|
3431
|
-
logger$
|
|
3442
|
+
logger$85.debug("Peer job progress", {
|
|
3432
3443
|
runId: msg.runId,
|
|
3433
3444
|
jobId: msg.jobId,
|
|
3434
3445
|
kind: msg.kind,
|
|
@@ -3461,7 +3472,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3461
3472
|
*/
|
|
3462
3473
|
onPeerJobComplete(runId, jobId, status, timestamp, data) {
|
|
3463
3474
|
if (this.executionTracker) this.executionTracker.onJobStatus(runId, jobId, status, timestamp, void 0, data).catch((err) => {
|
|
3464
|
-
logger$
|
|
3475
|
+
logger$85.error("Failed to track peer job completion", {
|
|
3465
3476
|
error: toErrorMessage(err),
|
|
3466
3477
|
runId,
|
|
3467
3478
|
jobId
|
|
@@ -3500,14 +3511,14 @@ var init_coordinator = __esmMin((() => {
|
|
|
3500
3511
|
jobId,
|
|
3501
3512
|
reason
|
|
3502
3513
|
};
|
|
3503
|
-
if (!(client ? client.send(cancelMsg) : this.sendToPeerViaHandler?.(peerId, cancelMsg) ?? false)) logger$
|
|
3514
|
+
if (!(client ? client.send(cancelMsg) : this.sendToPeerViaHandler?.(peerId, cancelMsg) ?? false)) logger$85.warn("Failed to send cancel to peer", {
|
|
3504
3515
|
peerId,
|
|
3505
3516
|
runId,
|
|
3506
3517
|
jobId
|
|
3507
3518
|
});
|
|
3508
3519
|
}
|
|
3509
3520
|
}
|
|
3510
|
-
logger$
|
|
3521
|
+
logger$85.info("Cancel propagated to peers", {
|
|
3511
3522
|
runId,
|
|
3512
3523
|
reason,
|
|
3513
3524
|
peerCount: peerJobs.size
|
|
@@ -3527,7 +3538,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3527
3538
|
if (this.staleEvictionTimer) clearInterval(this.staleEvictionTimer);
|
|
3528
3539
|
this.staleEvictionTimer = setInterval(() => {
|
|
3529
3540
|
const evicted = this.peerRegistry.evictStalePeers(staleTimeoutMs);
|
|
3530
|
-
if (evicted.length > 0) logger$
|
|
3541
|
+
if (evicted.length > 0) logger$85.warn("Evicted stale peers", {
|
|
3531
3542
|
evicted,
|
|
3532
3543
|
count: evicted.length
|
|
3533
3544
|
});
|
|
@@ -3564,7 +3575,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3564
3575
|
if (peers.length === 0) {
|
|
3565
3576
|
const peersWithLabels = this.peerRegistry.findPeersWithLabels(labelSets);
|
|
3566
3577
|
const allPeers = this.peerRegistry.getConnectedPeers();
|
|
3567
|
-
logger$
|
|
3578
|
+
logger$85.debug("Reroute failed — peer registry state", {
|
|
3568
3579
|
jobName: job.jobName,
|
|
3569
3580
|
requiredLabels: labelSets,
|
|
3570
3581
|
connectedPeers: allPeers.map((p) => ({
|
|
@@ -3594,7 +3605,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3594
3605
|
for (const peer of sortedPeers) {
|
|
3595
3606
|
const nakEntry = this.nakTracker.get(peer.instanceId);
|
|
3596
3607
|
if (nakEntry && nakEntry.backoffUntil > now) {
|
|
3597
|
-
logger$
|
|
3608
|
+
logger$85.debug("Skipping peer in NAK backoff", {
|
|
3598
3609
|
peerId: peer.instanceId,
|
|
3599
3610
|
jobName: job.jobName,
|
|
3600
3611
|
backoffRemainingMs: nakEntry.backoffUntil - now
|
|
@@ -3604,7 +3615,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3604
3615
|
const client = this.getPeerClient(peer.instanceId);
|
|
3605
3616
|
const canSendViaHandler = !client && this.sendAndWaitAckViaHandler;
|
|
3606
3617
|
if (!client && !canSendViaHandler) {
|
|
3607
|
-
logger$
|
|
3618
|
+
logger$85.warn("No PeerClient for peer (found in registry but no connection)", {
|
|
3608
3619
|
peerId: peer.instanceId,
|
|
3609
3620
|
jobName: job.jobName
|
|
3610
3621
|
});
|
|
@@ -3625,6 +3636,8 @@ var init_coordinator = __esmMin((() => {
|
|
|
3625
3636
|
workflowName: runContext.workflowName,
|
|
3626
3637
|
runsOnLabels: labelSets,
|
|
3627
3638
|
excludeLabels: job.excludeLabels,
|
|
3639
|
+
runsOnPatterns: job.runsOnPatterns,
|
|
3640
|
+
excludePatterns: job.excludePatterns,
|
|
3628
3641
|
triedConnections: [this.instanceId],
|
|
3629
3642
|
maxHops: DEFAULT_MAX_HOPS,
|
|
3630
3643
|
coordinatorId: this.instanceId,
|
|
@@ -3645,7 +3658,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3645
3658
|
if (client ? await client.sendAndWaitAck(rerouteMsg, this.ackTimeoutMs) : await this.sendAndWaitAckViaHandler(peer.instanceId, rerouteMsg, this.ackTimeoutMs)) {
|
|
3646
3659
|
this.nakTracker.delete(peer.instanceId);
|
|
3647
3660
|
this.trackReroutedJob(runContext.runId, allocatedJobId, peer.instanceId, job.jobName);
|
|
3648
|
-
logger$
|
|
3661
|
+
logger$85.info("Job rerouted to peer", {
|
|
3649
3662
|
runId: runContext.runId,
|
|
3650
3663
|
jobName: job.jobName,
|
|
3651
3664
|
jobId: allocatedJobId,
|
|
@@ -3663,7 +3676,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
3663
3676
|
count: nakCount,
|
|
3664
3677
|
backoffUntil: Date.now() + backoffMs
|
|
3665
3678
|
});
|
|
3666
|
-
logger$
|
|
3679
|
+
logger$85.warn("Peer NAKed job", {
|
|
3667
3680
|
peerId: peer.instanceId,
|
|
3668
3681
|
jobName: job.jobName,
|
|
3669
3682
|
nakCount,
|
|
@@ -3893,6 +3906,39 @@ function resolveDataDir(explicit) {
|
|
|
3893
3906
|
var init_data_dir = __esmMin((() => {}));
|
|
3894
3907
|
//#endregion
|
|
3895
3908
|
//#region src/metrics/prometheus.ts
|
|
3909
|
+
function meter$1() {
|
|
3910
|
+
if (!_meter$1) _meter$1 = createMeter("kici-orchestrator");
|
|
3911
|
+
return _meter$1;
|
|
3912
|
+
}
|
|
3913
|
+
/**
|
|
3914
|
+
* A counter whose real OTel instrument is created on first `.add()` — after
|
|
3915
|
+
* `initTelemetry()` has registered the global MeterProvider. The returned
|
|
3916
|
+
* value satisfies the OTel `Counter` interface, so call sites are unchanged.
|
|
3917
|
+
*/
|
|
3918
|
+
function lazyCounter$1(name, options) {
|
|
3919
|
+
let inst;
|
|
3920
|
+
const get = () => inst ??= meter$1().createCounter(name, options);
|
|
3921
|
+
return { add: (...args) => get().add(...args) };
|
|
3922
|
+
}
|
|
3923
|
+
/**
|
|
3924
|
+
* A histogram whose real OTel instrument is created on first `.record()` —
|
|
3925
|
+
* after `initTelemetry()`. Satisfies the OTel `Histogram` interface.
|
|
3926
|
+
*/
|
|
3927
|
+
function lazyHistogram$1(name, options) {
|
|
3928
|
+
let inst;
|
|
3929
|
+
const get = () => inst ??= meter$1().createHistogram(name, options);
|
|
3930
|
+
return { record: (...args) => get().record(...args) };
|
|
3931
|
+
}
|
|
3932
|
+
/**
|
|
3933
|
+
* Queue an observable gauge for registration. The gauge is created and its
|
|
3934
|
+
* callback attached lazily inside registerOrchestratorMetrics(), so the
|
|
3935
|
+
* instrument binds to the real MeterProvider rather than the no-op one.
|
|
3936
|
+
*/
|
|
3937
|
+
function defineObservableGauge(name, options, observe) {
|
|
3938
|
+
_observableGauges.push(() => {
|
|
3939
|
+
meter$1().createObservableGauge(name, options).addCallback(observe);
|
|
3940
|
+
});
|
|
3941
|
+
}
|
|
3896
3942
|
/** Set the current number of active agents. */
|
|
3897
3943
|
function setAgentsActive(value) {
|
|
3898
3944
|
_agentsActiveValue = value;
|
|
@@ -3901,6 +3947,10 @@ function setAgentsActive(value) {
|
|
|
3901
3947
|
function setConfigVersion(value) {
|
|
3902
3948
|
_configVersionValue = value;
|
|
3903
3949
|
}
|
|
3950
|
+
/** Set the current number of declared (static) roster hosts that are unreachable. */
|
|
3951
|
+
function setDeclaredHostsUnreachable(value) {
|
|
3952
|
+
_declaredHostsUnreachableValue = value;
|
|
3953
|
+
}
|
|
3904
3954
|
/** Set the current number of stale runs detected. */
|
|
3905
3955
|
function setStaleRunsCurrent(value) {
|
|
3906
3956
|
_staleRunsCurrentValue = value;
|
|
@@ -3958,21 +4008,32 @@ function setDispatchQueueDepthBreakdown(snapshot) {
|
|
|
3958
4008
|
function setEventDlqDepth(value) {
|
|
3959
4009
|
_eventDlqDepthValue = value;
|
|
3960
4010
|
}
|
|
3961
|
-
|
|
4011
|
+
/**
|
|
4012
|
+
* Register every queued orchestrator observable gauge on the real meter.
|
|
4013
|
+
*
|
|
4014
|
+
* Must run AFTER `initTelemetry()` has wired the global MeterProvider —
|
|
4015
|
+
* `createApp()` calls it once during bootstrap. Registering the gauges at
|
|
4016
|
+
* module-eval time instead would bind them to the no-op provider (the
|
|
4017
|
+
* bundler hoists some module init above the entry's `initTelemetry()` call),
|
|
4018
|
+
* leaving every `kici_orch_*` gauge absent from the /metrics scrape and the
|
|
4019
|
+
* Platform push. Idempotent: repeat calls are no-ops.
|
|
4020
|
+
*/
|
|
4021
|
+
function registerOrchestratorMetrics() {
|
|
4022
|
+
if (_observableGaugesRegistered) return;
|
|
4023
|
+
_observableGaugesRegistered = true;
|
|
4024
|
+
for (const register of _observableGauges) register();
|
|
4025
|
+
}
|
|
4026
|
+
var _meter$1, _observableGauges, _agentsActiveValue, _configVersionValue, _declaredHostsUnreachableValue, _staleRunsCurrentValue, _scalerUsageRows, _scalerSpawnRefusalsValue, _queueDepthBreakdown, _everSeenQueueLabels, webhooksReceivedTotal, webhooksProcessedTotal, triggerMatchDurationSeconds, dedupHitsTotal, pgPoolClientErrorsTotal, sourceCacheHitsTotal, sourceCacheMissesTotal, depCacheHitsTotal, depCacheMissesTotal, buildDurationSeconds, executionsTotal, executionDurationSeconds, stepsTotal, githubCheckRunTotal, logChunksReceivedTotal, logBytesStoredTotal, scalerConfigReloadsTotal, ScalerSpawnFailureBound, scalerSpawnFailuresTotal, configReloadTotal, staleRunsDetectedTotal, staleDetectionDurationSeconds, crossSourceFanoutSize, crossSourceErrorsTotal, universalGitRegistrationErrorsTotal, trustMatchRefusedNoIdTotal, eventDispatchSuccessTotal, eventRetryTotal, eventDlqTotal, eventLeaseExpirationsTotal, eventAttemptsHistogram, _eventDlqDepthValue, InstallSecretsDecisionReason, InstallSecretsChannel, installSecretsDecisionsTotal, installSecretsRegistryUsedTotal, installSecretsContributorStrippedTotal, installSecretsTokenResolutionDurationSeconds, _observableGaugesRegistered;
|
|
3962
4027
|
var init_prometheus = __esmMin((() => {
|
|
3963
|
-
|
|
4028
|
+
_observableGauges = [];
|
|
3964
4029
|
_agentsActiveValue = 0;
|
|
3965
|
-
|
|
3966
|
-
result.observe(_agentsActiveValue);
|
|
3967
|
-
});
|
|
4030
|
+
defineObservableGauge("kici_orch_agents_active", { description: "Current number of active agents connected" }, (result) => result.observe(_agentsActiveValue));
|
|
3968
4031
|
_configVersionValue = 0;
|
|
3969
|
-
|
|
3970
|
-
|
|
3971
|
-
});
|
|
4032
|
+
defineObservableGauge("kici_orch_config_version", { description: "Current shared config version number" }, (result) => result.observe(_configVersionValue));
|
|
4033
|
+
_declaredHostsUnreachableValue = 0;
|
|
4034
|
+
defineObservableGauge("kici_orch_declared_hosts_unreachable", { description: "Number of declared (static) roster hosts currently unreachable" }, (result) => result.observe(_declaredHostsUnreachableValue));
|
|
3972
4035
|
_staleRunsCurrentValue = 0;
|
|
3973
|
-
|
|
3974
|
-
result.observe(_staleRunsCurrentValue);
|
|
3975
|
-
});
|
|
4036
|
+
defineObservableGauge("kici_orch_stale_runs_current", { description: "Current number of stale runs detected in last scan" }, (result) => result.observe(_staleRunsCurrentValue));
|
|
3976
4037
|
_scalerUsageRows = [];
|
|
3977
4038
|
/**
|
|
3978
4039
|
* Current CPU reservations per scaler / machine pool.
|
|
@@ -3980,7 +4041,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
3980
4041
|
* - scaler: __global__ | container | firecracker | bare-metal | stateful
|
|
3981
4042
|
* - machinePool: optional pool name (operator-defined; capped at the Platform filter)
|
|
3982
4043
|
*/
|
|
3983
|
-
|
|
4044
|
+
defineObservableGauge("kici_orch_scaler_cpus_used", { description: "Current CPU reservations summed by scaler / pool. scaler=\"__global__\" is the orchestrator-wide total; machinePool=\"<name>\" rows reflect the on-disk ledger." }, (result) => {
|
|
3984
4045
|
for (const row of _scalerUsageRows) {
|
|
3985
4046
|
const attrs = { scaler: row.scaler };
|
|
3986
4047
|
if (row.machinePool) attrs.machinePool = row.machinePool;
|
|
@@ -3993,7 +4054,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
3993
4054
|
* - scaler: __global__ | container | firecracker | bare-metal | stateful
|
|
3994
4055
|
* - machinePool: optional pool name (operator-defined; capped at the Platform filter)
|
|
3995
4056
|
*/
|
|
3996
|
-
|
|
4057
|
+
defineObservableGauge("kici_orch_scaler_memory_bytes_used", { description: "Current memory reservations (bytes) summed by scaler / pool. scaler=\"__global__\" is the orchestrator-wide total; machinePool=\"<name>\" rows reflect the on-disk ledger." }, (result) => {
|
|
3997
4058
|
for (const row of _scalerUsageRows) {
|
|
3998
4059
|
const attrs = { scaler: row.scaler };
|
|
3999
4060
|
if (row.machinePool) attrs.machinePool = row.machinePool;
|
|
@@ -4001,9 +4062,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4001
4062
|
}
|
|
4002
4063
|
});
|
|
4003
4064
|
_scalerSpawnRefusalsValue = 0;
|
|
4004
|
-
|
|
4005
|
-
result.observe(_scalerSpawnRefusalsValue);
|
|
4006
|
-
});
|
|
4065
|
+
defineObservableGauge("kici_orch_scaler_spawn_refusals_total", { description: "Cumulative count of scaler spawn requests refused due to resource caps (maxAgents, resourceCap, globalResourceCap, machinePool)." }, (result) => result.observe(_scalerSpawnRefusalsValue));
|
|
4007
4066
|
_queueDepthBreakdown = {
|
|
4008
4067
|
byStatus: {
|
|
4009
4068
|
pending: 0,
|
|
@@ -4017,7 +4076,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4017
4076
|
* Labels:
|
|
4018
4077
|
* - status: pending | dispatched
|
|
4019
4078
|
*/
|
|
4020
|
-
|
|
4079
|
+
defineObservableGauge("kici_orch_dispatch_queue_depth", { description: "Current dispatch_queue depth per status (pending, dispatched)" }, (result) => {
|
|
4021
4080
|
result.observe(_queueDepthBreakdown.byStatus.pending, { status: "pending" });
|
|
4022
4081
|
result.observe(_queueDepthBreakdown.byStatus.dispatched, { status: "dispatched" });
|
|
4023
4082
|
});
|
|
@@ -4027,15 +4086,15 @@ var init_prometheus = __esmMin((() => {
|
|
|
4027
4086
|
* - status: pending (only emitted status)
|
|
4028
4087
|
* - label: a runs_on label string from the workflow's runtime config
|
|
4029
4088
|
*/
|
|
4030
|
-
|
|
4089
|
+
defineObservableGauge("kici_orch_dispatch_queue_depth_by_label", { description: "Current pending dispatch_queue depth per runs_on label (multi-label jobs fan out)" }, (result) => {
|
|
4031
4090
|
for (const [label, count] of Object.entries(_queueDepthBreakdown.byLabel)) result.observe(count, {
|
|
4032
4091
|
status: "pending",
|
|
4033
4092
|
label
|
|
4034
4093
|
});
|
|
4035
4094
|
});
|
|
4036
|
-
webhooksReceivedTotal =
|
|
4037
|
-
webhooksProcessedTotal =
|
|
4038
|
-
triggerMatchDurationSeconds =
|
|
4095
|
+
webhooksReceivedTotal = lazyCounter$1("kici_orch_webhooks_received_total", { description: "Total number of webhooks received" });
|
|
4096
|
+
webhooksProcessedTotal = lazyCounter$1("kici_orch_webhooks_processed_total", { description: "Total number of webhooks processed" });
|
|
4097
|
+
triggerMatchDurationSeconds = lazyHistogram$1("kici_orch_trigger_match_duration_seconds", {
|
|
4039
4098
|
description: "Duration of trigger matching operations in seconds",
|
|
4040
4099
|
advice: { explicitBucketBoundaries: [
|
|
4041
4100
|
.001,
|
|
@@ -4048,13 +4107,13 @@ var init_prometheus = __esmMin((() => {
|
|
|
4048
4107
|
5
|
|
4049
4108
|
] }
|
|
4050
4109
|
});
|
|
4051
|
-
dedupHitsTotal =
|
|
4052
|
-
pgPoolClientErrorsTotal =
|
|
4053
|
-
sourceCacheHitsTotal =
|
|
4054
|
-
sourceCacheMissesTotal =
|
|
4055
|
-
depCacheHitsTotal =
|
|
4056
|
-
depCacheMissesTotal =
|
|
4057
|
-
buildDurationSeconds =
|
|
4110
|
+
dedupHitsTotal = lazyCounter$1("kici_orch_dedup_hits_total", { description: "Total number of deduplication cache hits" });
|
|
4111
|
+
pgPoolClientErrorsTotal = lazyCounter$1("kici_orch_pg_pool_client_errors_total", { description: "Total pg connection errors absorbed without a process restart" });
|
|
4112
|
+
sourceCacheHitsTotal = lazyCounter$1("kici_orch_source_cache_hits_total", { description: "Total number of source tarball cache hits" });
|
|
4113
|
+
sourceCacheMissesTotal = lazyCounter$1("kici_orch_source_cache_misses_total", { description: "Total number of source tarball cache misses" });
|
|
4114
|
+
depCacheHitsTotal = lazyCounter$1("kici_orch_dep_cache_hits_total", { description: "Total number of dep cache hits" });
|
|
4115
|
+
depCacheMissesTotal = lazyCounter$1("kici_orch_dep_cache_misses_total", { description: "Total number of dep cache misses" });
|
|
4116
|
+
buildDurationSeconds = lazyHistogram$1("kici_orch_build_duration_seconds", {
|
|
4058
4117
|
description: "Duration of build agent operations in seconds",
|
|
4059
4118
|
advice: { explicitBucketBoundaries: [
|
|
4060
4119
|
1,
|
|
@@ -4067,8 +4126,8 @@ var init_prometheus = __esmMin((() => {
|
|
|
4067
4126
|
600
|
|
4068
4127
|
] }
|
|
4069
4128
|
});
|
|
4070
|
-
executionsTotal =
|
|
4071
|
-
executionDurationSeconds =
|
|
4129
|
+
executionsTotal = lazyCounter$1("kici_orch_executions_total", { description: "Total number of execution runs" });
|
|
4130
|
+
executionDurationSeconds = lazyHistogram$1("kici_orch_execution_duration_seconds", {
|
|
4072
4131
|
description: "Duration of execution runs in seconds",
|
|
4073
4132
|
advice: { explicitBucketBoundaries: [
|
|
4074
4133
|
1,
|
|
@@ -4082,19 +4141,19 @@ var init_prometheus = __esmMin((() => {
|
|
|
4082
4141
|
1800
|
|
4083
4142
|
] }
|
|
4084
4143
|
});
|
|
4085
|
-
stepsTotal =
|
|
4086
|
-
githubCheckRunTotal =
|
|
4087
|
-
logChunksReceivedTotal =
|
|
4088
|
-
logBytesStoredTotal =
|
|
4089
|
-
scalerConfigReloadsTotal =
|
|
4144
|
+
stepsTotal = lazyCounter$1("kici_orch_steps_total", { description: "Total number of steps executed" });
|
|
4145
|
+
githubCheckRunTotal = lazyCounter$1("kici_orch_github_check_run_total", { description: "Total number of GitHub check run API calls" });
|
|
4146
|
+
logChunksReceivedTotal = lazyCounter$1("kici_orch_log_chunks_received_total", { description: "Total number of log chunks received from agents" });
|
|
4147
|
+
logBytesStoredTotal = lazyCounter$1("kici_orch_log_bytes_stored_total", { description: "Total bytes of log data written to storage" });
|
|
4148
|
+
scalerConfigReloadsTotal = lazyCounter$1("kici_orch_scaler_config_reloads_total", { description: "Total number of scaler config reload operations" });
|
|
4090
4149
|
ScalerSpawnFailureBound = {
|
|
4091
4150
|
Bound: "true",
|
|
4092
4151
|
Unbound: "false"
|
|
4093
4152
|
};
|
|
4094
|
-
scalerSpawnFailuresTotal =
|
|
4095
|
-
configReloadTotal =
|
|
4096
|
-
staleRunsDetectedTotal =
|
|
4097
|
-
staleDetectionDurationSeconds =
|
|
4153
|
+
scalerSpawnFailuresTotal = lazyCounter$1("kici_orch_scaler_spawn_failures_total", { description: "Total scaler agent spawn failures (job-bound and warm-pool)" });
|
|
4154
|
+
configReloadTotal = lazyCounter$1("kici_orch_config_reload_total", { description: "Total number of config reload operations" });
|
|
4155
|
+
staleRunsDetectedTotal = lazyCounter$1("kici_orch_stale_runs_detected_total", { description: "Total number of stale runs detected and marked as failed" });
|
|
4156
|
+
staleDetectionDurationSeconds = lazyHistogram$1("kici_orch_stale_detection_duration_seconds", {
|
|
4098
4157
|
description: "Time between job becoming stale and detection (seconds)",
|
|
4099
4158
|
advice: { explicitBucketBoundaries: [
|
|
4100
4159
|
10,
|
|
@@ -4105,7 +4164,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4105
4164
|
600
|
|
4106
4165
|
] }
|
|
4107
4166
|
});
|
|
4108
|
-
crossSourceFanoutSize =
|
|
4167
|
+
crossSourceFanoutSize = lazyHistogram$1("kici_cross_source_fanout_size", {
|
|
4109
4168
|
description: "Number of webhook-trigger registrations matched when an inbound generic webhook fans out across sources in the same org",
|
|
4110
4169
|
advice: { explicitBucketBoundaries: [
|
|
4111
4170
|
0,
|
|
@@ -4118,18 +4177,16 @@ var init_prometheus = __esmMin((() => {
|
|
|
4118
4177
|
100
|
|
4119
4178
|
] }
|
|
4120
4179
|
});
|
|
4121
|
-
crossSourceErrorsTotal =
|
|
4122
|
-
universalGitRegistrationErrorsTotal =
|
|
4123
|
-
trustMatchRefusedNoIdTotal =
|
|
4124
|
-
eventDispatchSuccessTotal =
|
|
4125
|
-
eventRetryTotal =
|
|
4126
|
-
eventDlqTotal =
|
|
4127
|
-
eventLeaseExpirationsTotal =
|
|
4128
|
-
eventAttemptsHistogram =
|
|
4180
|
+
crossSourceErrorsTotal = lazyCounter$1("kici_cross_source_errors_total", { description: "Errors encountered during cross-source webhook dispatch" });
|
|
4181
|
+
universalGitRegistrationErrorsTotal = lazyCounter$1("kici_universal_git_registration_errors_total", { description: "Errors encountered while registering universal-git provider bundles" });
|
|
4182
|
+
trustMatchRefusedNoIdTotal = lazyCounter$1("kici_orch_trust_match_refused_no_id_total", { description: "Trust-resolution attempts refused because the provider numeric id was missing or did not match" });
|
|
4183
|
+
eventDispatchSuccessTotal = lazyCounter$1("kici_orch_event_dispatch_success_total", { description: "Internal events delivered to all matching workflows successfully (after the lease commits processed=true)" });
|
|
4184
|
+
eventRetryTotal = lazyCounter$1("kici_orch_event_retry_total", { description: "Internal-event dispatch failures that triggered a retry (lease released, next_retry_at scheduled)" });
|
|
4185
|
+
eventDlqTotal = lazyCounter$1("kici_orch_event_dlq_total", { description: "Internal events moved to the DLQ after exhausting retries" });
|
|
4186
|
+
eventLeaseExpirationsTotal = lazyCounter$1("kici_orch_event_lease_expirations_total", { description: "Dispatch leases that timed out before the holding node finalised them (signals node crash mid-dispatch)" });
|
|
4187
|
+
eventAttemptsHistogram = lazyHistogram$1("kici_orch_event_attempts", { description: "Distribution of dispatch attempts at terminal outcome (success or DLQ)" });
|
|
4129
4188
|
_eventDlqDepthValue = 0;
|
|
4130
|
-
|
|
4131
|
-
result.observe(_eventDlqDepthValue);
|
|
4132
|
-
});
|
|
4189
|
+
defineObservableGauge("kici_orch_event_dlq_depth", { description: "Current count of events sitting in the DLQ awaiting operator triage" }, (result) => result.observe(_eventDlqDepthValue));
|
|
4133
4190
|
InstallSecretsDecisionReason = {
|
|
4134
4191
|
Ok: "ok",
|
|
4135
4192
|
MalformedRef: "malformed_ref",
|
|
@@ -4147,10 +4204,10 @@ var init_prometheus = __esmMin((() => {
|
|
|
4147
4204
|
Registries: "registries",
|
|
4148
4205
|
InstallEnv: "install_env"
|
|
4149
4206
|
};
|
|
4150
|
-
installSecretsDecisionsTotal =
|
|
4151
|
-
installSecretsRegistryUsedTotal =
|
|
4152
|
-
installSecretsContributorStrippedTotal =
|
|
4153
|
-
installSecretsTokenResolutionDurationSeconds =
|
|
4207
|
+
installSecretsDecisionsTotal = lazyCounter$1("kici_orch_install_secrets_decisions_total", { description: "Total install-secrets resolution decisions (pass / reject / hold + reason)" });
|
|
4208
|
+
installSecretsRegistryUsedTotal = lazyCounter$1("kici_orch_install_secrets_npm_registry_used_total", { description: "Total registry / installEnv entries successfully resolved per channel / provider / scope" });
|
|
4209
|
+
installSecretsContributorStrippedTotal = lazyCounter$1("kici_orch_install_secrets_contributor_stripped_total", { description: "Dispatches where registry tokens were stripped because the contributor tier was not `trusted`" });
|
|
4210
|
+
installSecretsTokenResolutionDurationSeconds = lazyHistogram$1("kici_orch_install_secrets_token_resolution_duration_seconds", {
|
|
4154
4211
|
description: "Duration of per-environment secret resolution during install-secrets evaluation",
|
|
4155
4212
|
advice: { explicitBucketBoundaries: [
|
|
4156
4213
|
.001,
|
|
@@ -4163,6 +4220,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
4163
4220
|
5
|
|
4164
4221
|
] }
|
|
4165
4222
|
});
|
|
4223
|
+
_observableGaugesRegistered = false;
|
|
4166
4224
|
}));
|
|
4167
4225
|
//#endregion
|
|
4168
4226
|
//#region src/config/reload.ts
|
|
@@ -4428,6 +4486,8 @@ var init_schema = __esmMin((() => {
|
|
|
4428
4486
|
}).optional(),
|
|
4429
4487
|
agentAuth: z.enum(["token", "none"]).optional(),
|
|
4430
4488
|
agentTokenTtlMs: z.coerce.number().optional(),
|
|
4489
|
+
rosterGraceMs: z.coerce.number().optional(),
|
|
4490
|
+
rosterTtlMs: z.coerce.number().optional(),
|
|
4431
4491
|
queue: z.object({
|
|
4432
4492
|
maxDepth: z.coerce.number().optional(),
|
|
4433
4493
|
timeoutMs: z.coerce.number().optional(),
|
|
@@ -4504,6 +4564,8 @@ var init_schema = __esmMin((() => {
|
|
|
4504
4564
|
}).optional(),
|
|
4505
4565
|
agentAuth: z.enum(["token", "none"]).default("token"),
|
|
4506
4566
|
agentTokenTtlMs: z.coerce.number().default(36e5),
|
|
4567
|
+
rosterGraceMs: z.coerce.number().default(3e5),
|
|
4568
|
+
rosterTtlMs: z.coerce.number().default(18e5),
|
|
4507
4569
|
queueMaxDepth: z.coerce.number().default(1e3),
|
|
4508
4570
|
queueTimeoutMs: z.coerce.number().default(36e5),
|
|
4509
4571
|
/**
|
|
@@ -4925,6 +4987,9 @@ function getDefaults() {
|
|
|
4925
4987
|
return {
|
|
4926
4988
|
agentAuth: "token",
|
|
4927
4989
|
agentTokenTtlMs: 36e5,
|
|
4990
|
+
rosterGraceMs: 3e5,
|
|
4991
|
+
rosterTtlMs: 18e5,
|
|
4992
|
+
maxFanoutHosts: 1024,
|
|
4928
4993
|
queue: {
|
|
4929
4994
|
maxDepth: 1e3,
|
|
4930
4995
|
timeoutMs: 36e5,
|
|
@@ -4974,6 +5039,9 @@ function flattenToAppConfig(merged) {
|
|
|
4974
5039
|
if (merged.storage) flat.storage = merged.storage;
|
|
4975
5040
|
flat.agentAuth = merged.agentAuth ?? "token";
|
|
4976
5041
|
flat.agentTokenTtlMs = merged.agentTokenTtlMs ?? 36e5;
|
|
5042
|
+
flat.rosterGraceMs = merged.rosterGraceMs ?? 3e5;
|
|
5043
|
+
flat.rosterTtlMs = merged.rosterTtlMs ?? 18e5;
|
|
5044
|
+
flat.maxFanoutHosts = merged.maxFanoutHosts ?? 1024;
|
|
4977
5045
|
const queue = merged.queue;
|
|
4978
5046
|
flat.queueMaxDepth = queue?.maxDepth ?? 1e3;
|
|
4979
5047
|
flat.queueTimeoutMs = queue?.timeoutMs ?? 36e5;
|
|
@@ -5068,8 +5136,9 @@ function satisfiesMandatoryLabels(agent, requiredLabels) {
|
|
|
5068
5136
|
for (const label of agent.mandatoryLabels) if (!requiredSet.has(label)) return false;
|
|
5069
5137
|
return true;
|
|
5070
5138
|
}
|
|
5071
|
-
var AgentRegistry;
|
|
5139
|
+
var logger$84, AgentRegistry;
|
|
5072
5140
|
var init_registry = __esmMin((() => {
|
|
5141
|
+
logger$84 = createLogger({ prefix: "agent-registry" });
|
|
5073
5142
|
AgentRegistry = class {
|
|
5074
5143
|
/** Primary: agentId -> AgentEntry */
|
|
5075
5144
|
agents = /* @__PURE__ */ new Map();
|
|
@@ -5107,6 +5176,27 @@ var init_registry = __esmMin((() => {
|
|
|
5107
5176
|
*/
|
|
5108
5177
|
tokenExpiryTimers = /* @__PURE__ */ new Map();
|
|
5109
5178
|
/**
|
|
5179
|
+
* Per-agent last roster `last_seen` stamp time (epoch ms). Throttles the
|
|
5180
|
+
* coarse heartbeat write so we don't issue a DB write per agent per
|
|
5181
|
+
* heartbeat — `grace`/`ttl` are minute-scale, so a throttled stamp is
|
|
5182
|
+
* sufficient. Cleaned up alongside the agent in `removeFromIndexes`.
|
|
5183
|
+
*/
|
|
5184
|
+
lastRosterStampAt = /* @__PURE__ */ new Map();
|
|
5185
|
+
/** Minimum interval between coarse roster `last_seen` stamps per agent. */
|
|
5186
|
+
ROSTER_STAMP_THROTTLE_MS = 6e4;
|
|
5187
|
+
/** Optional host-roster reconciler + this instance's id (see AgentRegistryDeps). */
|
|
5188
|
+
rosterStore;
|
|
5189
|
+
instanceId;
|
|
5190
|
+
/**
|
|
5191
|
+
* The roster reconcile seam is optional: when no store is injected (workers
|
|
5192
|
+
* with no DB, unit tests), every reconcile hook is a no-op and the in-memory
|
|
5193
|
+
* registry behaves exactly as before.
|
|
5194
|
+
*/
|
|
5195
|
+
constructor(deps = {}) {
|
|
5196
|
+
this.rosterStore = deps.rosterStore;
|
|
5197
|
+
this.instanceId = deps.instanceId;
|
|
5198
|
+
}
|
|
5199
|
+
/**
|
|
5110
5200
|
* Register an agent. If the agentId already exists, updates the existing
|
|
5111
5201
|
* entry (agent reconnection scenario).
|
|
5112
5202
|
*/
|
|
@@ -5138,7 +5228,8 @@ var init_registry = __esmMin((() => {
|
|
|
5138
5228
|
runningAsUid: metadata?.runningAsUid ?? null,
|
|
5139
5229
|
memoryUsedMb: null,
|
|
5140
5230
|
memoryAvailableMb: null,
|
|
5141
|
-
uptimeSeconds: null
|
|
5231
|
+
uptimeSeconds: null,
|
|
5232
|
+
tokenAgentType: metadata?.tokenAgentType ?? null
|
|
5142
5233
|
};
|
|
5143
5234
|
this.agents.set(agentId, entry);
|
|
5144
5235
|
this.wsToAgentId.set(ws, agentId);
|
|
@@ -5158,6 +5249,22 @@ var init_registry = __esmMin((() => {
|
|
|
5158
5249
|
}
|
|
5159
5250
|
tokenAgents.add(agentId);
|
|
5160
5251
|
}
|
|
5252
|
+
if (this.rosterStore && this.instanceId) {
|
|
5253
|
+
const instanceId = this.instanceId;
|
|
5254
|
+
this.rosterStore.upsert({
|
|
5255
|
+
agentId,
|
|
5256
|
+
tokenId,
|
|
5257
|
+
lifecycleClass: metadata?.tokenAgentType ?? "ephemeral",
|
|
5258
|
+
labels,
|
|
5259
|
+
hostname: metadata?.hostname ?? null,
|
|
5260
|
+
platform,
|
|
5261
|
+
arch,
|
|
5262
|
+
instanceId
|
|
5263
|
+
}).catch((err) => logger$84.warn("host_roster upsert failed (best-effort)", {
|
|
5264
|
+
agentId,
|
|
5265
|
+
error: toErrorMessage(err)
|
|
5266
|
+
}));
|
|
5267
|
+
}
|
|
5161
5268
|
}
|
|
5162
5269
|
/**
|
|
5163
5270
|
* Unregister an agent by ID. Removes from all indexes.
|
|
@@ -5168,6 +5275,7 @@ var init_registry = __esmMin((() => {
|
|
|
5168
5275
|
if (!entry) return void 0;
|
|
5169
5276
|
this.removeFromIndexes(agentId, entry);
|
|
5170
5277
|
this.agents.delete(agentId);
|
|
5278
|
+
if (this.rosterStore && this.instanceId) this.rosterStore.markDisconnected(agentId, this.instanceId).catch(() => {});
|
|
5171
5279
|
return entry;
|
|
5172
5280
|
}
|
|
5173
5281
|
/**
|
|
@@ -5268,7 +5376,7 @@ var init_registry = __esmMin((() => {
|
|
|
5268
5376
|
* @param requiredLabels - All labels the agent must have (intersection semantics).
|
|
5269
5377
|
* @returns Array of matching available AgentEntry objects.
|
|
5270
5378
|
*/
|
|
5271
|
-
findAvailable(requiredLabels, excludeLabels = []) {
|
|
5379
|
+
findAvailable(requiredLabels, requiredPatterns = [], excludeLabels = [], excludePatterns = []) {
|
|
5272
5380
|
let candidates;
|
|
5273
5381
|
if (requiredLabels.length === 0) candidates = [...this.agents.values()].filter((e) => e.activeJobs < e.maxConcurrency);
|
|
5274
5382
|
else {
|
|
@@ -5287,14 +5395,29 @@ var init_registry = __esmMin((() => {
|
|
|
5287
5395
|
if (entry.activeJobs < entry.maxConcurrency) candidates.push(entry);
|
|
5288
5396
|
}
|
|
5289
5397
|
}
|
|
5398
|
+
if (requiredPatterns.length > 0) candidates = candidates.filter((a) => requiredPatterns.every((p) => matcherSatisfiedBy(p, a.labels)));
|
|
5290
5399
|
let filtered = candidates;
|
|
5291
5400
|
if (excludeLabels.length > 0) filtered = filtered.filter((agent) => {
|
|
5292
5401
|
for (const excluded of excludeLabels) if (agent.labels.has(excluded)) return false;
|
|
5293
5402
|
return true;
|
|
5294
5403
|
});
|
|
5404
|
+
if (excludePatterns.length > 0) filtered = filtered.filter((a) => !excludePatterns.some((p) => matcherSatisfiedBy(p, a.labels)));
|
|
5295
5405
|
return filtered.filter((agent) => satisfiesMandatoryLabels(agent, requiredLabels));
|
|
5296
5406
|
}
|
|
5297
5407
|
/**
|
|
5408
|
+
* Whether a specific agent satisfies the same label / exclude / mandatory
|
|
5409
|
+
* gate `findAvailable` applies per-agent, ignoring capacity. Used by the
|
|
5410
|
+
* host-fanout pin to verify the resolved agent still matches before pinning
|
|
5411
|
+
* (guards against roster label drift between resolution and dispatch).
|
|
5412
|
+
*/
|
|
5413
|
+
agentSatisfies(agent, requiredLabels, requiredPatterns = [], excludeLabels = [], excludePatterns = []) {
|
|
5414
|
+
for (const required of requiredLabels) if (!agent.labels.has(required)) return false;
|
|
5415
|
+
for (const p of requiredPatterns) if (!matcherSatisfiedBy(p, agent.labels)) return false;
|
|
5416
|
+
for (const excluded of excludeLabels) if (agent.labels.has(excluded)) return false;
|
|
5417
|
+
for (const p of excludePatterns) if (matcherSatisfiedBy(p, agent.labels)) return false;
|
|
5418
|
+
return satisfiesMandatoryLabels(agent, requiredLabels);
|
|
5419
|
+
}
|
|
5420
|
+
/**
|
|
5298
5421
|
* Check if ANY registered agent matches the required labels, regardless of capacity.
|
|
5299
5422
|
* Used to decide whether to queue (agent exists but busy) vs reject (no agent at all).
|
|
5300
5423
|
*
|
|
@@ -5305,23 +5428,20 @@ var init_registry = __esmMin((() => {
|
|
|
5305
5428
|
* an off-gate gated agent as "matching but busy" and skip the peer
|
|
5306
5429
|
* reroute path even though the local agent can never accept the job.
|
|
5307
5430
|
*/
|
|
5308
|
-
hasMatchingAgent(requiredLabels, excludeLabels = []) {
|
|
5431
|
+
hasMatchingAgent(requiredLabels, requiredPatterns = [], excludeLabels = [], excludePatterns = []) {
|
|
5309
5432
|
const candidateIds = this.intersectLabelCandidates(requiredLabels);
|
|
5310
5433
|
if (candidateIds === null) return false;
|
|
5434
|
+
const ok = (entry) => {
|
|
5435
|
+
if (excludeLabels.some((e) => entry.labels.has(e))) return false;
|
|
5436
|
+
if (requiredPatterns.some((p) => !matcherSatisfiedBy(p, entry.labels))) return false;
|
|
5437
|
+
if (excludePatterns.some((p) => matcherSatisfiedBy(p, entry.labels))) return false;
|
|
5438
|
+
return satisfiesMandatoryLabels(entry, requiredLabels);
|
|
5439
|
+
};
|
|
5311
5440
|
if (requiredLabels.length === 0) {
|
|
5312
|
-
for (const entry of this.agents.values())
|
|
5313
|
-
if (excludeLabels.some((e) => entry.labels.has(e))) continue;
|
|
5314
|
-
if (!satisfiesMandatoryLabels(entry, requiredLabels)) continue;
|
|
5315
|
-
return true;
|
|
5316
|
-
}
|
|
5441
|
+
for (const entry of this.agents.values()) if (ok(entry)) return true;
|
|
5317
5442
|
return false;
|
|
5318
5443
|
}
|
|
5319
|
-
for (const id of candidateIds)
|
|
5320
|
-
const entry = this.agents.get(id);
|
|
5321
|
-
if (excludeLabels.length > 0 && excludeLabels.some((e) => entry.labels.has(e))) continue;
|
|
5322
|
-
if (!satisfiesMandatoryLabels(entry, requiredLabels)) continue;
|
|
5323
|
-
return true;
|
|
5324
|
-
}
|
|
5444
|
+
for (const id of candidateIds) if (ok(this.agents.get(id))) return true;
|
|
5325
5445
|
return false;
|
|
5326
5446
|
}
|
|
5327
5447
|
/**
|
|
@@ -5371,6 +5491,16 @@ var init_registry = __esmMin((() => {
|
|
|
5371
5491
|
const entry = this.agents.get(agentId);
|
|
5372
5492
|
if (!entry) return false;
|
|
5373
5493
|
entry.lastHeartbeatAt = Date.now();
|
|
5494
|
+
if (this.rosterStore && this.instanceId) {
|
|
5495
|
+
const now = Date.now();
|
|
5496
|
+
const last = this.lastRosterStampAt.get(agentId) ?? now;
|
|
5497
|
+
if (!this.lastRosterStampAt.has(agentId)) this.lastRosterStampAt.set(agentId, now);
|
|
5498
|
+
else if (now - last >= this.ROSTER_STAMP_THROTTLE_MS) {
|
|
5499
|
+
this.lastRosterStampAt.set(agentId, now);
|
|
5500
|
+
const instanceId = this.instanceId;
|
|
5501
|
+
this.rosterStore.stampLastSeen(agentId, instanceId).catch(() => {});
|
|
5502
|
+
}
|
|
5503
|
+
}
|
|
5374
5504
|
return true;
|
|
5375
5505
|
}
|
|
5376
5506
|
/**
|
|
@@ -5405,6 +5535,7 @@ var init_registry = __esmMin((() => {
|
|
|
5405
5535
|
*/
|
|
5406
5536
|
removeFromIndexes(agentId, entry) {
|
|
5407
5537
|
this.wsToAgentId.delete(entry.ws);
|
|
5538
|
+
this.lastRosterStampAt.delete(agentId);
|
|
5408
5539
|
for (const label of entry.labels) {
|
|
5409
5540
|
const agentIds = this.labelIndex.get(label);
|
|
5410
5541
|
if (agentIds) {
|
|
@@ -5428,7 +5559,222 @@ var init_registry = __esmMin((() => {
|
|
|
5428
5559
|
}
|
|
5429
5560
|
}
|
|
5430
5561
|
};
|
|
5431
|
-
}))
|
|
5562
|
+
}));
|
|
5563
|
+
//#endregion
|
|
5564
|
+
//#region src/agent/host-roster.ts
|
|
5565
|
+
/**
|
|
5566
|
+
* The ONE status-derivation function — used by the store and the kici-admin
|
|
5567
|
+
* host CLI. `ready` requires the host to be genuinely live (connected to some
|
|
5568
|
+
* instance AND heartbeat fresh — the freshness check catches a crashed
|
|
5569
|
+
* instance that never cleared `connected_instance_id`). It never returns
|
|
5570
|
+
* `ready` for a not-currently-live host: a not-live `static` reads
|
|
5571
|
+
* `unreachable` (the declared-but-absent alarm), a not-live `ephemeral` reads
|
|
5572
|
+
* `stale` (scaled down, awaiting reap).
|
|
5573
|
+
*/
|
|
5574
|
+
function deriveHostStatus(row, nowMs, graceMs) {
|
|
5575
|
+
const ageMs = nowMs - new Date(row.last_seen).getTime();
|
|
5576
|
+
if (row.connected_instance_id !== null && ageMs <= graceMs) return "ready";
|
|
5577
|
+
if (row.lifecycle_class === "ephemeral") return "stale";
|
|
5578
|
+
return "unreachable";
|
|
5579
|
+
}
|
|
5580
|
+
var HostRosterStore;
|
|
5581
|
+
var init_host_roster = __esmMin((() => {
|
|
5582
|
+
HostRosterStore = class {
|
|
5583
|
+
db;
|
|
5584
|
+
constructor(db) {
|
|
5585
|
+
this.db = db;
|
|
5586
|
+
}
|
|
5587
|
+
/** Idempotent upsert on agent_id; stamps connected_instance_id + last_seen. */
|
|
5588
|
+
async upsert(input) {
|
|
5589
|
+
const labelsJson = JSON.stringify(input.labels);
|
|
5590
|
+
await this.db.insertInto("host_roster").values({
|
|
5591
|
+
agent_id: input.agentId,
|
|
5592
|
+
token_id: input.tokenId,
|
|
5593
|
+
lifecycle_class: input.lifecycleClass,
|
|
5594
|
+
labels: labelsJson,
|
|
5595
|
+
hostname: input.hostname,
|
|
5596
|
+
platform: input.platform,
|
|
5597
|
+
arch: input.arch,
|
|
5598
|
+
connected_instance_id: input.instanceId,
|
|
5599
|
+
last_seen: sql`now()`,
|
|
5600
|
+
updated_at: sql`now()`
|
|
5601
|
+
}).onConflict((oc) => oc.column("agent_id").doUpdateSet({
|
|
5602
|
+
token_id: input.tokenId,
|
|
5603
|
+
lifecycle_class: input.lifecycleClass,
|
|
5604
|
+
labels: labelsJson,
|
|
5605
|
+
hostname: input.hostname,
|
|
5606
|
+
platform: input.platform,
|
|
5607
|
+
arch: input.arch,
|
|
5608
|
+
connected_instance_id: input.instanceId,
|
|
5609
|
+
last_seen: sql`now()`,
|
|
5610
|
+
updated_at: sql`now()`
|
|
5611
|
+
})).execute();
|
|
5612
|
+
}
|
|
5613
|
+
/** Clear liveness on disconnect — but only if THIS instance still owns it. */
|
|
5614
|
+
async markDisconnected(agentId, instanceId) {
|
|
5615
|
+
await this.db.updateTable("host_roster").set({
|
|
5616
|
+
connected_instance_id: null,
|
|
5617
|
+
updated_at: sql`now()`
|
|
5618
|
+
}).where("agent_id", "=", agentId).where("connected_instance_id", "=", instanceId).execute();
|
|
5619
|
+
}
|
|
5620
|
+
/** Coarse heartbeat stamp — same owner-guard as markDisconnected. */
|
|
5621
|
+
async stampLastSeen(agentId, instanceId) {
|
|
5622
|
+
await this.db.updateTable("host_roster").set({ last_seen: sql`now()` }).where("agent_id", "=", agentId).where("connected_instance_id", "=", instanceId).execute();
|
|
5623
|
+
}
|
|
5624
|
+
/** Operator pre-declare of a static host before its agent dials in. */
|
|
5625
|
+
async declareStatic(input) {
|
|
5626
|
+
await this.db.insertInto("host_roster").values({
|
|
5627
|
+
agent_id: input.agentId,
|
|
5628
|
+
token_id: null,
|
|
5629
|
+
lifecycle_class: "static",
|
|
5630
|
+
labels: JSON.stringify(input.labels),
|
|
5631
|
+
hostname: input.hostname ?? null,
|
|
5632
|
+
connected_instance_id: null,
|
|
5633
|
+
last_seen: sql`now()`,
|
|
5634
|
+
updated_at: sql`now()`
|
|
5635
|
+
}).onConflict((oc) => oc.column("agent_id").doNothing()).execute();
|
|
5636
|
+
}
|
|
5637
|
+
async get(agentId) {
|
|
5638
|
+
return await this.db.selectFrom("host_roster").selectAll().where("agent_id", "=", agentId).executeTakeFirst() ?? null;
|
|
5639
|
+
}
|
|
5640
|
+
async listAll() {
|
|
5641
|
+
return this.db.selectFrom("host_roster").selectAll().orderBy("agent_id", "asc").execute();
|
|
5642
|
+
}
|
|
5643
|
+
/**
|
|
5644
|
+
* Resolve every roster host matching a `runsOnAll` predicate (OR-of-AND
|
|
5645
|
+
* include groups, minus exclude labels), tagged with its derived status. This
|
|
5646
|
+
* is the host-fanout resolver: it returns declared-but-absent static hosts
|
|
5647
|
+
* (status `unreachable`) so the caller can apply `onUnreachable` — the live
|
|
5648
|
+
* registry alone cannot name an expected-but-absent host.
|
|
5649
|
+
*/
|
|
5650
|
+
async findMatching(include, exclude, graceMs) {
|
|
5651
|
+
const rows = await this.db.selectFrom("host_roster").selectAll().execute();
|
|
5652
|
+
const now = Date.now();
|
|
5653
|
+
const out = [];
|
|
5654
|
+
for (const row of rows) {
|
|
5655
|
+
const labels = JSON.parse(row.labels);
|
|
5656
|
+
const set = new Set(labels);
|
|
5657
|
+
if (exclude.some((e) => matcherSatisfiedBy(e, set))) continue;
|
|
5658
|
+
if (include.length && !include.some((grp) => grp.every((m) => matcherSatisfiedBy(m, set)))) continue;
|
|
5659
|
+
out.push({
|
|
5660
|
+
agentId: row.agent_id,
|
|
5661
|
+
host: row.hostname ?? row.agent_id,
|
|
5662
|
+
labels,
|
|
5663
|
+
lifecycleClass: row.lifecycle_class,
|
|
5664
|
+
connectedInstanceId: row.connected_instance_id,
|
|
5665
|
+
status: deriveHostStatus(row, now, graceMs),
|
|
5666
|
+
platform: row.platform,
|
|
5667
|
+
arch: row.arch
|
|
5668
|
+
});
|
|
5669
|
+
}
|
|
5670
|
+
out.sort((a, b) => a.agentId.localeCompare(b.agentId));
|
|
5671
|
+
return out;
|
|
5672
|
+
}
|
|
5673
|
+
/**
|
|
5674
|
+
* Count `static` (declared) hosts whose derived status is `unreachable` —
|
|
5675
|
+
* the "declared-but-absent" alarm population. Reuses the single-source
|
|
5676
|
+
* {@link deriveHostStatus} so the count never diverges from what
|
|
5677
|
+
* `kici-admin host list` shows. A not-currently-connected static host reads
|
|
5678
|
+
* `unreachable` regardless of grace (only the connected-but-stale case
|
|
5679
|
+
* depends on `graceMs`).
|
|
5680
|
+
*/
|
|
5681
|
+
async countStaticUnreachable(graceMs) {
|
|
5682
|
+
const rows = await this.db.selectFrom("host_roster").selectAll().where("lifecycle_class", "=", "static").execute();
|
|
5683
|
+
const now = Date.now();
|
|
5684
|
+
return rows.filter((r) => deriveHostStatus(r, now, graceMs) === "unreachable").length;
|
|
5685
|
+
}
|
|
5686
|
+
/** Delete ephemeral rows whose last_seen is older than ttl. Returns count. */
|
|
5687
|
+
async reapEphemeralPastTtl(ttlMs) {
|
|
5688
|
+
const res = await this.db.deleteFrom("host_roster").where("lifecycle_class", "=", "ephemeral").where("last_seen", "<", sql`now() - (${ttlMs}::text || ' milliseconds')::interval`).executeTakeFirst();
|
|
5689
|
+
return Number(res.numDeletedRows ?? 0n);
|
|
5690
|
+
}
|
|
5691
|
+
};
|
|
5692
|
+
}));
|
|
5693
|
+
//#endregion
|
|
5694
|
+
//#region src/agent/host-roster-reaper.ts
|
|
5695
|
+
var logger$83, HostRosterReaper;
|
|
5696
|
+
var init_host_roster_reaper = __esmMin((() => {
|
|
5697
|
+
logger$83 = createLogger({ prefix: "host-roster-reaper" });
|
|
5698
|
+
HostRosterReaper = class {
|
|
5699
|
+
store;
|
|
5700
|
+
ttlMs;
|
|
5701
|
+
graceMs;
|
|
5702
|
+
scanIntervalMs;
|
|
5703
|
+
setUnreachableGauge;
|
|
5704
|
+
timer = null;
|
|
5705
|
+
isLeader = false;
|
|
5706
|
+
constructor(opts) {
|
|
5707
|
+
this.store = opts.store;
|
|
5708
|
+
this.ttlMs = opts.ttlMs;
|
|
5709
|
+
this.graceMs = opts.graceMs;
|
|
5710
|
+
this.scanIntervalMs = opts.scanIntervalMs;
|
|
5711
|
+
this.setUnreachableGauge = opts.setUnreachableGauge;
|
|
5712
|
+
}
|
|
5713
|
+
onBecomeLeader() {
|
|
5714
|
+
if (this.timer) {
|
|
5715
|
+
clearInterval(this.timer);
|
|
5716
|
+
this.timer = null;
|
|
5717
|
+
}
|
|
5718
|
+
this.isLeader = true;
|
|
5719
|
+
logger$83.info("Became leader, starting host roster reaper", {
|
|
5720
|
+
ttlMs: this.ttlMs,
|
|
5721
|
+
scanIntervalMs: this.scanIntervalMs
|
|
5722
|
+
});
|
|
5723
|
+
this.timer = setInterval(() => {
|
|
5724
|
+
this.tick().catch((err) => logger$83.error("roster reaper tick failed", { error: toErrorMessage(err) }));
|
|
5725
|
+
}, this.scanIntervalMs);
|
|
5726
|
+
this.timer.unref?.();
|
|
5727
|
+
}
|
|
5728
|
+
onLoseLeadership() {
|
|
5729
|
+
this.isLeader = false;
|
|
5730
|
+
if (this.timer) {
|
|
5731
|
+
clearInterval(this.timer);
|
|
5732
|
+
this.timer = null;
|
|
5733
|
+
}
|
|
5734
|
+
logger$83.info("Lost leadership, stopped host roster reaper");
|
|
5735
|
+
}
|
|
5736
|
+
stop() {
|
|
5737
|
+
if (this.timer) {
|
|
5738
|
+
clearInterval(this.timer);
|
|
5739
|
+
this.timer = null;
|
|
5740
|
+
}
|
|
5741
|
+
this.isLeader = false;
|
|
5742
|
+
}
|
|
5743
|
+
/** One reap pass. Public for tests. */
|
|
5744
|
+
async tick() {
|
|
5745
|
+
if (!this.isLeader) return;
|
|
5746
|
+
const deleted = await this.store.reapEphemeralPastTtl(this.ttlMs);
|
|
5747
|
+
if (deleted > 0) logger$83.info("Reaped expired ephemeral hosts", { deleted });
|
|
5748
|
+
const unreachable = await this.store.countStaticUnreachable(this.graceMs);
|
|
5749
|
+
this.setUnreachableGauge(unreachable);
|
|
5750
|
+
if (unreachable > 0) logger$83.warn("Declared hosts unreachable", { count: unreachable });
|
|
5751
|
+
}
|
|
5752
|
+
};
|
|
5753
|
+
}));
|
|
5754
|
+
//#endregion
|
|
5755
|
+
//#region src/queue/job-queue.ts
|
|
5756
|
+
/**
|
|
5757
|
+
* Parse a `dispatch_queue` jsonb pattern column into a `LabelMatcher[]`. Handles
|
|
5758
|
+
* both the auto-parsed array form (from the pg driver) and the JSON string form
|
|
5759
|
+
* (from tests / a non-parsing driver). A missing / malformed value yields `[]`.
|
|
5760
|
+
*/
|
|
5761
|
+
function parseMatcherColumn(v) {
|
|
5762
|
+
if (Array.isArray(v)) return v;
|
|
5763
|
+
if (typeof v === "string") return JSON.parse(v);
|
|
5764
|
+
return [];
|
|
5765
|
+
}
|
|
5766
|
+
/**
|
|
5767
|
+
* Whether an agent's label set satisfies a job's regex matchers: every
|
|
5768
|
+
* `runsOnPatterns` matcher must match some label AND no `excludePatterns`
|
|
5769
|
+
* matcher may match any label. The single matching authority is the engine's
|
|
5770
|
+
* `matcherSatisfiedBy` (JS RegExp) — never a Postgres `~`.
|
|
5771
|
+
*/
|
|
5772
|
+
function jobPatternsSatisfiedBy(job, labels) {
|
|
5773
|
+
if (!job.runsOnPatterns.every((p) => matcherSatisfiedBy(p, labels))) return false;
|
|
5774
|
+
if (job.excludePatterns.some((p) => matcherSatisfiedBy(p, labels))) return false;
|
|
5775
|
+
return true;
|
|
5776
|
+
}
|
|
5777
|
+
var JobQueue;
|
|
5432
5778
|
var init_job_queue = __esmMin((() => {
|
|
5433
5779
|
JobQueue = class {
|
|
5434
5780
|
db;
|
|
@@ -5480,7 +5826,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
5480
5826
|
deps_hash: job.depsHash ?? null,
|
|
5481
5827
|
request_id: job.requestId ?? null,
|
|
5482
5828
|
exclude_labels: JSON.stringify(job.excludeLabels ?? []),
|
|
5483
|
-
|
|
5829
|
+
runs_on_patterns: JSON.stringify(job.runsOnPatterns ?? []),
|
|
5830
|
+
exclude_patterns: JSON.stringify(job.excludePatterns ?? []),
|
|
5831
|
+
routing_key: job.routingKey,
|
|
5832
|
+
pinned_agent_id: job.pinnedAgentId ?? null
|
|
5484
5833
|
}).execute();
|
|
5485
5834
|
this.depthCache = null;
|
|
5486
5835
|
this.breakdownCache = null;
|
|
@@ -5507,18 +5856,79 @@ var init_job_queue = __esmMin((() => {
|
|
|
5507
5856
|
* (empty for static / non-scaler agents).
|
|
5508
5857
|
* @returns The matching job, or null if none found.
|
|
5509
5858
|
*/
|
|
5510
|
-
async dequeueForLabels(agentLabels, agentMandatoryLabels = []) {
|
|
5859
|
+
async dequeueForLabels(agentLabels, agentMandatoryLabels = [], agentId) {
|
|
5860
|
+
const fast = await this.claimPatternFree(agentLabels, agentMandatoryLabels, agentId);
|
|
5861
|
+
if (fast) return fast;
|
|
5862
|
+
return this.claimWithPatterns(agentLabels, agentMandatoryLabels, agentId);
|
|
5863
|
+
}
|
|
5864
|
+
/**
|
|
5865
|
+
* Build the shared drain WHERE chain (status / expiry / exact-label @> /
|
|
5866
|
+
* exclude-label / pin / mandatory-label gate) common to both drain passes.
|
|
5867
|
+
* The pattern columns are NOT filtered here — each pass adds its own
|
|
5868
|
+
* pattern-free / pattern-bearing guard on top.
|
|
5869
|
+
*/
|
|
5870
|
+
drainBaseQuery(agentLabels, agentMandatoryLabels, agentId) {
|
|
5511
5871
|
const agentLabelsJson = JSON.stringify(agentLabels);
|
|
5512
5872
|
const mandatoryLabelsJson = JSON.stringify(agentMandatoryLabels);
|
|
5513
5873
|
let query = this.db.selectFrom("dispatch_queue").selectAll().where("status", "=", "pending").where(sql`(expires_at IS NULL OR expires_at >= now())`).where(sql`${sql.lit(agentLabelsJson)}::jsonb @> runs_on_labels`).where(sql`NOT EXISTS (
|
|
5514
5874
|
SELECT 1 FROM jsonb_array_elements_text(exclude_labels) AS e
|
|
5515
5875
|
WHERE e.value = ANY(${sql.val(agentLabels)}::text[])
|
|
5516
5876
|
)`);
|
|
5877
|
+
query = query.where(sql`(pinned_agent_id IS NULL${agentId ? sql` OR pinned_agent_id = ${sql.val(agentId)}` : sql``})`);
|
|
5517
5878
|
if (agentMandatoryLabels.length > 0) query = query.where(sql`runs_on_labels @> ${sql.lit(mandatoryLabelsJson)}::jsonb`);
|
|
5518
|
-
|
|
5879
|
+
return query;
|
|
5880
|
+
}
|
|
5881
|
+
/**
|
|
5882
|
+
* Fast path: claim the oldest pending pattern-free row. The
|
|
5883
|
+
* `runs_on_patterns = '[]' AND exclude_patterns = '[]'` guard restricts this
|
|
5884
|
+
* pass to rows that need no JS post-filter, so the single-row atomic claim
|
|
5885
|
+
* (FOR UPDATE SKIP LOCKED) keeps the original hot-path semantics intact.
|
|
5886
|
+
*/
|
|
5887
|
+
async claimPatternFree(agentLabels, agentMandatoryLabels, agentId) {
|
|
5888
|
+
const row = await this.drainBaseQuery(agentLabels, agentMandatoryLabels, agentId).where(sql`runs_on_patterns = '[]'::jsonb AND exclude_patterns = '[]'::jsonb`).orderBy("created_at", "asc").limit(1).forUpdate().skipLocked().executeTakeFirst();
|
|
5519
5889
|
return row ? this.rowToQueuedJob(row) : null;
|
|
5520
5890
|
}
|
|
5521
5891
|
/**
|
|
5892
|
+
* Pattern path: load a small batch of pattern-bearing candidate rows, apply
|
|
5893
|
+
* the JS regex post-filter (matcherSatisfiedBy), and atomically claim the
|
|
5894
|
+
* first match by id with a conditional `status = Pending` guard. The claim is
|
|
5895
|
+
* a conditional UPDATE rather than relying on the SELECT lock alone because
|
|
5896
|
+
* the JS filter runs after the per-statement lock window has closed, so two
|
|
5897
|
+
* agents could both pass the filter for the same row; the `where status =
|
|
5898
|
+
* Pending` makes exactly one of them win. The claim transitions the row to
|
|
5899
|
+
* Dispatched, matching the value the caller-side markDispatched would set
|
|
5900
|
+
* (which then re-sets it idempotently).
|
|
5901
|
+
*/
|
|
5902
|
+
async claimWithPatterns(agentLabels, agentMandatoryLabels, agentId) {
|
|
5903
|
+
const labelSet = new Set(agentLabels);
|
|
5904
|
+
const rows = await this.drainBaseQuery(agentLabels, agentMandatoryLabels, agentId).where(sql`(runs_on_patterns <> '[]'::jsonb OR exclude_patterns <> '[]'::jsonb)`).orderBy("created_at", "asc").limit(10).forUpdate().skipLocked().execute();
|
|
5905
|
+
for (const row of rows) {
|
|
5906
|
+
const job = this.rowToQueuedJob(row);
|
|
5907
|
+
if (!jobPatternsSatisfiedBy(job, labelSet)) continue;
|
|
5908
|
+
if (((await this.db.updateTable("dispatch_queue").set({ status: "dispatched" }).where("id", "=", row.id).where("status", "=", "pending").executeTakeFirst()).numUpdatedRows ?? 0n) > 0n) return job;
|
|
5909
|
+
}
|
|
5910
|
+
return null;
|
|
5911
|
+
}
|
|
5912
|
+
/**
|
|
5913
|
+
* Atomically claim the oldest pending job pinned to a specific agent. Used by
|
|
5914
|
+
* the eager pin drain when the pinned agent (re)registers or frees a slot —
|
|
5915
|
+
* the host-fanout analog of `dispatchBoundJob`'s eager path. Ignores the exact
|
|
5916
|
+
* label gate: the pin was resolved against the roster at materialize time.
|
|
5917
|
+
*
|
|
5918
|
+
* Still applies the JS regex post-filter (`jobPatternsSatisfiedBy`) when
|
|
5919
|
+
* `agentLabels` is supplied, mirroring `dequeueById`: a pinned child whose
|
|
5920
|
+
* `runsOn`/`exclude` patterns no longer match the agent's current labels must
|
|
5921
|
+
* not be claimed. The single matching authority is the engine's
|
|
5922
|
+
* `matcherSatisfiedBy` (never a Postgres `~`).
|
|
5923
|
+
*/
|
|
5924
|
+
async dequeueByPinnedAgent(agentId, agentLabels) {
|
|
5925
|
+
const row = await this.db.selectFrom("dispatch_queue").selectAll().where("status", "=", "pending").where("pinned_agent_id", "=", agentId).where(sql`(expires_at IS NULL OR expires_at >= now())`).orderBy("created_at", "asc").limit(1).forUpdate().skipLocked().executeTakeFirst();
|
|
5926
|
+
if (!row) return null;
|
|
5927
|
+
const job = this.rowToQueuedJob(row);
|
|
5928
|
+
if (agentLabels && !jobPatternsSatisfiedBy(job, new Set(agentLabels))) return null;
|
|
5929
|
+
return job;
|
|
5930
|
+
}
|
|
5931
|
+
/**
|
|
5522
5932
|
* Atomically claim a specific pending job by ID, validating it still
|
|
5523
5933
|
* matches the agent's labels and isn't expired.
|
|
5524
5934
|
*
|
|
@@ -5544,7 +5954,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
5544
5954
|
)`);
|
|
5545
5955
|
if (agentMandatoryLabels.length > 0) query = query.where(sql`runs_on_labels @> ${sql.lit(mandatoryLabelsJson)}::jsonb`);
|
|
5546
5956
|
const row = await query.forUpdate().skipLocked().executeTakeFirst();
|
|
5547
|
-
|
|
5957
|
+
if (!row) return null;
|
|
5958
|
+
const job = this.rowToQueuedJob(row);
|
|
5959
|
+
if (!jobPatternsSatisfiedBy(job, new Set(agentLabels))) return null;
|
|
5960
|
+
return job;
|
|
5548
5961
|
}
|
|
5549
5962
|
/**
|
|
5550
5963
|
* Insert a job directly with status='dispatched' (bypasses the queue).
|
|
@@ -5576,7 +5989,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
5576
5989
|
deps_hash: job.depsHash ?? null,
|
|
5577
5990
|
request_id: job.requestId ?? null,
|
|
5578
5991
|
exclude_labels: JSON.stringify(job.excludeLabels ?? []),
|
|
5579
|
-
|
|
5992
|
+
runs_on_patterns: JSON.stringify(job.runsOnPatterns ?? []),
|
|
5993
|
+
exclude_patterns: JSON.stringify(job.excludePatterns ?? []),
|
|
5994
|
+
routing_key: job.routingKey,
|
|
5995
|
+
pinned_agent_id: job.pinnedAgentId ?? null
|
|
5580
5996
|
}).execute();
|
|
5581
5997
|
return id;
|
|
5582
5998
|
}
|
|
@@ -6006,7 +6422,10 @@ var init_job_queue = __esmMin((() => {
|
|
|
6006
6422
|
depsHash: row.deps_hash ?? void 0,
|
|
6007
6423
|
requestId: row.request_id ?? void 0,
|
|
6008
6424
|
excludeLabels: typeof row.exclude_labels === "string" ? JSON.parse(row.exclude_labels) : Array.isArray(row.exclude_labels) ? row.exclude_labels : [],
|
|
6009
|
-
|
|
6425
|
+
runsOnPatterns: parseMatcherColumn(row.runs_on_patterns),
|
|
6426
|
+
excludePatterns: parseMatcherColumn(row.exclude_patterns),
|
|
6427
|
+
routingKey: row.routing_key,
|
|
6428
|
+
pinnedAgentId: row.pinned_agent_id ?? void 0
|
|
6010
6429
|
};
|
|
6011
6430
|
}
|
|
6012
6431
|
};
|
|
@@ -6111,14 +6530,66 @@ var init_cleanup$1 = __esmMin((() => {
|
|
|
6111
6530
|
* Prometheus can address multiple KiCI services from one dashboard.
|
|
6112
6531
|
* The orchestrator runs setInterval-driven schedulers (no pg-boss
|
|
6113
6532
|
* dependency) but the outward observability shape is identical.
|
|
6533
|
+
*
|
|
6534
|
+
* Lazy meter + instrument initialization: the `@kici-dev/shared` barrel
|
|
6535
|
+
* is imported statically at the top of the orchestrator entry points
|
|
6536
|
+
* (server.ts, standalone.ts), which evaluates this module BEFORE
|
|
6537
|
+
* `initTelemetry()` sets the global MeterProvider. Creating the meter or
|
|
6538
|
+
* any instrument at module-eval time would bind it to the no-op provider,
|
|
6539
|
+
* so its samples never reach the Prometheus exporter (the `/metrics`
|
|
6540
|
+
* scrape) or the Platform push. We therefore resolve the meter and every
|
|
6541
|
+
* instrument on first use — counters/histograms on first `.add()` /
|
|
6542
|
+
* `.record()`, observable gauges on first `.set()` — all of which happen
|
|
6543
|
+
* at job-run time, long after telemetry is wired up. Same hazard, same
|
|
6544
|
+
* fix as `prometheus.ts` and `@kici-dev/shared`'s cold-store metrics
|
|
6545
|
+
* module.
|
|
6546
|
+
*/
|
|
6547
|
+
function meter() {
|
|
6548
|
+
if (!_meter) _meter = createMeter("kici-orchestrator");
|
|
6549
|
+
return _meter;
|
|
6550
|
+
}
|
|
6551
|
+
/**
|
|
6552
|
+
* A counter whose real OTel instrument is created on first `.add()` —
|
|
6553
|
+
* after `initTelemetry()` has registered the global MeterProvider. The
|
|
6554
|
+
* returned value satisfies the OTel `Counter` interface, so call sites
|
|
6555
|
+
* are unchanged.
|
|
6556
|
+
*/
|
|
6557
|
+
function lazyCounter(name, description) {
|
|
6558
|
+
let inst;
|
|
6559
|
+
const get = () => inst ??= meter().createCounter(name, { description });
|
|
6560
|
+
return { add: (...args) => get().add(...args) };
|
|
6561
|
+
}
|
|
6562
|
+
/**
|
|
6563
|
+
* A histogram whose real OTel instrument is created on first `.record()` —
|
|
6564
|
+
* after `initTelemetry()`. Satisfies the OTel `Histogram` interface.
|
|
6565
|
+
*/
|
|
6566
|
+
function lazyHistogram(name, description, explicitBucketBoundaries) {
|
|
6567
|
+
let inst;
|
|
6568
|
+
const get = () => inst ??= meter().createHistogram(name, {
|
|
6569
|
+
description,
|
|
6570
|
+
advice: { explicitBucketBoundaries }
|
|
6571
|
+
});
|
|
6572
|
+
return { record: (...args) => get().record(...args) };
|
|
6573
|
+
}
|
|
6574
|
+
/**
|
|
6575
|
+
* An observable gauge backed by per-label state. The underlying OTel
|
|
6576
|
+
* instrument + scrape callback are created on first `.set()`, so the
|
|
6577
|
+
* gauge binds to the real MeterProvider rather than the no-op one. The
|
|
6578
|
+
* callback reads the latest per-label value on each Prometheus scrape.
|
|
6114
6579
|
*/
|
|
6115
6580
|
function createGaugeWithState(name, description) {
|
|
6116
6581
|
const state = /* @__PURE__ */ new Map();
|
|
6117
|
-
|
|
6118
|
-
|
|
6119
|
-
|
|
6582
|
+
let registered = false;
|
|
6583
|
+
function ensureRegistered() {
|
|
6584
|
+
if (registered) return;
|
|
6585
|
+
registered = true;
|
|
6586
|
+
meter().createObservableGauge(name, { description }).addCallback((result) => {
|
|
6587
|
+
for (const entry of state.values()) result.observe(entry.value, entry.attributes);
|
|
6588
|
+
});
|
|
6589
|
+
}
|
|
6120
6590
|
return {
|
|
6121
6591
|
set(labels, value) {
|
|
6592
|
+
ensureRegistered();
|
|
6122
6593
|
const key = JSON.stringify(labels);
|
|
6123
6594
|
state.set(key, {
|
|
6124
6595
|
attributes: labels,
|
|
@@ -6130,24 +6601,20 @@ function createGaugeWithState(name, description) {
|
|
|
6130
6601
|
}
|
|
6131
6602
|
};
|
|
6132
6603
|
}
|
|
6133
|
-
var
|
|
6604
|
+
var _meter, jobRunsTotal, jobDurationSeconds, jobLastSuccessTimestamp, jobLastFailureTimestamp, jobConsecutiveFailures;
|
|
6134
6605
|
var init_scheduled_jobs = __esmMin((() => {
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
120,
|
|
6148
|
-
600
|
|
6149
|
-
] }
|
|
6150
|
-
});
|
|
6606
|
+
jobRunsTotal = lazyCounter("kici_orch_job_runs_total", "Scheduled job run outcomes, by job and result (success/failure)");
|
|
6607
|
+
jobDurationSeconds = lazyHistogram("kici_orch_job_duration_seconds", "Scheduled job tick duration in seconds", [
|
|
6608
|
+
.01,
|
|
6609
|
+
.05,
|
|
6610
|
+
.1,
|
|
6611
|
+
.5,
|
|
6612
|
+
1,
|
|
6613
|
+
5,
|
|
6614
|
+
30,
|
|
6615
|
+
120,
|
|
6616
|
+
600
|
|
6617
|
+
]);
|
|
6151
6618
|
jobLastSuccessTimestamp = createGaugeWithState("kici_orch_job_last_success_timestamp_seconds", "Unix timestamp of the most recent successful scheduled job tick");
|
|
6152
6619
|
jobLastFailureTimestamp = createGaugeWithState("kici_orch_job_last_failure_timestamp_seconds", "Unix timestamp of the most recent failed scheduled job tick");
|
|
6153
6620
|
jobConsecutiveFailures = createGaugeWithState("kici_orch_job_consecutive_failures", "Consecutive failures for each scheduled job since its last success");
|
|
@@ -8428,7 +8895,8 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8428
8895
|
* 4. If queue full: return 'rejected'.
|
|
8429
8896
|
*/
|
|
8430
8897
|
async dispatch(job) {
|
|
8431
|
-
|
|
8898
|
+
if (job.pinnedAgentId) return this.dispatchPinned(job);
|
|
8899
|
+
const available = this.registry.findAvailable(job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? []);
|
|
8432
8900
|
if (available.length > 0) {
|
|
8433
8901
|
const agent = available[0];
|
|
8434
8902
|
this.registry.incrementActiveJobs(agent.agentId);
|
|
@@ -8461,6 +8929,8 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8461
8929
|
depsHash: job.depsHash,
|
|
8462
8930
|
requestId: job.requestId,
|
|
8463
8931
|
excludeLabels: job.excludeLabels ?? [],
|
|
8932
|
+
runsOnPatterns: job.runsOnPatterns ?? [],
|
|
8933
|
+
excludePatterns: job.excludePatterns ?? [],
|
|
8464
8934
|
routingKey: job.routingKey,
|
|
8465
8935
|
resources: job.resources
|
|
8466
8936
|
};
|
|
@@ -8490,7 +8960,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8490
8960
|
throw err;
|
|
8491
8961
|
}
|
|
8492
8962
|
if (this.onNoMatchingAgent) {
|
|
8493
|
-
if ((await this.onNoMatchingAgent(job.runsOnLabels, jobId, job.runId, job.excludeLabels ?? [], job.resources)).action === "no-backend" && !this.registry.hasMatchingAgent(job.runsOnLabels, job.excludeLabels ?? [])) return {
|
|
8963
|
+
if ((await this.onNoMatchingAgent(job.runsOnLabels, jobId, job.runId, job.excludeLabels ?? [], job.resources)).action === "no-backend" && !this.registry.hasMatchingAgent(job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? [])) return {
|
|
8494
8964
|
status: "queued-no-backend",
|
|
8495
8965
|
jobId
|
|
8496
8966
|
};
|
|
@@ -8501,6 +8971,84 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8501
8971
|
};
|
|
8502
8972
|
}
|
|
8503
8973
|
/**
|
|
8974
|
+
* Dispatch a host-fanout pinned child. The job targets exactly
|
|
8975
|
+
* `job.pinnedAgentId`: if that agent is locally connected, satisfies the
|
|
8976
|
+
* runsOn/exclude/mandatory gate, and has capacity, dispatch immediately;
|
|
8977
|
+
* otherwise queue it WITH the pin so the pin-aware drain delivers it when the
|
|
8978
|
+
* agent frees up or (re)connects. A pinned job never falls through to a
|
|
8979
|
+
* different agent.
|
|
8980
|
+
*/
|
|
8981
|
+
async dispatchPinned(job) {
|
|
8982
|
+
const agentId = job.pinnedAgentId;
|
|
8983
|
+
const agent = this.registry.get(agentId);
|
|
8984
|
+
if (agent && this.registry.agentSatisfies(agent, job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? []) && agent.activeJobs < agent.maxConcurrency) {
|
|
8985
|
+
this.registry.incrementActiveJobs(agentId);
|
|
8986
|
+
let jobId;
|
|
8987
|
+
try {
|
|
8988
|
+
jobId = await this.queue.insertDispatched(job);
|
|
8989
|
+
} catch (err) {
|
|
8990
|
+
this.registry.decrementActiveJobs(agentId);
|
|
8991
|
+
throw err;
|
|
8992
|
+
}
|
|
8993
|
+
const queuedJob = {
|
|
8994
|
+
id: jobId,
|
|
8995
|
+
runId: job.runId,
|
|
8996
|
+
workflowName: job.workflowName,
|
|
8997
|
+
jobName: job.jobName,
|
|
8998
|
+
runsOnLabels: job.runsOnLabels,
|
|
8999
|
+
jobConfig: job.jobConfig,
|
|
9000
|
+
repoUrl: job.repoUrl,
|
|
9001
|
+
ref: job.ref,
|
|
9002
|
+
sha: job.sha,
|
|
9003
|
+
status: "dispatched",
|
|
9004
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9005
|
+
expiresAt: null,
|
|
9006
|
+
deliveryId: job.deliveryId,
|
|
9007
|
+
provider: job.provider,
|
|
9008
|
+
providerContext: job.providerContext,
|
|
9009
|
+
sourceTarUrl: job.sourceTarUrl,
|
|
9010
|
+
sourceTarHash: job.sourceTarHash,
|
|
9011
|
+
depsUrl: job.depsUrl,
|
|
9012
|
+
depsHash: job.depsHash,
|
|
9013
|
+
requestId: job.requestId,
|
|
9014
|
+
excludeLabels: job.excludeLabels ?? [],
|
|
9015
|
+
runsOnPatterns: job.runsOnPatterns ?? [],
|
|
9016
|
+
excludePatterns: job.excludePatterns ?? [],
|
|
9017
|
+
routingKey: job.routingKey,
|
|
9018
|
+
resources: job.resources,
|
|
9019
|
+
pinnedAgentId: agentId
|
|
9020
|
+
};
|
|
9021
|
+
this.trackJobForAgent(agentId, jobId, queuedJob.runId);
|
|
9022
|
+
await this.onDispatch(agentId, queuedJob);
|
|
9023
|
+
await this.armAckDeadline(agentId, queuedJob);
|
|
9024
|
+
this.metrics.incJobsDispatched("dispatched");
|
|
9025
|
+
return {
|
|
9026
|
+
status: "dispatched",
|
|
9027
|
+
agentId,
|
|
9028
|
+
jobId
|
|
9029
|
+
};
|
|
9030
|
+
}
|
|
9031
|
+
let jobId;
|
|
9032
|
+
try {
|
|
9033
|
+
jobId = await this.queue.enqueue(job);
|
|
9034
|
+
this.metrics.incJobsDispatched("queued");
|
|
9035
|
+
await this.updateQueueDepthMetric();
|
|
9036
|
+
} catch (err) {
|
|
9037
|
+
if (toErrorMessage(err) === "queue full") {
|
|
9038
|
+
this.metrics.incJobsDispatched("rejected");
|
|
9039
|
+
return {
|
|
9040
|
+
status: "rejected",
|
|
9041
|
+
reason: "queue full"
|
|
9042
|
+
};
|
|
9043
|
+
}
|
|
9044
|
+
throw err;
|
|
9045
|
+
}
|
|
9046
|
+
return {
|
|
9047
|
+
status: "queued",
|
|
9048
|
+
jobId
|
|
9049
|
+
};
|
|
9050
|
+
}
|
|
9051
|
+
/**
|
|
8504
9052
|
* Eagerly dispatch a specific bound job to a freshly-registered
|
|
8505
9053
|
* scaler-managed agent.
|
|
8506
9054
|
*
|
|
@@ -8554,7 +9102,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8554
9102
|
this.registry.incrementActiveJobs(agentId);
|
|
8555
9103
|
let job = null;
|
|
8556
9104
|
try {
|
|
8557
|
-
job = await this.queue.dequeueForLabels(agentLabels, agentMandatoryLabels);
|
|
9105
|
+
job = await this.queue.dequeueByPinnedAgent(agentId, agentLabels) ?? await this.queue.dequeueForLabels(agentLabels, agentMandatoryLabels, agentId);
|
|
8558
9106
|
} finally {
|
|
8559
9107
|
if (!job) this.registry.decrementActiveJobs(agentId);
|
|
8560
9108
|
}
|
|
@@ -8770,7 +9318,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
8770
9318
|
async redispatch(jobId) {
|
|
8771
9319
|
const job = await this.queue.getFullJobById(jobId);
|
|
8772
9320
|
if (!job || job.status !== "pending") return;
|
|
8773
|
-
const available = this.registry.findAvailable(job.runsOnLabels, job.excludeLabels ?? []);
|
|
9321
|
+
const available = this.registry.findAvailable(job.runsOnLabels, job.runsOnPatterns ?? [], job.excludeLabels ?? [], job.excludePatterns ?? []);
|
|
8774
9322
|
if (available.length > 0) {
|
|
8775
9323
|
if (await this.dispatchBoundJob(available[0].agentId, jobId)) return;
|
|
8776
9324
|
}
|
|
@@ -9220,6 +9768,37 @@ var init_dispatcher = __esmMin((() => {
|
|
|
9220
9768
|
};
|
|
9221
9769
|
}));
|
|
9222
9770
|
//#endregion
|
|
9771
|
+
//#region src/lockfile-redos-guard.ts
|
|
9772
|
+
/**
|
|
9773
|
+
* Lock-load ReDoS revalidation.
|
|
9774
|
+
*
|
|
9775
|
+
* Re-validates every regex matcher in a fetched lock file before it is cached
|
|
9776
|
+
* or dispatched. Defense-in-depth against a hand-edited or non-compiled lock
|
|
9777
|
+
* that smuggled a ReDoS-prone pattern past the compile-time gate: the compiler
|
|
9778
|
+
* runs the same `assertMatchersSafe` check when it emits the lock, but the
|
|
9779
|
+
* orchestrator does not trust that the lock it fetched was produced by our
|
|
9780
|
+
* compiler.
|
|
9781
|
+
*/
|
|
9782
|
+
/**
|
|
9783
|
+
* Walk every static job's `runsOn` / `excludeLabels` / `runsOnAll` matchers and
|
|
9784
|
+
* throw if any regex matcher is ReDoS-prone. Dynamic job generators carry no
|
|
9785
|
+
* static routing matchers (they materialize jobs at eval time, which re-runs the
|
|
9786
|
+
* compile-time gate), so only static jobs are checked.
|
|
9787
|
+
*/
|
|
9788
|
+
function assertLockFileRegexesSafe(lockFile) {
|
|
9789
|
+
for (const wf of lockFile.workflows ?? []) for (const job of wf.jobs ?? []) {
|
|
9790
|
+
if (!isLockStaticJob(job)) continue;
|
|
9791
|
+
const ctx = `lock workflow '${wf.name}' job '${job.name}'`;
|
|
9792
|
+
if (job.runsOn) assertMatchersSafe(job.runsOn, `${ctx} runsOn`);
|
|
9793
|
+
if (job.excludeLabels) assertMatchersSafe(job.excludeLabels, `${ctx} excludeLabels`);
|
|
9794
|
+
if (job.runsOnAll) {
|
|
9795
|
+
for (const grp of job.runsOnAll.include ?? []) assertMatchersSafe(grp, `${ctx} runsOnAll`);
|
|
9796
|
+
if (job.runsOnAll.exclude) assertMatchersSafe(job.runsOnAll.exclude, `${ctx} runsOnAll exclude`);
|
|
9797
|
+
}
|
|
9798
|
+
}
|
|
9799
|
+
}
|
|
9800
|
+
var init_lockfile_redos_guard = __esmMin((() => {}));
|
|
9801
|
+
//#endregion
|
|
9223
9802
|
//#region src/lockfile-cache.ts
|
|
9224
9803
|
/**
|
|
9225
9804
|
* Provider-agnostic LRU cache for lock files.
|
|
@@ -9232,6 +9811,7 @@ var init_dispatcher = __esmMin((() => {
|
|
|
9232
9811
|
*/
|
|
9233
9812
|
var logger$76, LockFileCache;
|
|
9234
9813
|
var init_lockfile_cache = __esmMin((() => {
|
|
9814
|
+
init_lockfile_redos_guard();
|
|
9235
9815
|
logger$76 = createLogger({ prefix: "lockfile-cache" });
|
|
9236
9816
|
LockFileCache = class {
|
|
9237
9817
|
cache;
|
|
@@ -9287,6 +9867,17 @@ var init_lockfile_cache = __esmMin((() => {
|
|
|
9287
9867
|
});
|
|
9288
9868
|
return null;
|
|
9289
9869
|
}
|
|
9870
|
+
try {
|
|
9871
|
+
assertLockFileRegexesSafe(lockFile);
|
|
9872
|
+
} catch (error) {
|
|
9873
|
+
const message = toErrorMessage(error);
|
|
9874
|
+
logger$76.warn("Lock file carries a ReDoS-prone regex matcher", {
|
|
9875
|
+
repoIdentifier,
|
|
9876
|
+
ref,
|
|
9877
|
+
error: message
|
|
9878
|
+
});
|
|
9879
|
+
throw new LockFileParseError(repoIdentifier, ref, message);
|
|
9880
|
+
}
|
|
9290
9881
|
this.cache.set(cacheKey, lockFile);
|
|
9291
9882
|
return lockFile;
|
|
9292
9883
|
}
|
|
@@ -11697,6 +12288,15 @@ var init_agent_job_failed_error = __esmMin((() => {
|
|
|
11697
12288
|
//#endregion
|
|
11698
12289
|
//#region src/ws/agent-handler.ts
|
|
11699
12290
|
/**
|
|
12291
|
+
* Narrow the token's `agent_type` column (a free string at the DB layer) to
|
|
12292
|
+
* the host-roster lifecycle class. Anything other than the two known values
|
|
12293
|
+
* (including `undefined` under `agentAuth: 'none'`) maps to `null` so the
|
|
12294
|
+
* roster's reconcile hook treats it as the GC-able default class.
|
|
12295
|
+
*/
|
|
12296
|
+
function toLifecycleClass(agentType) {
|
|
12297
|
+
return agentType === "static" || agentType === "ephemeral" ? agentType : null;
|
|
12298
|
+
}
|
|
12299
|
+
/**
|
|
11700
12300
|
* Run the three token-bound authorization gates against a wire-supplied
|
|
11701
12301
|
* `agent.register` payload. The gates are identical at first register
|
|
11702
12302
|
* (Phase 2 / `pendingRegistration`) and on every subsequent re-register
|
|
@@ -12080,7 +12680,8 @@ function createAgentWsHandler(deps) {
|
|
|
12080
12680
|
runningAsUid: parsed.data.runningAsUid,
|
|
12081
12681
|
tokenId: regEntry.tokenId ?? null,
|
|
12082
12682
|
mandatoryLabels: scalerInfo?.mandatoryLabels,
|
|
12083
|
-
scalerManaged: scalerInfo !== null
|
|
12683
|
+
scalerManaged: scalerInfo !== null,
|
|
12684
|
+
tokenAgentType: toLifecycleClass(regEntry.tokenAgentType)
|
|
12084
12685
|
});
|
|
12085
12686
|
wsToAgentId.set(ws, agentId);
|
|
12086
12687
|
if (regEntry.tokenId !== void 0 && regEntry.tokenExpiresAt !== void 0 && regEntry.tokenExpiresAt !== null) registry.scheduleExpiryKick(regEntry.tokenId, regEntry.tokenExpiresAt);
|
|
@@ -12192,7 +12793,8 @@ function createAgentWsHandler(deps) {
|
|
|
12192
12793
|
runningAsUser: msg.runningAsUser,
|
|
12193
12794
|
runningAsUid: msg.runningAsUid,
|
|
12194
12795
|
mandatoryLabels: existingEntry ? [...existingEntry.mandatoryLabels] : void 0,
|
|
12195
|
-
scalerManaged: existingEntry?.scalerManaged ?? false
|
|
12796
|
+
scalerManaged: existingEntry?.scalerManaged ?? false,
|
|
12797
|
+
tokenAgentType: toLifecycleClass(reregisterAuthState?.tokenAgentType) ?? existingEntry?.tokenAgentType ?? null
|
|
12196
12798
|
});
|
|
12197
12799
|
wsToAgentId.set(ws, msg.agentId);
|
|
12198
12800
|
setAgentsActive(registry.getActiveCount());
|
|
@@ -13614,7 +14216,8 @@ var init_rbac = __esmMin((() => {
|
|
|
13614
14216
|
* Shared error handler for admin route files.
|
|
13615
14217
|
*
|
|
13616
14218
|
* Handles common error types: RBAC permission denied, Zod validation,
|
|
13617
|
-
* PostgreSQL unique constraint violations,
|
|
14219
|
+
* PostgreSQL unique constraint violations, PostgreSQL invalid-text-representation
|
|
14220
|
+
* (malformed typed input → 400), and generic errors.
|
|
13618
14221
|
*/
|
|
13619
14222
|
function handleAdminError(c, err, logger) {
|
|
13620
14223
|
if (err instanceof PermissionDeniedError) return c.json({ error: err.message }, 403);
|
|
@@ -13623,6 +14226,7 @@ function handleAdminError(c, err, logger) {
|
|
|
13623
14226
|
details: err.issues
|
|
13624
14227
|
}, 400);
|
|
13625
14228
|
if (err instanceof Error && "code" in err && err.code === "23505") return c.json({ error: "Conflict: resource already exists" }, 409);
|
|
14229
|
+
if (err instanceof Error && "code" in err && err.code === "22P02") return c.json({ error: "Invalid request: malformed value for a typed field" }, 400);
|
|
13626
14230
|
logger.error("Admin API error", {
|
|
13627
14231
|
error: toErrorMessage(err),
|
|
13628
14232
|
stack: err instanceof Error ? err.stack : void 0
|
|
@@ -13865,10 +14469,10 @@ var init_admin_sources = __esmMin((() => {
|
|
|
13865
14469
|
//#endregion
|
|
13866
14470
|
//#region src/db/migrations/001_initial.ts
|
|
13867
14471
|
var _001_initial_exports = /* @__PURE__ */ __exportAll({
|
|
13868
|
-
down: () => down$
|
|
13869
|
-
up: () => up$
|
|
14472
|
+
down: () => down$41,
|
|
14473
|
+
up: () => up$41
|
|
13870
14474
|
});
|
|
13871
|
-
async function up$
|
|
14475
|
+
async function up$41(db) {
|
|
13872
14476
|
for (const stmt of DDL_STATEMENTS) await sql.raw(stmt).execute(db);
|
|
13873
14477
|
await sql`
|
|
13874
14478
|
INSERT INTO cluster_meta (key, value)
|
|
@@ -13890,7 +14494,7 @@ async function up$37(db) {
|
|
|
13890
14494
|
* Rollback drops everything created above. Uses CASCADE on table drops to cut
|
|
13891
14495
|
* through the FK graph without relying on exact topological order.
|
|
13892
14496
|
*/
|
|
13893
|
-
async function down$
|
|
14497
|
+
async function down$41(db) {
|
|
13894
14498
|
for (const [trig, tbl] of [["source_secrets_change_trigger", "scoped_secrets"], ["sources_change_trigger", "sources"]]) await sql.raw(`DROP TRIGGER IF EXISTS ${trig} ON public.${tbl}`).execute(db);
|
|
13895
14499
|
for (const table of [
|
|
13896
14500
|
"workflow_registrations",
|
|
@@ -14610,8 +15214,8 @@ var init__001_initial = __esmMin((() => {
|
|
|
14610
15214
|
//#endregion
|
|
14611
15215
|
//#region src/db/migrations/002_config_versions_key_version.ts
|
|
14612
15216
|
var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
14613
|
-
down: () => down$
|
|
14614
|
-
up: () => up$
|
|
15217
|
+
down: () => down$40,
|
|
15218
|
+
up: () => up$40
|
|
14615
15219
|
});
|
|
14616
15220
|
/**
|
|
14617
15221
|
* Add key_version column to config_versions so that sensitive-field encryption
|
|
@@ -14625,13 +15229,13 @@ var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
|
14625
15229
|
* No index is needed: rotation does a full-table scan; reads are by
|
|
14626
15230
|
* `version` primary key and never filter on `key_version`.
|
|
14627
15231
|
*/
|
|
14628
|
-
async function up$
|
|
15232
|
+
async function up$40(db) {
|
|
14629
15233
|
await sql`
|
|
14630
15234
|
ALTER TABLE public.config_versions
|
|
14631
15235
|
ADD COLUMN key_version integer NOT NULL DEFAULT 1
|
|
14632
15236
|
`.execute(db);
|
|
14633
15237
|
}
|
|
14634
|
-
async function down$
|
|
15238
|
+
async function down$40(db) {
|
|
14635
15239
|
await sql`
|
|
14636
15240
|
ALTER TABLE public.config_versions
|
|
14637
15241
|
DROP COLUMN key_version
|
|
@@ -14641,8 +15245,8 @@ var init__002_config_versions_key_version = __esmMin((() => {}));
|
|
|
14641
15245
|
//#endregion
|
|
14642
15246
|
//#region src/db/migrations/003_access_log.ts
|
|
14643
15247
|
var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
14644
|
-
down: () => down$
|
|
14645
|
-
up: () => up$
|
|
15248
|
+
down: () => down$39,
|
|
15249
|
+
up: () => up$39
|
|
14646
15250
|
});
|
|
14647
15251
|
/**
|
|
14648
15252
|
* Access log: one row per read or orchestrator-admin mutation attributable
|
|
@@ -14666,7 +15270,7 @@ var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
|
14666
15270
|
* Retention is TTL-based via expires_at; packages/orchestrator/src/queue/
|
|
14667
15271
|
* cleanup.ts picks up the prune pass.
|
|
14668
15272
|
*/
|
|
14669
|
-
async function up$
|
|
15273
|
+
async function up$39(db) {
|
|
14670
15274
|
await sql`
|
|
14671
15275
|
CREATE TABLE public.access_log (
|
|
14672
15276
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
@@ -14704,21 +15308,21 @@ async function up$35(db) {
|
|
|
14704
15308
|
ON public.access_log (actor_type, actor_id, created_at DESC)
|
|
14705
15309
|
`.execute(db);
|
|
14706
15310
|
}
|
|
14707
|
-
async function down$
|
|
15311
|
+
async function down$39(db) {
|
|
14708
15312
|
await sql`DROP TABLE IF EXISTS public.access_log`.execute(db);
|
|
14709
15313
|
}
|
|
14710
15314
|
var init__003_access_log = __esmMin((() => {}));
|
|
14711
15315
|
//#endregion
|
|
14712
15316
|
//#region src/db/migrations/004_rename_bundle_to_source.ts
|
|
14713
15317
|
var _004_rename_bundle_to_source_exports = /* @__PURE__ */ __exportAll({
|
|
14714
|
-
down: () => down$
|
|
14715
|
-
up: () => up$
|
|
15318
|
+
down: () => down$38,
|
|
15319
|
+
up: () => up$38
|
|
14716
15320
|
});
|
|
14717
|
-
async function up$
|
|
15321
|
+
async function up$38(db) {
|
|
14718
15322
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_url", "source_tar_url").execute();
|
|
14719
15323
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_hash", "source_tar_hash").execute();
|
|
14720
15324
|
}
|
|
14721
|
-
async function down$
|
|
15325
|
+
async function down$38(db) {
|
|
14722
15326
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_url", "bundle_url").execute();
|
|
14723
15327
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_hash", "bundle_hash").execute();
|
|
14724
15328
|
}
|
|
@@ -14726,8 +15330,8 @@ var init__004_rename_bundle_to_source = __esmMin((() => {}));
|
|
|
14726
15330
|
//#endregion
|
|
14727
15331
|
//#region src/db/migrations/005_cold_store_chunk_counter.ts
|
|
14728
15332
|
var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
14729
|
-
down: () => down$
|
|
14730
|
-
up: () => up$
|
|
15333
|
+
down: () => down$37,
|
|
15334
|
+
up: () => up$37
|
|
14731
15335
|
});
|
|
14732
15336
|
/**
|
|
14733
15337
|
* Cold-store chunk counter table.
|
|
@@ -14744,7 +15348,7 @@ var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
|
14744
15348
|
*
|
|
14745
15349
|
* sections 5 and 8.
|
|
14746
15350
|
*/
|
|
14747
|
-
async function up$
|
|
15351
|
+
async function up$37(db) {
|
|
14748
15352
|
await sql`
|
|
14749
15353
|
CREATE TABLE public.cold_store_chunk_counts (
|
|
14750
15354
|
db TEXT NOT NULL,
|
|
@@ -14762,15 +15366,15 @@ async function up$33(db) {
|
|
|
14762
15366
|
ON public.cold_store_chunk_counts (db, table_name)
|
|
14763
15367
|
`.execute(db);
|
|
14764
15368
|
}
|
|
14765
|
-
async function down$
|
|
15369
|
+
async function down$37(db) {
|
|
14766
15370
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunk_counts`.execute(db);
|
|
14767
15371
|
}
|
|
14768
15372
|
var init__005_cold_store_chunk_counter = __esmMin((() => {}));
|
|
14769
15373
|
//#endregion
|
|
14770
15374
|
//#region src/db/migrations/006_runs_jobs_steps_archived_at.ts
|
|
14771
15375
|
var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14772
|
-
down: () => down$
|
|
14773
|
-
up: () => up$
|
|
15376
|
+
down: () => down$36,
|
|
15377
|
+
up: () => up$36
|
|
14774
15378
|
});
|
|
14775
15379
|
/**
|
|
14776
15380
|
* `execution_runs` / `execution_jobs` / `execution_steps` cold-store
|
|
@@ -14806,7 +15410,7 @@ var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
14806
15410
|
* - `idx_execution_jobs_routing_key_created (routing_key, created_at)`
|
|
14807
15411
|
* - `idx_execution_steps_routing_key_created (routing_key, created_at)`
|
|
14808
15412
|
*/
|
|
14809
|
-
async function up$
|
|
15413
|
+
async function up$36(db) {
|
|
14810
15414
|
await sql`
|
|
14811
15415
|
ALTER TABLE public.execution_runs
|
|
14812
15416
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -14851,7 +15455,7 @@ async function up$32(db) {
|
|
|
14851
15455
|
ON public.execution_steps (routing_key, created_at)
|
|
14852
15456
|
`.execute(db);
|
|
14853
15457
|
}
|
|
14854
|
-
async function down$
|
|
15458
|
+
async function down$36(db) {
|
|
14855
15459
|
await sql`DROP INDEX IF EXISTS public.idx_execution_steps_routing_key_created`.execute(db);
|
|
14856
15460
|
await sql`
|
|
14857
15461
|
ALTER TABLE public.execution_steps
|
|
@@ -14877,8 +15481,8 @@ var init__006_runs_jobs_steps_archived_at = __esmMin((() => {}));
|
|
|
14877
15481
|
//#endregion
|
|
14878
15482
|
//#region src/db/migrations/007_audit_logs_archived_at.ts
|
|
14879
15483
|
var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14880
|
-
down: () => down$
|
|
14881
|
-
up: () => up$
|
|
15484
|
+
down: () => down$35,
|
|
15485
|
+
up: () => up$35
|
|
14882
15486
|
});
|
|
14883
15487
|
/**
|
|
14884
15488
|
* `secret_audit_log` and `access_log` cold-store schema additions, plus
|
|
@@ -14918,7 +15522,7 @@ var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
14918
15522
|
* `down()` would not have meaningful retention bounds. Acceptable for
|
|
14919
15523
|
* staging.
|
|
14920
15524
|
*/
|
|
14921
|
-
async function up$
|
|
15525
|
+
async function up$35(db) {
|
|
14922
15526
|
await sql`
|
|
14923
15527
|
ALTER TABLE public.secret_audit_log
|
|
14924
15528
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -14939,7 +15543,7 @@ async function up$31(db) {
|
|
|
14939
15543
|
DROP COLUMN IF EXISTS expires_at
|
|
14940
15544
|
`.execute(db);
|
|
14941
15545
|
}
|
|
14942
|
-
async function down$
|
|
15546
|
+
async function down$35(db) {
|
|
14943
15547
|
await sql`
|
|
14944
15548
|
ALTER TABLE public.access_log
|
|
14945
15549
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '90 days')
|
|
@@ -14968,8 +15572,8 @@ var init__007_audit_logs_archived_at = __esmMin((() => {}));
|
|
|
14968
15572
|
//#endregion
|
|
14969
15573
|
//#region src/db/migrations/008_event_log_archived_at.ts
|
|
14970
15574
|
var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14971
|
-
down: () => down$
|
|
14972
|
-
up: () => up$
|
|
15575
|
+
down: () => down$34,
|
|
15576
|
+
up: () => up$34
|
|
14973
15577
|
});
|
|
14974
15578
|
/**
|
|
14975
15579
|
* `event_log` cold-store schema additions plus removal of the
|
|
@@ -15007,7 +15611,7 @@ var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
15007
15611
|
* — best effort; rows inserted between `up()` and a hypothetical
|
|
15008
15612
|
* `down()` would not have meaningful retention bounds.
|
|
15009
15613
|
*/
|
|
15010
|
-
async function up$
|
|
15614
|
+
async function up$34(db) {
|
|
15011
15615
|
await sql`
|
|
15012
15616
|
ALTER TABLE public.event_log
|
|
15013
15617
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -15023,7 +15627,7 @@ async function up$30(db) {
|
|
|
15023
15627
|
DROP COLUMN IF EXISTS expires_at
|
|
15024
15628
|
`.execute(db);
|
|
15025
15629
|
}
|
|
15026
|
-
async function down$
|
|
15630
|
+
async function down$34(db) {
|
|
15027
15631
|
await sql`
|
|
15028
15632
|
ALTER TABLE public.event_log
|
|
15029
15633
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '30 days')
|
|
@@ -15043,8 +15647,8 @@ var init__008_event_log_archived_at = __esmMin((() => {}));
|
|
|
15043
15647
|
//#endregion
|
|
15044
15648
|
//#region src/db/migrations/009_access_log_trigram.ts
|
|
15045
15649
|
var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
15046
|
-
down: () => down$
|
|
15047
|
-
up: () => up$
|
|
15650
|
+
down: () => down$33,
|
|
15651
|
+
up: () => up$33
|
|
15048
15652
|
});
|
|
15049
15653
|
/**
|
|
15050
15654
|
* Trigram (pg_trgm) index on access_log.error_message for the federated
|
|
@@ -15057,7 +15661,7 @@ var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
|
15057
15661
|
* EXISTS` are both safe to re-run. No CONCURRENTLY because Kysely runs
|
|
15058
15662
|
* migrations inside a transaction; the lock is brief on a sampled table.
|
|
15059
15663
|
*/
|
|
15060
|
-
async function up$
|
|
15664
|
+
async function up$33(db) {
|
|
15061
15665
|
await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
|
|
15062
15666
|
await sql`
|
|
15063
15667
|
CREATE INDEX IF NOT EXISTS access_log_error_message_trgm_idx
|
|
@@ -15066,15 +15670,15 @@ async function up$29(db) {
|
|
|
15066
15670
|
WHERE error_message IS NOT NULL
|
|
15067
15671
|
`.execute(db);
|
|
15068
15672
|
}
|
|
15069
|
-
async function down$
|
|
15673
|
+
async function down$33(db) {
|
|
15070
15674
|
await sql`DROP INDEX IF EXISTS public.access_log_error_message_trgm_idx`.execute(db);
|
|
15071
15675
|
}
|
|
15072
15676
|
var init__009_access_log_trigram = __esmMin((() => {}));
|
|
15073
15677
|
//#endregion
|
|
15074
15678
|
//#region src/db/migrations/010_cold_store_chunks.ts
|
|
15075
15679
|
var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
15076
|
-
down: () => down$
|
|
15077
|
-
up: () => up$
|
|
15680
|
+
down: () => down$32,
|
|
15681
|
+
up: () => up$32
|
|
15078
15682
|
});
|
|
15079
15683
|
/**
|
|
15080
15684
|
* Cold-store chunk index — Phase 2 (cold-store purge).
|
|
@@ -15106,7 +15710,7 @@ var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
|
15106
15710
|
* forever. Adapters that don't opt into per-bucket archival via
|
|
15107
15711
|
* `coldTtlDays` don't insert here either.
|
|
15108
15712
|
*/
|
|
15109
|
-
async function up$
|
|
15713
|
+
async function up$32(db) {
|
|
15110
15714
|
await sql`
|
|
15111
15715
|
CREATE TABLE public.cold_store_chunks (
|
|
15112
15716
|
db TEXT NOT NULL,
|
|
@@ -15133,15 +15737,15 @@ async function up$28(db) {
|
|
|
15133
15737
|
ON public.cold_store_chunks (db, table_name, tenant_id, archived_at DESC)
|
|
15134
15738
|
`.execute(db);
|
|
15135
15739
|
}
|
|
15136
|
-
async function down$
|
|
15740
|
+
async function down$32(db) {
|
|
15137
15741
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunks`.execute(db);
|
|
15138
15742
|
}
|
|
15139
15743
|
var init__010_cold_store_chunks = __esmMin((() => {}));
|
|
15140
15744
|
//#endregion
|
|
15141
15745
|
//#region src/db/migrations/011_drop_source_secrets_notify.ts
|
|
15142
15746
|
var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
15143
|
-
down: () => down$
|
|
15144
|
-
up: () => up$
|
|
15747
|
+
down: () => down$31,
|
|
15748
|
+
up: () => up$31
|
|
15145
15749
|
});
|
|
15146
15750
|
/**
|
|
15147
15751
|
* Drop the `source_secrets_change_trigger` and the
|
|
@@ -15160,11 +15764,11 @@ var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
15160
15764
|
* in `001_initial.ts`. They wake up no consumer until the `WebhookSecretManager`
|
|
15161
15765
|
* is restored, so this migration is safe to roll back.
|
|
15162
15766
|
*/
|
|
15163
|
-
async function up$
|
|
15767
|
+
async function up$31(db) {
|
|
15164
15768
|
await sql`DROP TRIGGER IF EXISTS source_secrets_change_trigger ON public.scoped_secrets`.execute(db);
|
|
15165
15769
|
await sql`DROP FUNCTION IF EXISTS public.notify_source_secrets_change() CASCADE`.execute(db);
|
|
15166
15770
|
}
|
|
15167
|
-
async function down$
|
|
15771
|
+
async function down$31(db) {
|
|
15168
15772
|
await sql`
|
|
15169
15773
|
CREATE OR REPLACE FUNCTION public.notify_source_secrets_change() RETURNS trigger
|
|
15170
15774
|
LANGUAGE plpgsql
|
|
@@ -15204,8 +15808,8 @@ var init__011_drop_source_secrets_notify = __esmMin((() => {}));
|
|
|
15204
15808
|
//#endregion
|
|
15205
15809
|
//#region src/db/migrations/012_peer_credentials_active_uniq.ts
|
|
15206
15810
|
var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
15207
|
-
down: () => down$
|
|
15208
|
-
up: () => up$
|
|
15811
|
+
down: () => down$30,
|
|
15812
|
+
up: () => up$30
|
|
15209
15813
|
});
|
|
15210
15814
|
/**
|
|
15211
15815
|
* Add a partial unique index on `peer_credentials (instance_id) WHERE
|
|
@@ -15231,7 +15835,7 @@ var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
|
15231
15835
|
* `down()` only drops the index; it does NOT undo the dedupe (there's no
|
|
15232
15836
|
* safe way to recreate revoked rows, and the dedupe is monotonic).
|
|
15233
15837
|
*/
|
|
15234
|
-
async function up$
|
|
15838
|
+
async function up$30(db) {
|
|
15235
15839
|
await sql`
|
|
15236
15840
|
UPDATE public.peer_credentials
|
|
15237
15841
|
SET revoked_at = NOW()
|
|
@@ -15249,15 +15853,15 @@ async function up$26(db) {
|
|
|
15249
15853
|
WHERE revoked_at IS NULL
|
|
15250
15854
|
`.execute(db);
|
|
15251
15855
|
}
|
|
15252
|
-
async function down$
|
|
15856
|
+
async function down$30(db) {
|
|
15253
15857
|
await sql`DROP INDEX IF EXISTS public.peer_credentials_active_uniq`.execute(db);
|
|
15254
15858
|
}
|
|
15255
15859
|
var init__012_peer_credentials_active_uniq = __esmMin((() => {}));
|
|
15256
15860
|
//#endregion
|
|
15257
15861
|
//#region src/db/migrations/013_execution_log_bytes.ts
|
|
15258
15862
|
var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
15259
|
-
down: () => down$
|
|
15260
|
-
up: () => up$
|
|
15863
|
+
down: () => down$29,
|
|
15864
|
+
up: () => up$29
|
|
15261
15865
|
});
|
|
15262
15866
|
/**
|
|
15263
15867
|
* Add `log_bytes BIGINT NOT NULL DEFAULT 0` columns to `execution_runs` and
|
|
@@ -15276,7 +15880,7 @@ var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
|
15276
15880
|
*
|
|
15277
15881
|
* Idempotent (`ADD COLUMN IF NOT EXISTS`).
|
|
15278
15882
|
*/
|
|
15279
|
-
async function up$
|
|
15883
|
+
async function up$29(db) {
|
|
15280
15884
|
await sql`
|
|
15281
15885
|
ALTER TABLE public.execution_runs
|
|
15282
15886
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
@@ -15286,7 +15890,7 @@ async function up$25(db) {
|
|
|
15286
15890
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
15287
15891
|
`.execute(db);
|
|
15288
15892
|
}
|
|
15289
|
-
async function down$
|
|
15893
|
+
async function down$29(db) {
|
|
15290
15894
|
await sql`ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
15291
15895
|
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
15292
15896
|
}
|
|
@@ -15294,8 +15898,8 @@ var init__013_execution_log_bytes = __esmMin((() => {}));
|
|
|
15294
15898
|
//#endregion
|
|
15295
15899
|
//#region src/db/migrations/014_kici_events_lease_retry.ts
|
|
15296
15900
|
var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
15297
|
-
down: () => down$
|
|
15298
|
-
up: () => up$
|
|
15901
|
+
down: () => down$28,
|
|
15902
|
+
up: () => up$28
|
|
15299
15903
|
});
|
|
15300
15904
|
/**
|
|
15301
15905
|
* Add lease + retry + DLQ columns to `kici_events` so the EventRouter can
|
|
@@ -15329,7 +15933,7 @@ var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
|
15329
15933
|
*
|
|
15330
15934
|
* Idempotent (`ADD COLUMN IF NOT EXISTS` + `CREATE INDEX IF NOT EXISTS`).
|
|
15331
15935
|
*/
|
|
15332
|
-
async function up$
|
|
15936
|
+
async function up$28(db) {
|
|
15333
15937
|
await sql`
|
|
15334
15938
|
ALTER TABLE public.kici_events
|
|
15335
15939
|
ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ,
|
|
@@ -15360,7 +15964,7 @@ async function up$24(db) {
|
|
|
15360
15964
|
WHERE dlq_at IS NOT NULL
|
|
15361
15965
|
`.execute(db);
|
|
15362
15966
|
}
|
|
15363
|
-
async function down$
|
|
15967
|
+
async function down$28(db) {
|
|
15364
15968
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_dlq`.execute(db);
|
|
15365
15969
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_lease_expired`.execute(db);
|
|
15366
15970
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_retry_due`.execute(db);
|
|
@@ -15379,8 +15983,8 @@ var init__014_kici_events_lease_retry = __esmMin((() => {}));
|
|
|
15379
15983
|
//#endregion
|
|
15380
15984
|
//#region src/db/migrations/015_org_settings_customer_scoped.ts
|
|
15381
15985
|
var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
15382
|
-
down: () => down$
|
|
15383
|
-
up: () => up$
|
|
15986
|
+
down: () => down$27,
|
|
15987
|
+
up: () => up$27
|
|
15384
15988
|
});
|
|
15385
15989
|
/**
|
|
15386
15990
|
* Org-scope `org_settings` and qualify each glob entry by source.
|
|
@@ -15408,7 +16012,7 @@ var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
|
15408
16012
|
* Idempotent: a re-run on an already-migrated DB sees `customer_id` exists
|
|
15409
16013
|
* and the list columns are already jsonb, so it is a no-op.
|
|
15410
16014
|
*/
|
|
15411
|
-
async function up$
|
|
16015
|
+
async function up$27(db) {
|
|
15412
16016
|
if ((await sql`
|
|
15413
16017
|
SELECT EXISTS (
|
|
15414
16018
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15533,7 +16137,7 @@ async function up$23(db) {
|
|
|
15533
16137
|
await sql`DROP TABLE _org_settings_merged`.execute(db);
|
|
15534
16138
|
await sql`DROP TABLE _org_settings_stage`.execute(db);
|
|
15535
16139
|
}
|
|
15536
|
-
async function down$
|
|
16140
|
+
async function down$27(db) {
|
|
15537
16141
|
if (!(await sql`
|
|
15538
16142
|
SELECT EXISTS (
|
|
15539
16143
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15559,8 +16163,8 @@ var init__015_org_settings_customer_scoped = __esmMin((() => {}));
|
|
|
15559
16163
|
//#endregion
|
|
15560
16164
|
//#region src/db/migrations/016_org_settings_allow_http_npm.ts
|
|
15561
16165
|
var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
15562
|
-
down: () => down$
|
|
15563
|
-
up: () => up$
|
|
16166
|
+
down: () => down$26,
|
|
16167
|
+
up: () => up$26
|
|
15564
16168
|
});
|
|
15565
16169
|
/**
|
|
15566
16170
|
* Add `org_settings.allow_http_npm_registries boolean NOT NULL DEFAULT false`.
|
|
@@ -15573,7 +16177,7 @@ var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
|
15573
16177
|
*
|
|
15574
16178
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
15575
16179
|
*/
|
|
15576
|
-
async function up$
|
|
16180
|
+
async function up$26(db) {
|
|
15577
16181
|
if ((await sql`
|
|
15578
16182
|
SELECT EXISTS (
|
|
15579
16183
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15587,7 +16191,7 @@ async function up$22(db) {
|
|
|
15587
16191
|
ADD COLUMN allow_http_npm_registries boolean NOT NULL DEFAULT false
|
|
15588
16192
|
`.execute(db);
|
|
15589
16193
|
}
|
|
15590
|
-
async function down$
|
|
16194
|
+
async function down$26(db) {
|
|
15591
16195
|
await sql`
|
|
15592
16196
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_http_npm_registries
|
|
15593
16197
|
`.execute(db);
|
|
@@ -15596,13 +16200,13 @@ var init__016_org_settings_allow_http_npm = __esmMin((() => {}));
|
|
|
15596
16200
|
//#endregion
|
|
15597
16201
|
//#region src/db/migrations/017_org_id_widen.ts
|
|
15598
16202
|
var _017_org_id_widen_exports = /* @__PURE__ */ __exportAll({
|
|
15599
|
-
down: () => down$
|
|
15600
|
-
up: () => up$
|
|
16203
|
+
down: () => down$25,
|
|
16204
|
+
up: () => up$25
|
|
15601
16205
|
});
|
|
15602
|
-
async function up$
|
|
16206
|
+
async function up$25(db) {
|
|
15603
16207
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(16)`).execute(db);
|
|
15604
16208
|
}
|
|
15605
|
-
async function down$
|
|
16209
|
+
async function down$25(db) {
|
|
15606
16210
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(12)`).execute(db);
|
|
15607
16211
|
}
|
|
15608
16212
|
var ORG_ID_TABLES$1;
|
|
@@ -15621,14 +16225,14 @@ var init__017_org_id_widen = __esmMin((() => {
|
|
|
15621
16225
|
//#endregion
|
|
15622
16226
|
//#region src/db/migrations/018_org_id_prefix_backfill.ts
|
|
15623
16227
|
var _018_org_id_prefix_backfill_exports = /* @__PURE__ */ __exportAll({
|
|
15624
|
-
down: () => down$
|
|
15625
|
-
up: () => up$
|
|
16228
|
+
down: () => down$24,
|
|
16229
|
+
up: () => up$24
|
|
15626
16230
|
});
|
|
15627
|
-
async function up$
|
|
16231
|
+
async function up$24(db) {
|
|
15628
16232
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = 'org_' || org_id WHERE org_id <> 'kici-admin' AND org_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15629
16233
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = 'org_' || customer_id WHERE customer_id <> 'kici-admin' AND customer_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15630
16234
|
}
|
|
15631
|
-
async function down$
|
|
16235
|
+
async function down$24(db) {
|
|
15632
16236
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = substring(customer_id from 5) WHERE customer_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15633
16237
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = substring(org_id from 5) WHERE org_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15634
16238
|
}
|
|
@@ -15654,8 +16258,8 @@ var init__018_org_id_prefix_backfill = __esmMin((() => {
|
|
|
15654
16258
|
//#endregion
|
|
15655
16259
|
//#region src/db/migrations/019_generic_sources_change_notify.ts
|
|
15656
16260
|
var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
15657
|
-
down: () => down$
|
|
15658
|
-
up: () => up$
|
|
16261
|
+
down: () => down$23,
|
|
16262
|
+
up: () => up$23
|
|
15659
16263
|
});
|
|
15660
16264
|
/**
|
|
15661
16265
|
* Add a Postgres trigger on `generic_webhook_sources` that emits
|
|
@@ -15678,7 +16282,7 @@ var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
15678
16282
|
* (`notify_sources_change()` + `sources_change_trigger`, defined in
|
|
15679
16283
|
* `001_initial.ts`).
|
|
15680
16284
|
*/
|
|
15681
|
-
async function up$
|
|
16285
|
+
async function up$23(db) {
|
|
15682
16286
|
await sql`
|
|
15683
16287
|
CREATE FUNCTION public.notify_generic_sources_change() RETURNS trigger
|
|
15684
16288
|
LANGUAGE plpgsql
|
|
@@ -15699,7 +16303,7 @@ async function up$19(db) {
|
|
|
15699
16303
|
FOR EACH ROW EXECUTE FUNCTION public.notify_generic_sources_change()
|
|
15700
16304
|
`.execute(db);
|
|
15701
16305
|
}
|
|
15702
|
-
async function down$
|
|
16306
|
+
async function down$23(db) {
|
|
15703
16307
|
await sql`DROP TRIGGER IF EXISTS generic_sources_change_trigger ON public.generic_webhook_sources`.execute(db);
|
|
15704
16308
|
await sql`DROP FUNCTION IF EXISTS public.notify_generic_sources_change()`.execute(db);
|
|
15705
16309
|
}
|
|
@@ -15707,8 +16311,8 @@ var init__019_generic_sources_change_notify = __esmMin((() => {}));
|
|
|
15707
16311
|
//#endregion
|
|
15708
16312
|
//#region src/db/migrations/020_org_settings_dashboard_write_policy.ts
|
|
15709
16313
|
var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportAll({
|
|
15710
|
-
down: () => down$
|
|
15711
|
-
up: () => up$
|
|
16314
|
+
down: () => down$22,
|
|
16315
|
+
up: () => up$22
|
|
15712
16316
|
});
|
|
15713
16317
|
/**
|
|
15714
16318
|
* Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
|
|
@@ -15723,7 +16327,7 @@ var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportA
|
|
|
15723
16327
|
*
|
|
15724
16328
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
15725
16329
|
*/
|
|
15726
|
-
async function up$
|
|
16330
|
+
async function up$22(db) {
|
|
15727
16331
|
if ((await sql`
|
|
15728
16332
|
SELECT EXISTS (
|
|
15729
16333
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15737,7 +16341,7 @@ async function up$18(db) {
|
|
|
15737
16341
|
ADD COLUMN dashboard_write_policy jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
15738
16342
|
`.execute(db);
|
|
15739
16343
|
}
|
|
15740
|
-
async function down$
|
|
16344
|
+
async function down$22(db) {
|
|
15741
16345
|
await sql`
|
|
15742
16346
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dashboard_write_policy
|
|
15743
16347
|
`.execute(db);
|
|
@@ -15746,8 +16350,8 @@ var init__020_org_settings_dashboard_write_policy = __esmMin((() => {}));
|
|
|
15746
16350
|
//#endregion
|
|
15747
16351
|
//#region src/db/migrations/021_check_run_tracking.ts
|
|
15748
16352
|
var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
15749
|
-
down: () => down$
|
|
15750
|
-
up: () => up$
|
|
16353
|
+
down: () => down$21,
|
|
16354
|
+
up: () => up$21
|
|
15751
16355
|
});
|
|
15752
16356
|
/**
|
|
15753
16357
|
* Add `check_run_tracking` table for HA-safe check-run state persistence.
|
|
@@ -15771,7 +16375,7 @@ var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
|
15771
16375
|
*
|
|
15772
16376
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
15773
16377
|
*/
|
|
15774
|
-
async function up$
|
|
16378
|
+
async function up$21(db) {
|
|
15775
16379
|
if ((await sql`
|
|
15776
16380
|
SELECT EXISTS (
|
|
15777
16381
|
SELECT 1 FROM information_schema.tables
|
|
@@ -15802,15 +16406,15 @@ async function up$17(db) {
|
|
|
15802
16406
|
WHERE run_id IS NOT NULL
|
|
15803
16407
|
`.execute(db);
|
|
15804
16408
|
}
|
|
15805
|
-
async function down$
|
|
16409
|
+
async function down$21(db) {
|
|
15806
16410
|
await sql`DROP TABLE IF EXISTS public.check_run_tracking`.execute(db);
|
|
15807
16411
|
}
|
|
15808
16412
|
var init__021_check_run_tracking = __esmMin((() => {}));
|
|
15809
16413
|
//#endregion
|
|
15810
16414
|
//#region src/db/migrations/022_scaler_manager_state.ts
|
|
15811
16415
|
var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
15812
|
-
down: () => down$
|
|
15813
|
-
up: () => up$
|
|
16416
|
+
down: () => down$20,
|
|
16417
|
+
up: () => up$20
|
|
15814
16418
|
});
|
|
15815
16419
|
/**
|
|
15816
16420
|
* Add three tables persisting `ScalerManager` per-coord state:
|
|
@@ -15837,7 +16441,7 @@ var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
|
15837
16441
|
* Idempotent: a re-run on a DB that already has any of these tables
|
|
15838
16442
|
* leaves the existing one alone.
|
|
15839
16443
|
*/
|
|
15840
|
-
async function up$
|
|
16444
|
+
async function up$20(db) {
|
|
15841
16445
|
const tableExists = async (name) => {
|
|
15842
16446
|
return (await sql`
|
|
15843
16447
|
SELECT EXISTS (
|
|
@@ -15887,7 +16491,7 @@ async function up$16(db) {
|
|
|
15887
16491
|
`.execute(db);
|
|
15888
16492
|
}
|
|
15889
16493
|
}
|
|
15890
|
-
async function down$
|
|
16494
|
+
async function down$20(db) {
|
|
15891
16495
|
await sql`DROP TABLE IF EXISTS public.scaler_reservations`.execute(db);
|
|
15892
16496
|
await sql`DROP TABLE IF EXISTS public.scaler_agent_jobs`.execute(db);
|
|
15893
16497
|
await sql`DROP TABLE IF EXISTS public.scaler_spawning_agents`.execute(db);
|
|
@@ -15896,8 +16500,8 @@ var init__022_scaler_manager_state = __esmMin((() => {}));
|
|
|
15896
16500
|
//#endregion
|
|
15897
16501
|
//#region src/db/migrations/023_dispatch_queue_recovery_deadline.ts
|
|
15898
16502
|
var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
15899
|
-
down: () => down$
|
|
15900
|
-
up: () => up$
|
|
16503
|
+
down: () => down$19,
|
|
16504
|
+
up: () => up$19
|
|
15901
16505
|
});
|
|
15902
16506
|
/**
|
|
15903
16507
|
* Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
|
|
@@ -15921,7 +16525,7 @@ var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll(
|
|
|
15921
16525
|
* Idempotent: re-running on a DB that already has either column is a
|
|
15922
16526
|
* no-op.
|
|
15923
16527
|
*/
|
|
15924
|
-
async function up$
|
|
16528
|
+
async function up$19(db) {
|
|
15925
16529
|
const colExists = async (name) => {
|
|
15926
16530
|
return (await sql`
|
|
15927
16531
|
SELECT EXISTS (
|
|
@@ -15946,7 +16550,7 @@ async function up$15(db) {
|
|
|
15946
16550
|
WHERE recovery_deadline IS NOT NULL
|
|
15947
16551
|
`.execute(db);
|
|
15948
16552
|
}
|
|
15949
|
-
async function down$
|
|
16553
|
+
async function down$19(db) {
|
|
15950
16554
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_recovery_deadline`.execute(db);
|
|
15951
16555
|
await sql`
|
|
15952
16556
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS recovery_agent_id
|
|
@@ -15959,8 +16563,8 @@ var init__023_dispatch_queue_recovery_deadline = __esmMin((() => {}));
|
|
|
15959
16563
|
//#endregion
|
|
15960
16564
|
//#region src/db/migrations/024_dispatch_queue_provisioning_error.ts
|
|
15961
16565
|
var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll({
|
|
15962
|
-
down: () => down$
|
|
15963
|
-
up: () => up$
|
|
16566
|
+
down: () => down$18,
|
|
16567
|
+
up: () => up$18
|
|
15964
16568
|
});
|
|
15965
16569
|
/**
|
|
15966
16570
|
* Add `dispatch_queue.last_provisioning_error TEXT` recording the most
|
|
@@ -15976,7 +16580,7 @@ var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll
|
|
|
15976
16580
|
*
|
|
15977
16581
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
15978
16582
|
*/
|
|
15979
|
-
async function up$
|
|
16583
|
+
async function up$18(db) {
|
|
15980
16584
|
const colExists = async (name) => {
|
|
15981
16585
|
return (await sql`
|
|
15982
16586
|
SELECT EXISTS (
|
|
@@ -15992,7 +16596,7 @@ async function up$14(db) {
|
|
|
15992
16596
|
ADD COLUMN last_provisioning_error TEXT
|
|
15993
16597
|
`.execute(db);
|
|
15994
16598
|
}
|
|
15995
|
-
async function down$
|
|
16599
|
+
async function down$18(db) {
|
|
15996
16600
|
await sql`
|
|
15997
16601
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS last_provisioning_error
|
|
15998
16602
|
`.execute(db);
|
|
@@ -16001,8 +16605,8 @@ var init__024_dispatch_queue_provisioning_error = __esmMin((() => {}));
|
|
|
16001
16605
|
//#endregion
|
|
16002
16606
|
//#region src/db/migrations/025_init_failure.ts
|
|
16003
16607
|
var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
16004
|
-
down: () => down$
|
|
16005
|
-
up: () => up$
|
|
16608
|
+
down: () => down$17,
|
|
16609
|
+
up: () => up$17
|
|
16006
16610
|
});
|
|
16007
16611
|
/**
|
|
16008
16612
|
* Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
|
|
@@ -16016,7 +16620,7 @@ var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
|
16016
16620
|
* Idempotent: re-running on a DB that already has either column is a no-op
|
|
16017
16621
|
* for that column.
|
|
16018
16622
|
*/
|
|
16019
|
-
async function up$
|
|
16623
|
+
async function up$17(db) {
|
|
16020
16624
|
const colExists = async (table, name) => {
|
|
16021
16625
|
return (await sql`
|
|
16022
16626
|
SELECT EXISTS (
|
|
@@ -16036,7 +16640,7 @@ async function up$13(db) {
|
|
|
16036
16640
|
ADD COLUMN init_failure JSONB DEFAULT NULL
|
|
16037
16641
|
`.execute(db);
|
|
16038
16642
|
}
|
|
16039
|
-
async function down$
|
|
16643
|
+
async function down$17(db) {
|
|
16040
16644
|
await sql`
|
|
16041
16645
|
ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS init_failure
|
|
16042
16646
|
`.execute(db);
|
|
@@ -16048,8 +16652,8 @@ var init__025_init_failure = __esmMin((() => {}));
|
|
|
16048
16652
|
//#endregion
|
|
16049
16653
|
//#region src/db/migrations/026_event_log_lockfile_corrupt.ts
|
|
16050
16654
|
var _026_event_log_lockfile_corrupt_exports = /* @__PURE__ */ __exportAll({
|
|
16051
|
-
down: () => down$
|
|
16052
|
-
up: () => up$
|
|
16655
|
+
down: () => down$16,
|
|
16656
|
+
up: () => up$16
|
|
16053
16657
|
});
|
|
16054
16658
|
/**
|
|
16055
16659
|
* Extend the event_log.status CHECK constraint with 'lockfile_corrupt' so the
|
|
@@ -16058,7 +16662,7 @@ var _026_event_log_lockfile_corrupt_exports = /* @__PURE__ */ __exportAll({
|
|
|
16058
16662
|
*
|
|
16059
16663
|
* Idempotent: the DROP ... IF EXISTS / re-ADD pair re-runs cleanly.
|
|
16060
16664
|
*/
|
|
16061
|
-
async function up$
|
|
16665
|
+
async function up$16(db) {
|
|
16062
16666
|
await sql`ALTER TABLE event_log DROP CONSTRAINT IF EXISTS event_log_status_check`.execute(db);
|
|
16063
16667
|
await sql`
|
|
16064
16668
|
ALTER TABLE event_log ADD CONSTRAINT event_log_status_check
|
|
@@ -16068,7 +16672,7 @@ async function up$12(db) {
|
|
|
16068
16672
|
])))
|
|
16069
16673
|
`.execute(db);
|
|
16070
16674
|
}
|
|
16071
|
-
async function down$
|
|
16675
|
+
async function down$16(db) {
|
|
16072
16676
|
await sql`ALTER TABLE event_log DROP CONSTRAINT IF EXISTS event_log_status_check`.execute(db);
|
|
16073
16677
|
await sql`
|
|
16074
16678
|
ALTER TABLE event_log ADD CONSTRAINT event_log_status_check
|
|
@@ -16082,8 +16686,8 @@ var init__026_event_log_lockfile_corrupt = __esmMin((() => {}));
|
|
|
16082
16686
|
//#endregion
|
|
16083
16687
|
//#region src/db/migrations/027_workflow_timeout.ts
|
|
16084
16688
|
var _027_workflow_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
16085
|
-
down: () => down$
|
|
16086
|
-
up: () => up$
|
|
16689
|
+
down: () => down$15,
|
|
16690
|
+
up: () => up$15
|
|
16087
16691
|
});
|
|
16088
16692
|
/**
|
|
16089
16693
|
* Add `workflow_timeout_ms integer` to `execution_runs`.
|
|
@@ -16101,13 +16705,13 @@ var _027_workflow_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
|
16101
16705
|
*
|
|
16102
16706
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
16103
16707
|
*/
|
|
16104
|
-
async function up$
|
|
16708
|
+
async function up$15(db) {
|
|
16105
16709
|
await sql`
|
|
16106
16710
|
ALTER TABLE public.execution_runs
|
|
16107
16711
|
ADD COLUMN IF NOT EXISTS workflow_timeout_ms INTEGER DEFAULT NULL
|
|
16108
16712
|
`.execute(db);
|
|
16109
16713
|
}
|
|
16110
|
-
async function down$
|
|
16714
|
+
async function down$15(db) {
|
|
16111
16715
|
await sql`
|
|
16112
16716
|
ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS workflow_timeout_ms
|
|
16113
16717
|
`.execute(db);
|
|
@@ -16116,8 +16720,8 @@ var init__027_workflow_timeout = __esmMin((() => {}));
|
|
|
16116
16720
|
//#endregion
|
|
16117
16721
|
//#region src/db/migrations/028_org_settings_user_cache.ts
|
|
16118
16722
|
var _028_org_settings_user_cache_exports = /* @__PURE__ */ __exportAll({
|
|
16119
|
-
down: () => down$
|
|
16120
|
-
up: () => up$
|
|
16723
|
+
down: () => down$14,
|
|
16724
|
+
up: () => up$14
|
|
16121
16725
|
});
|
|
16122
16726
|
/**
|
|
16123
16727
|
* Add `org_settings.user_cache_quota_bytes bigint` and
|
|
@@ -16146,7 +16750,7 @@ async function columnExists(db, column) {
|
|
|
16146
16750
|
) AS exists
|
|
16147
16751
|
`.execute(db)).rows[0]?.exists ?? false;
|
|
16148
16752
|
}
|
|
16149
|
-
async function up$
|
|
16753
|
+
async function up$14(db) {
|
|
16150
16754
|
if (!await columnExists(db, "user_cache_quota_bytes")) await sql`
|
|
16151
16755
|
ALTER TABLE public.org_settings
|
|
16152
16756
|
ADD COLUMN user_cache_quota_bytes bigint
|
|
@@ -16156,7 +16760,7 @@ async function up$10(db) {
|
|
|
16156
16760
|
ADD COLUMN user_cache_ttl_ms bigint
|
|
16157
16761
|
`.execute(db);
|
|
16158
16762
|
}
|
|
16159
|
-
async function down$
|
|
16763
|
+
async function down$14(db) {
|
|
16160
16764
|
await sql`
|
|
16161
16765
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS user_cache_quota_bytes
|
|
16162
16766
|
`.execute(db);
|
|
@@ -16168,8 +16772,8 @@ var init__028_org_settings_user_cache = __esmMin((() => {}));
|
|
|
16168
16772
|
//#endregion
|
|
16169
16773
|
//#region src/db/migrations/029_dispatch_queue_attempts.ts
|
|
16170
16774
|
var _029_dispatch_queue_attempts_exports = /* @__PURE__ */ __exportAll({
|
|
16171
|
-
down: () => down$
|
|
16172
|
-
up: () => up$
|
|
16775
|
+
down: () => down$13,
|
|
16776
|
+
up: () => up$13
|
|
16173
16777
|
});
|
|
16174
16778
|
/**
|
|
16175
16779
|
* Add `dispatch_queue.dispatch_attempts INT NOT NULL DEFAULT 0`.
|
|
@@ -16183,7 +16787,7 @@ var _029_dispatch_queue_attempts_exports = /* @__PURE__ */ __exportAll({
|
|
|
16183
16787
|
*
|
|
16184
16788
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
16185
16789
|
*/
|
|
16186
|
-
async function up$
|
|
16790
|
+
async function up$13(db) {
|
|
16187
16791
|
if (!((await sql`
|
|
16188
16792
|
SELECT EXISTS (
|
|
16189
16793
|
SELECT 1 FROM information_schema.columns
|
|
@@ -16196,7 +16800,7 @@ async function up$9(db) {
|
|
|
16196
16800
|
ADD COLUMN dispatch_attempts INT NOT NULL DEFAULT 0
|
|
16197
16801
|
`.execute(db);
|
|
16198
16802
|
}
|
|
16199
|
-
async function down$
|
|
16803
|
+
async function down$13(db) {
|
|
16200
16804
|
await sql`
|
|
16201
16805
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS dispatch_attempts
|
|
16202
16806
|
`.execute(db);
|
|
@@ -16205,8 +16809,8 @@ var init__029_dispatch_queue_attempts = __esmMin((() => {}));
|
|
|
16205
16809
|
//#endregion
|
|
16206
16810
|
//#region src/db/migrations/030_held_runs_env_set_null.ts
|
|
16207
16811
|
var _030_held_runs_env_set_null_exports = /* @__PURE__ */ __exportAll({
|
|
16208
|
-
down: () => down$
|
|
16209
|
-
up: () => up$
|
|
16812
|
+
down: () => down$12,
|
|
16813
|
+
up: () => up$12
|
|
16210
16814
|
});
|
|
16211
16815
|
/**
|
|
16212
16816
|
* held_runs.environment_id becomes nullable with ON DELETE SET NULL so
|
|
@@ -16217,14 +16821,14 @@ var _030_held_runs_env_set_null_exports = /* @__PURE__ */ __exportAll({
|
|
|
16217
16821
|
* Idempotent: dropping the NOT NULL and the constraint are both no-ops on a
|
|
16218
16822
|
* re-run, and the constraint is re-created with the SET NULL action.
|
|
16219
16823
|
*/
|
|
16220
|
-
async function up$
|
|
16824
|
+
async function up$12(db) {
|
|
16221
16825
|
await sql`ALTER TABLE public.held_runs ALTER COLUMN environment_id DROP NOT NULL`.execute(db);
|
|
16222
16826
|
await sql`ALTER TABLE public.held_runs DROP CONSTRAINT IF EXISTS held_runs_environment_id_fkey`.execute(db);
|
|
16223
16827
|
await sql`ALTER TABLE public.held_runs
|
|
16224
16828
|
ADD CONSTRAINT held_runs_environment_id_fkey
|
|
16225
16829
|
FOREIGN KEY (environment_id) REFERENCES public.environments(id) ON DELETE SET NULL`.execute(db);
|
|
16226
16830
|
}
|
|
16227
|
-
async function down$
|
|
16831
|
+
async function down$12(db) {
|
|
16228
16832
|
await sql`ALTER TABLE public.held_runs DROP CONSTRAINT IF EXISTS held_runs_environment_id_fkey`.execute(db);
|
|
16229
16833
|
await sql`ALTER TABLE public.held_runs
|
|
16230
16834
|
ADD CONSTRAINT held_runs_environment_id_fkey
|
|
@@ -16235,8 +16839,8 @@ var init__030_held_runs_env_set_null = __esmMin((() => {}));
|
|
|
16235
16839
|
//#endregion
|
|
16236
16840
|
//#region src/db/migrations/031_dispatch_queue_ack_deadline.ts
|
|
16237
16841
|
var _031_dispatch_queue_ack_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
16238
|
-
down: () => down$
|
|
16239
|
-
up: () => up$
|
|
16842
|
+
down: () => down$11,
|
|
16843
|
+
up: () => up$11
|
|
16240
16844
|
});
|
|
16241
16845
|
/**
|
|
16242
16846
|
* Add `dispatch_queue.ack_deadline TIMESTAMPTZ` and
|
|
@@ -16253,7 +16857,7 @@ var _031_dispatch_queue_ack_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
|
16253
16857
|
*
|
|
16254
16858
|
* Idempotent: re-running on a DB that already has either column is a no-op.
|
|
16255
16859
|
*/
|
|
16256
|
-
async function up$
|
|
16860
|
+
async function up$11(db) {
|
|
16257
16861
|
const colExists = async (name) => {
|
|
16258
16862
|
return (await sql`
|
|
16259
16863
|
SELECT EXISTS (
|
|
@@ -16278,7 +16882,7 @@ async function up$7(db) {
|
|
|
16278
16882
|
WHERE ack_deadline IS NOT NULL
|
|
16279
16883
|
`.execute(db);
|
|
16280
16884
|
}
|
|
16281
|
-
async function down$
|
|
16885
|
+
async function down$11(db) {
|
|
16282
16886
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_ack_deadline`.execute(db);
|
|
16283
16887
|
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS ack_agent_id`.execute(db);
|
|
16284
16888
|
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS ack_deadline`.execute(db);
|
|
@@ -16287,8 +16891,8 @@ var init__031_dispatch_queue_ack_deadline = __esmMin((() => {}));
|
|
|
16287
16891
|
//#endregion
|
|
16288
16892
|
//#region src/db/migrations/032_org_settings_dispatch_ack_timeout.ts
|
|
16289
16893
|
var _032_org_settings_dispatch_ack_timeout_exports = /* @__PURE__ */ __exportAll({
|
|
16290
|
-
down: () => down$
|
|
16291
|
-
up: () => up$
|
|
16894
|
+
down: () => down$10,
|
|
16895
|
+
up: () => up$10
|
|
16292
16896
|
});
|
|
16293
16897
|
/**
|
|
16294
16898
|
* Add `org_settings.dispatch_ack_timeout_ms BIGINT` (nullable).
|
|
@@ -16300,7 +16904,7 @@ var _032_org_settings_dispatch_ack_timeout_exports = /* @__PURE__ */ __exportAll
|
|
|
16300
16904
|
*
|
|
16301
16905
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
16302
16906
|
*/
|
|
16303
|
-
async function up$
|
|
16907
|
+
async function up$10(db) {
|
|
16304
16908
|
if ((await sql`
|
|
16305
16909
|
SELECT EXISTS (
|
|
16306
16910
|
SELECT 1 FROM information_schema.columns
|
|
@@ -16314,7 +16918,7 @@ async function up$6(db) {
|
|
|
16314
16918
|
ADD COLUMN dispatch_ack_timeout_ms BIGINT
|
|
16315
16919
|
`.execute(db);
|
|
16316
16920
|
}
|
|
16317
|
-
async function down$
|
|
16921
|
+
async function down$10(db) {
|
|
16318
16922
|
await sql`
|
|
16319
16923
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dispatch_ack_timeout_ms
|
|
16320
16924
|
`.execute(db);
|
|
@@ -16323,8 +16927,8 @@ var init__032_org_settings_dispatch_ack_timeout = __esmMin((() => {}));
|
|
|
16323
16927
|
//#endregion
|
|
16324
16928
|
//#region src/db/migrations/033_org_settings_approval.ts
|
|
16325
16929
|
var _033_org_settings_approval_exports = /* @__PURE__ */ __exportAll({
|
|
16326
|
-
down: () => down$
|
|
16327
|
-
up: () => up$
|
|
16930
|
+
down: () => down$9,
|
|
16931
|
+
up: () => up$9
|
|
16328
16932
|
});
|
|
16329
16933
|
/**
|
|
16330
16934
|
* Add the two approval-policy columns to `org_settings`:
|
|
@@ -16341,7 +16945,7 @@ var _033_org_settings_approval_exports = /* @__PURE__ */ __exportAll({
|
|
|
16341
16945
|
* and the orchestrator admin route. Idempotent: a re-run on a DB that already
|
|
16342
16946
|
* has the columns is a no-op (each column is guarded independently).
|
|
16343
16947
|
*/
|
|
16344
|
-
async function up$
|
|
16948
|
+
async function up$9(db) {
|
|
16345
16949
|
const colExists = async (column) => {
|
|
16346
16950
|
return (await sql`
|
|
16347
16951
|
SELECT EXISTS (
|
|
@@ -16361,7 +16965,7 @@ async function up$5(db) {
|
|
|
16361
16965
|
ADD COLUMN allow_self_approval BOOLEAN NOT NULL DEFAULT true
|
|
16362
16966
|
`.execute(db);
|
|
16363
16967
|
}
|
|
16364
|
-
async function down$
|
|
16968
|
+
async function down$9(db) {
|
|
16365
16969
|
await sql`
|
|
16366
16970
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS approval_expiry_seconds
|
|
16367
16971
|
`.execute(db);
|
|
@@ -16373,8 +16977,8 @@ var init__033_org_settings_approval = __esmMin((() => {}));
|
|
|
16373
16977
|
//#endregion
|
|
16374
16978
|
//#region src/db/migrations/034_held_runs_generalize.ts
|
|
16375
16979
|
var _034_held_runs_generalize_exports = /* @__PURE__ */ __exportAll({
|
|
16376
|
-
down: () => down$
|
|
16377
|
-
up: () => up$
|
|
16980
|
+
down: () => down$8,
|
|
16981
|
+
up: () => up$8
|
|
16378
16982
|
});
|
|
16379
16983
|
/**
|
|
16380
16984
|
* Generalize `held_runs` from an environment-only hold into the unified
|
|
@@ -16396,7 +17000,7 @@ var _034_held_runs_generalize_exports = /* @__PURE__ */ __exportAll({
|
|
|
16396
17000
|
* New `held_run_approvals` table: one row per approver decision, FK to
|
|
16397
17001
|
* `held_runs.id` (uuid) with ON DELETE CASCADE.
|
|
16398
17002
|
*/
|
|
16399
|
-
async function up$
|
|
17003
|
+
async function up$8(db) {
|
|
16400
17004
|
const colExists = async (column) => {
|
|
16401
17005
|
return (await sql`
|
|
16402
17006
|
SELECT EXISTS (
|
|
@@ -16429,7 +17033,7 @@ async function up$4(db) {
|
|
|
16429
17033
|
ON public.held_run_approvals USING btree (held_run_id)
|
|
16430
17034
|
`.execute(db);
|
|
16431
17035
|
}
|
|
16432
|
-
async function down$
|
|
17036
|
+
async function down$8(db) {
|
|
16433
17037
|
await sql`DROP TABLE IF EXISTS public.held_run_approvals`.execute(db);
|
|
16434
17038
|
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS approval_requirement`.execute(db);
|
|
16435
17039
|
await sql`ALTER TABLE public.held_runs DROP COLUMN IF EXISTS trigger_source`.execute(db);
|
|
@@ -16440,8 +17044,8 @@ var init__034_held_runs_generalize = __esmMin((() => {}));
|
|
|
16440
17044
|
//#endregion
|
|
16441
17045
|
//#region src/db/migrations/035_pending_workflow_contexts.ts
|
|
16442
17046
|
var _035_pending_workflow_contexts_exports = /* @__PURE__ */ __exportAll({
|
|
16443
|
-
down: () => down$
|
|
16444
|
-
up: () => up$
|
|
17047
|
+
down: () => down$7,
|
|
17048
|
+
up: () => up$7
|
|
16445
17049
|
});
|
|
16446
17050
|
/**
|
|
16447
17051
|
* Pending workflow dispatch context — backs resume of a workflow whose install
|
|
@@ -16450,7 +17054,7 @@ var _035_pending_workflow_contexts_exports = /* @__PURE__ */ __exportAll({
|
|
|
16450
17054
|
* wait-timer expiry, concurrency slot free). The row is deleted once the resume
|
|
16451
17055
|
* dispatch has been kicked off.
|
|
16452
17056
|
*/
|
|
16453
|
-
async function up$
|
|
17057
|
+
async function up$7(db) {
|
|
16454
17058
|
await sql`
|
|
16455
17059
|
CREATE TABLE IF NOT EXISTS public.pending_workflow_contexts (
|
|
16456
17060
|
run_id text PRIMARY KEY,
|
|
@@ -16460,15 +17064,15 @@ async function up$3(db) {
|
|
|
16460
17064
|
)
|
|
16461
17065
|
`.execute(db);
|
|
16462
17066
|
}
|
|
16463
|
-
async function down$
|
|
17067
|
+
async function down$7(db) {
|
|
16464
17068
|
await sql`DROP TABLE IF EXISTS public.pending_workflow_contexts`.execute(db);
|
|
16465
17069
|
}
|
|
16466
17070
|
var init__035_pending_workflow_contexts = __esmMin((() => {}));
|
|
16467
17071
|
//#endregion
|
|
16468
17072
|
//#region src/db/migrations/036_attestations.ts
|
|
16469
17073
|
var _036_attestations_exports = /* @__PURE__ */ __exportAll({
|
|
16470
|
-
down: () => down$
|
|
16471
|
-
up: () => up$
|
|
17074
|
+
down: () => down$6,
|
|
17075
|
+
up: () => up$6
|
|
16472
17076
|
});
|
|
16473
17077
|
/**
|
|
16474
17078
|
* Add the `attestations` table for build-provenance bundles.
|
|
@@ -16481,7 +17085,7 @@ var _036_attestations_exports = /* @__PURE__ */ __exportAll({
|
|
|
16481
17085
|
*
|
|
16482
17086
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
16483
17087
|
*/
|
|
16484
|
-
async function up$
|
|
17088
|
+
async function up$6(db) {
|
|
16485
17089
|
if ((await sql`
|
|
16486
17090
|
SELECT EXISTS (
|
|
16487
17091
|
SELECT 1 FROM information_schema.tables
|
|
@@ -16507,15 +17111,15 @@ async function up$2(db) {
|
|
|
16507
17111
|
ON public.attestations (run_id, job_id)
|
|
16508
17112
|
`.execute(db);
|
|
16509
17113
|
}
|
|
16510
|
-
async function down$
|
|
17114
|
+
async function down$6(db) {
|
|
16511
17115
|
await sql`DROP TABLE IF EXISTS public.attestations`.execute(db);
|
|
16512
17116
|
}
|
|
16513
17117
|
var init__036_attestations = __esmMin((() => {}));
|
|
16514
17118
|
//#endregion
|
|
16515
17119
|
//#region src/db/migrations/037_generic_sources_provider_type_local.ts
|
|
16516
17120
|
var _037_generic_sources_provider_type_local_exports = /* @__PURE__ */ __exportAll({
|
|
16517
|
-
down: () => down$
|
|
16518
|
-
up: () => up$
|
|
17121
|
+
down: () => down$5,
|
|
17122
|
+
up: () => up$5
|
|
16519
17123
|
});
|
|
16520
17124
|
/**
|
|
16521
17125
|
* Replace the `generic_webhook_sources.provider_type` CHECK constraint so it
|
|
@@ -16532,7 +17136,7 @@ var _037_generic_sources_provider_type_local_exports = /* @__PURE__ */ __exportA
|
|
|
16532
17136
|
* Idempotent: the constraint is dropped IF EXISTS and recreated; the data
|
|
16533
17137
|
* backfill is a plain UPDATE that is a no-op once no `'internal'` rows remain.
|
|
16534
17138
|
*/
|
|
16535
|
-
async function up$
|
|
17139
|
+
async function up$5(db) {
|
|
16536
17140
|
await sql`
|
|
16537
17141
|
ALTER TABLE public.generic_webhook_sources
|
|
16538
17142
|
DROP CONSTRAINT IF EXISTS generic_webhook_sources_provider_type_check
|
|
@@ -16548,7 +17152,7 @@ async function up$1(db) {
|
|
|
16548
17152
|
CHECK (provider_type = ANY (ARRAY['generic'::text, 'local'::text]))
|
|
16549
17153
|
`.execute(db);
|
|
16550
17154
|
}
|
|
16551
|
-
async function down$
|
|
17155
|
+
async function down$5(db) {
|
|
16552
17156
|
await sql`
|
|
16553
17157
|
ALTER TABLE public.generic_webhook_sources
|
|
16554
17158
|
DROP CONSTRAINT IF EXISTS generic_webhook_sources_provider_type_check
|
|
@@ -16568,8 +17172,8 @@ var init__037_generic_sources_provider_type_local = __esmMin((() => {}));
|
|
|
16568
17172
|
//#endregion
|
|
16569
17173
|
//#region src/db/migrations/038_remote_sources.ts
|
|
16570
17174
|
var _038_remote_sources_exports = /* @__PURE__ */ __exportAll({
|
|
16571
|
-
down: () => down,
|
|
16572
|
-
up: () => up
|
|
17175
|
+
down: () => down$4,
|
|
17176
|
+
up: () => up$4
|
|
16573
17177
|
});
|
|
16574
17178
|
/**
|
|
16575
17179
|
* `remote_sources` anchors a Platform-relayed `kici run remote` to its real
|
|
@@ -16581,7 +17185,7 @@ var _038_remote_sources_exports = /* @__PURE__ */ __exportAll({
|
|
|
16581
17185
|
*
|
|
16582
17186
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
16583
17187
|
*/
|
|
16584
|
-
async function up(db) {
|
|
17188
|
+
async function up$4(db) {
|
|
16585
17189
|
if ((await sql`
|
|
16586
17190
|
SELECT EXISTS (
|
|
16587
17191
|
SELECT 1 FROM information_schema.tables
|
|
@@ -16600,11 +17204,205 @@ async function up(db) {
|
|
|
16600
17204
|
)
|
|
16601
17205
|
`.execute(db);
|
|
16602
17206
|
}
|
|
16603
|
-
async function down(db) {
|
|
17207
|
+
async function down$4(db) {
|
|
16604
17208
|
await sql`DROP TABLE IF EXISTS public.remote_sources`.execute(db);
|
|
16605
17209
|
}
|
|
16606
17210
|
var init__038_remote_sources = __esmMin((() => {}));
|
|
16607
17211
|
//#endregion
|
|
17212
|
+
//#region src/db/migrations/039_host_roster.ts
|
|
17213
|
+
var _039_host_roster_exports = /* @__PURE__ */ __exportAll({
|
|
17214
|
+
down: () => down$3,
|
|
17215
|
+
up: () => up$3
|
|
17216
|
+
});
|
|
17217
|
+
/**
|
|
17218
|
+
* `host_roster` is KiCI's declared inventory: one durable row per agent the
|
|
17219
|
+
* cluster has ever enrolled, reconciled from the in-memory AgentRegistry on
|
|
17220
|
+
* every register/unregister. `lifecycle_class` (snapshot of the auth token's
|
|
17221
|
+
* agent_type) drives reaping — `ephemeral` rows are GC'd past their TTL,
|
|
17222
|
+
* `static` rows persist and read as `unreachable` when their heartbeat goes
|
|
17223
|
+
* stale. `connected_instance_id` records which orchestrator holds the live WS
|
|
17224
|
+
* (cluster liveness + the host-fanout reroute target); NULL = disconnected.
|
|
17225
|
+
*
|
|
17226
|
+
* The roster lives in the shared cluster DB (one table, all instances). Status
|
|
17227
|
+
* is derived at read from the shared `last_seen` + `connected_instance_id`, so
|
|
17228
|
+
* every instance agrees regardless of which one holds the agent's live WS.
|
|
17229
|
+
*
|
|
17230
|
+
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
17231
|
+
*/
|
|
17232
|
+
async function up$3(db) {
|
|
17233
|
+
if ((await sql`
|
|
17234
|
+
SELECT EXISTS (
|
|
17235
|
+
SELECT 1 FROM information_schema.tables
|
|
17236
|
+
WHERE table_schema = 'public' AND table_name = 'host_roster'
|
|
17237
|
+
) AS exists
|
|
17238
|
+
`.execute(db)).rows[0]?.exists) return;
|
|
17239
|
+
await sql`
|
|
17240
|
+
CREATE TABLE public.host_roster (
|
|
17241
|
+
id uuid DEFAULT gen_random_uuid() NOT NULL,
|
|
17242
|
+
agent_id text NOT NULL,
|
|
17243
|
+
token_id uuid,
|
|
17244
|
+
lifecycle_class text NOT NULL,
|
|
17245
|
+
labels text NOT NULL DEFAULT '[]',
|
|
17246
|
+
hostname text,
|
|
17247
|
+
platform text,
|
|
17248
|
+
arch text,
|
|
17249
|
+
connected_instance_id text,
|
|
17250
|
+
last_seen timestamptz NOT NULL DEFAULT now(),
|
|
17251
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
17252
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
17253
|
+
CONSTRAINT host_roster_pkey PRIMARY KEY (id),
|
|
17254
|
+
CONSTRAINT host_roster_agent_id_key UNIQUE (agent_id),
|
|
17255
|
+
CONSTRAINT host_roster_lifecycle_class_check
|
|
17256
|
+
CHECK (lifecycle_class = ANY (ARRAY['static'::text, 'ephemeral'::text]))
|
|
17257
|
+
)
|
|
17258
|
+
`.execute(db);
|
|
17259
|
+
await sql`CREATE INDEX idx_host_roster_reap
|
|
17260
|
+
ON public.host_roster (lifecycle_class, last_seen)`.execute(db);
|
|
17261
|
+
}
|
|
17262
|
+
async function down$3(db) {
|
|
17263
|
+
await sql`DROP TABLE IF EXISTS public.host_roster`.execute(db);
|
|
17264
|
+
}
|
|
17265
|
+
var init__039_host_roster = __esmMin((() => {}));
|
|
17266
|
+
//#endregion
|
|
17267
|
+
//#region src/db/migrations/040_runsonall_pin.ts
|
|
17268
|
+
var _040_runsonall_pin_exports = /* @__PURE__ */ __exportAll({
|
|
17269
|
+
down: () => down$2,
|
|
17270
|
+
up: () => up$2
|
|
17271
|
+
});
|
|
17272
|
+
/**
|
|
17273
|
+
* Add the `runsOnAll` host fan-out columns:
|
|
17274
|
+
*
|
|
17275
|
+
* - `dispatch_queue.pinned_agent_id TEXT` — when set, the queued/dispatched job
|
|
17276
|
+
* targets exactly that agent (a host-fanout child). The dispatcher routes it
|
|
17277
|
+
* only to that agent; the queue drain never hands it to a different one.
|
|
17278
|
+
* - `execution_jobs.base_job_name` / `variant_kind` / `variant_label` — generic
|
|
17279
|
+
* fan-out columns (matrix + host uniform). `variant_kind` is `'matrix'` or
|
|
17280
|
+
* `'host'`; `variant_label` is the matrix suffix or the hostname. They make the
|
|
17281
|
+
* logical fan-out job first-class server-side so the dashboard groups on real
|
|
17282
|
+
* fields instead of string-parsing the job name. (`matrix_values` / `group_name`
|
|
17283
|
+
* already exist; the matrix path now also backfills `variant_kind='matrix'`.)
|
|
17284
|
+
*
|
|
17285
|
+
* Idempotent: re-running on a DB that already has any column is a no-op.
|
|
17286
|
+
*/
|
|
17287
|
+
async function colExists$2(db, table, name) {
|
|
17288
|
+
return (await sql`
|
|
17289
|
+
SELECT EXISTS (
|
|
17290
|
+
SELECT 1 FROM information_schema.columns
|
|
17291
|
+
WHERE table_schema = 'public'
|
|
17292
|
+
AND table_name = ${table}
|
|
17293
|
+
AND column_name = ${name}
|
|
17294
|
+
) AS exists
|
|
17295
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
17296
|
+
}
|
|
17297
|
+
async function up$2(db) {
|
|
17298
|
+
if (!await colExists$2(db, "dispatch_queue", "pinned_agent_id")) await sql`ALTER TABLE public.dispatch_queue ADD COLUMN pinned_agent_id TEXT`.execute(db);
|
|
17299
|
+
if (!await colExists$2(db, "execution_jobs", "base_job_name")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN base_job_name TEXT`.execute(db);
|
|
17300
|
+
if (!await colExists$2(db, "execution_jobs", "variant_kind")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN variant_kind TEXT`.execute(db);
|
|
17301
|
+
if (!await colExists$2(db, "execution_jobs", "variant_label")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN variant_label TEXT`.execute(db);
|
|
17302
|
+
await sql`
|
|
17303
|
+
CREATE INDEX IF NOT EXISTS idx_dispatch_queue_pinned_agent
|
|
17304
|
+
ON public.dispatch_queue (pinned_agent_id)
|
|
17305
|
+
WHERE pinned_agent_id IS NOT NULL
|
|
17306
|
+
`.execute(db);
|
|
17307
|
+
}
|
|
17308
|
+
async function down$2(db) {
|
|
17309
|
+
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_pinned_agent`.execute(db);
|
|
17310
|
+
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS pinned_agent_id`.execute(db);
|
|
17311
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS variant_label`.execute(db);
|
|
17312
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS variant_kind`.execute(db);
|
|
17313
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS base_job_name`.execute(db);
|
|
17314
|
+
}
|
|
17315
|
+
var init__040_runsonall_pin = __esmMin((() => {}));
|
|
17316
|
+
//#endregion
|
|
17317
|
+
//#region src/db/migrations/041_wave_gated.ts
|
|
17318
|
+
var _041_wave_gated_exports = /* @__PURE__ */ __exportAll({
|
|
17319
|
+
down: () => down$1,
|
|
17320
|
+
up: () => up$1
|
|
17321
|
+
});
|
|
17322
|
+
/**
|
|
17323
|
+
* Add the rolling fan-out wave-gate columns:
|
|
17324
|
+
*
|
|
17325
|
+
* - `execution_jobs.wave_gated boolean NOT NULL DEFAULT false` — when a fan-out
|
|
17326
|
+
* job declares `maxParallel`, children beyond the sliding window are persisted
|
|
17327
|
+
* `wave_gated=true` (held, not enqueued). The dispatch loop skips them; the
|
|
17328
|
+
* wave-scheduler clears the flag one-per-terminal as siblings complete (or, on
|
|
17329
|
+
* `failFast`, skips the held remainder).
|
|
17330
|
+
* - `execution_jobs.wave_max_parallel int` / `wave_fail_fast boolean` — the
|
|
17331
|
+
* base's wave policy, stamped on every fan-out child so the wave-scheduler can
|
|
17332
|
+
* read it at terminal time without re-fetching the lock file (the tracker has
|
|
17333
|
+
* no lock access). NULL for any job not part of a bounded wave.
|
|
17334
|
+
*
|
|
17335
|
+
* A composite index on (run_id, base_job_name, wave_gated) supports the
|
|
17336
|
+
* wave-scheduler's "next held sibling of this base" lookups.
|
|
17337
|
+
*
|
|
17338
|
+
* Idempotent: re-running on a DB that already has the columns is a no-op.
|
|
17339
|
+
*/
|
|
17340
|
+
async function colExists$1(db, table, name) {
|
|
17341
|
+
return (await sql`
|
|
17342
|
+
SELECT EXISTS (
|
|
17343
|
+
SELECT 1 FROM information_schema.columns
|
|
17344
|
+
WHERE table_schema = 'public'
|
|
17345
|
+
AND table_name = ${table}
|
|
17346
|
+
AND column_name = ${name}
|
|
17347
|
+
) AS exists
|
|
17348
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
17349
|
+
}
|
|
17350
|
+
async function up$1(db) {
|
|
17351
|
+
if (!await colExists$1(db, "execution_jobs", "wave_gated")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN wave_gated boolean NOT NULL DEFAULT false`.execute(db);
|
|
17352
|
+
if (!await colExists$1(db, "execution_jobs", "wave_max_parallel")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN wave_max_parallel integer`.execute(db);
|
|
17353
|
+
if (!await colExists$1(db, "execution_jobs", "wave_fail_fast")) await sql`ALTER TABLE public.execution_jobs ADD COLUMN wave_fail_fast boolean`.execute(db);
|
|
17354
|
+
await sql`
|
|
17355
|
+
CREATE INDEX IF NOT EXISTS idx_execution_jobs_wave
|
|
17356
|
+
ON public.execution_jobs (run_id, base_job_name, wave_gated)
|
|
17357
|
+
`.execute(db);
|
|
17358
|
+
}
|
|
17359
|
+
async function down$1(db) {
|
|
17360
|
+
await sql`DROP INDEX IF EXISTS public.idx_execution_jobs_wave`.execute(db);
|
|
17361
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS wave_fail_fast`.execute(db);
|
|
17362
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS wave_max_parallel`.execute(db);
|
|
17363
|
+
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS wave_gated`.execute(db);
|
|
17364
|
+
}
|
|
17365
|
+
var init__041_wave_gated = __esmMin((() => {}));
|
|
17366
|
+
//#endregion
|
|
17367
|
+
//#region src/db/migrations/042_dispatch_queue_patterns.ts
|
|
17368
|
+
var _042_dispatch_queue_patterns_exports = /* @__PURE__ */ __exportAll({
|
|
17369
|
+
down: () => down,
|
|
17370
|
+
up: () => up
|
|
17371
|
+
});
|
|
17372
|
+
/**
|
|
17373
|
+
* Add pattern columns to dispatch_queue. Exact labels stay in runs_on_labels /
|
|
17374
|
+
* exclude_labels (the SQL @> prefilter); regex matchers go here and are applied
|
|
17375
|
+
* as a JS post-filter at drain time, since Postgres `~` regex semantics differ
|
|
17376
|
+
* from JavaScript `RegExp` and the engine's `matcherSatisfiedBy` is the single
|
|
17377
|
+
* matching authority.
|
|
17378
|
+
*
|
|
17379
|
+
* - `runs_on_patterns jsonb NOT NULL DEFAULT '[]'` — regex matchers the agent's
|
|
17380
|
+
* labels must satisfy.
|
|
17381
|
+
* - `exclude_patterns jsonb NOT NULL DEFAULT '[]'` — regex matchers that
|
|
17382
|
+
* disqualify an agent.
|
|
17383
|
+
*
|
|
17384
|
+
* Idempotent: re-running on a DB that already has the columns is a no-op.
|
|
17385
|
+
*/
|
|
17386
|
+
async function colExists(db, table, name) {
|
|
17387
|
+
return (await sql`
|
|
17388
|
+
SELECT EXISTS (
|
|
17389
|
+
SELECT 1 FROM information_schema.columns
|
|
17390
|
+
WHERE table_schema = 'public'
|
|
17391
|
+
AND table_name = ${table}
|
|
17392
|
+
AND column_name = ${name}
|
|
17393
|
+
) AS exists
|
|
17394
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
17395
|
+
}
|
|
17396
|
+
async function up(db) {
|
|
17397
|
+
if (!await colExists(db, "dispatch_queue", "runs_on_patterns")) await sql`ALTER TABLE public.dispatch_queue ADD COLUMN runs_on_patterns jsonb NOT NULL DEFAULT '[]'::jsonb`.execute(db);
|
|
17398
|
+
if (!await colExists(db, "dispatch_queue", "exclude_patterns")) await sql`ALTER TABLE public.dispatch_queue ADD COLUMN exclude_patterns jsonb NOT NULL DEFAULT '[]'::jsonb`.execute(db);
|
|
17399
|
+
}
|
|
17400
|
+
async function down(db) {
|
|
17401
|
+
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS exclude_patterns`.execute(db);
|
|
17402
|
+
await sql`ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS runs_on_patterns`.execute(db);
|
|
17403
|
+
}
|
|
17404
|
+
var init__042_dispatch_queue_patterns = __esmMin((() => {}));
|
|
17405
|
+
//#endregion
|
|
16608
17406
|
//#region src/db/migration-provider.ts
|
|
16609
17407
|
function createMigrationProvider() {
|
|
16610
17408
|
return { async getMigrations() {
|
|
@@ -16646,7 +17444,11 @@ function createMigrationProvider() {
|
|
|
16646
17444
|
"035_pending_workflow_contexts": _035_pending_workflow_contexts_exports,
|
|
16647
17445
|
"036_attestations": _036_attestations_exports,
|
|
16648
17446
|
"037_generic_sources_provider_type_local": _037_generic_sources_provider_type_local_exports,
|
|
16649
|
-
"038_remote_sources": _038_remote_sources_exports
|
|
17447
|
+
"038_remote_sources": _038_remote_sources_exports,
|
|
17448
|
+
"039_host_roster": _039_host_roster_exports,
|
|
17449
|
+
"040_runsonall_pin": _040_runsonall_pin_exports,
|
|
17450
|
+
"041_wave_gated": _041_wave_gated_exports,
|
|
17451
|
+
"042_dispatch_queue_patterns": _042_dispatch_queue_patterns_exports
|
|
16650
17452
|
};
|
|
16651
17453
|
} };
|
|
16652
17454
|
}
|
|
@@ -16689,6 +17491,10 @@ var init_migration_provider = __esmMin((() => {
|
|
|
16689
17491
|
init__036_attestations();
|
|
16690
17492
|
init__037_generic_sources_provider_type_local();
|
|
16691
17493
|
init__038_remote_sources();
|
|
17494
|
+
init__039_host_roster();
|
|
17495
|
+
init__040_runsonall_pin();
|
|
17496
|
+
init__041_wave_gated();
|
|
17497
|
+
init__042_dispatch_queue_patterns();
|
|
16692
17498
|
}));
|
|
16693
17499
|
//#endregion
|
|
16694
17500
|
//#region src/db/migrator.ts
|
|
@@ -22739,15 +23545,15 @@ var init_admin_config = __esmMin((() => {
|
|
|
22739
23545
|
function createHealthRoutes$1(deps = {}) {
|
|
22740
23546
|
return createHealthRoutes({
|
|
22741
23547
|
livenessInfo: () => ({
|
|
22742
|
-
version: "0.1.
|
|
22743
|
-
buildDate: "2026-06-
|
|
22744
|
-
buildCommit: "
|
|
22745
|
-
sdkVersion: "0.1.
|
|
22746
|
-
sdkBundleHash: "
|
|
22747
|
-
sharedVersion: "0.1.
|
|
22748
|
-
sharedBundleHash: "
|
|
22749
|
-
engineVersion: "0.1.
|
|
22750
|
-
engineBundleHash: "
|
|
23548
|
+
version: "0.1.18",
|
|
23549
|
+
buildDate: "2026-06-17T09:57:52.557Z",
|
|
23550
|
+
buildCommit: "d8cff38bb",
|
|
23551
|
+
sdkVersion: "0.1.18",
|
|
23552
|
+
sdkBundleHash: "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f",
|
|
23553
|
+
sharedVersion: "0.1.18",
|
|
23554
|
+
sharedBundleHash: "5f2c220f24d166b0f13d0620a2e80d19689ac8b683a12154daef44e938fd46a7",
|
|
23555
|
+
engineVersion: "0.1.18",
|
|
23556
|
+
engineBundleHash: "79ce14640d1798eaaa7cb3aa6c4bc325da3bff1e9f4716936e19614eabb1d858"
|
|
22751
23557
|
}),
|
|
22752
23558
|
readinessCheck: deps.db ? async () => {
|
|
22753
23559
|
const checks = {};
|
|
@@ -22781,7 +23587,7 @@ function createCapabilitiesRoutes() {
|
|
|
22781
23587
|
const app = new Hono();
|
|
22782
23588
|
app.get("/api/v1/capabilities", (c) => {
|
|
22783
23589
|
const manifest = {
|
|
22784
|
-
orchestratorVersion: "0.1.
|
|
23590
|
+
orchestratorVersion: "0.1.18",
|
|
22785
23591
|
protocolVersion: PROTOCOL_VERSION,
|
|
22786
23592
|
minProtocolVersion: MIN_PROTOCOL_VERSION
|
|
22787
23593
|
};
|
|
@@ -24987,8 +25793,9 @@ function chooseTargetPlatform(workflow, agentRegistry) {
|
|
|
24987
25793
|
targetPlatform,
|
|
24988
25794
|
targetArch
|
|
24989
25795
|
};
|
|
24990
|
-
const
|
|
24991
|
-
const
|
|
25796
|
+
const firstJob = workflow.jobs.filter(isLockStaticJob)[0];
|
|
25797
|
+
const firstExact = partitionMatchers(firstJob?.runsOn ?? []).exact;
|
|
25798
|
+
const representativeLabels = workflow.jobs.length > 0 ? firstExact.length > 0 ? firstExact : ["default"] : ["default"];
|
|
24992
25799
|
const candidates = agentRegistry.findAvailable(representativeLabels);
|
|
24993
25800
|
if (candidates.length > 0) {
|
|
24994
25801
|
targetPlatform = candidates[0].platform;
|
|
@@ -25279,12 +26086,112 @@ async function readPostBuildCacheUrls(args) {
|
|
|
25279
26086
|
* other jobs still proceed. Dynamic-matrix jobs pass through with a
|
|
25280
26087
|
* `pendingDynamicMatrix` marker for the eval flow.
|
|
25281
26088
|
*/
|
|
25282
|
-
|
|
26089
|
+
/**
|
|
26090
|
+
* Resolve a `runsOnAll` lock job against the declared host roster and partition
|
|
26091
|
+
* the matched hosts into the target set per the `onUnreachable` policy (R2):
|
|
26092
|
+
* `ready` hosts always run; unreachable durable (`static`) hosts hold / fail /
|
|
26093
|
+
* skip; stale ephemeral hosts are always skipped. Throws {@link FanoutError}
|
|
26094
|
+
* when the run can't proceed (fail policy with an absent host, or zero targets).
|
|
26095
|
+
*/
|
|
26096
|
+
async function resolveHostFanoutTargets(lockJob, deps) {
|
|
26097
|
+
if (!deps.hostRosterStore) throw new FanoutError(lockJob.name, `runsOnAll for job '${lockJob.name}': roster unavailable`);
|
|
26098
|
+
const predicate = lockJob.runsOnAll;
|
|
26099
|
+
const onUnreachable = lockJob.onUnreachable ?? "hold";
|
|
26100
|
+
const matched = await deps.hostRosterStore.findMatching(predicate.include, predicate.exclude, deps.rosterGraceMs ?? 3e5);
|
|
26101
|
+
const targets = [];
|
|
26102
|
+
const unreachableDurable = [];
|
|
26103
|
+
for (const h of matched) if (h.status === "ready") targets.push(h);
|
|
26104
|
+
else if (h.lifecycleClass === "ephemeral") continue;
|
|
26105
|
+
else unreachableDurable.push(h);
|
|
26106
|
+
if (unreachableDurable.length > 0) {
|
|
26107
|
+
if (onUnreachable === "fail") throw new FanoutError(lockJob.name, `runsOnAll '${lockJob.name}': ${unreachableDurable.length} expected host(s) unreachable`);
|
|
26108
|
+
if (onUnreachable === "hold") targets.push(...unreachableDurable);
|
|
26109
|
+
}
|
|
26110
|
+
if (targets.length === 0) throw new FanoutError(lockJob.name, `runsOnAll '${lockJob.name}' matched zero usable hosts`);
|
|
26111
|
+
return targets.map((h) => ({
|
|
26112
|
+
agentId: h.agentId,
|
|
26113
|
+
host: h.host,
|
|
26114
|
+
labels: h.labels,
|
|
26115
|
+
platform: h.platform ?? void 0,
|
|
26116
|
+
arch: h.arch ?? void 0,
|
|
26117
|
+
connectedInstanceId: h.connectedInstanceId
|
|
26118
|
+
}));
|
|
26119
|
+
}
|
|
26120
|
+
/**
|
|
26121
|
+
* Partition a lock job's runsOn / excludeLabels matchers into exact labels (SQL
|
|
26122
|
+
* `@>` prefilter + registry index) and regex patterns (JS post-filter). A
|
|
26123
|
+
* `runsOnAll` host-fanout job has no `runsOn`; its pinned children carry no
|
|
26124
|
+
* routing (the pin targets the resolved agent directly).
|
|
26125
|
+
*/
|
|
26126
|
+
function runsOnSelectorsForLockJob(lockJob) {
|
|
26127
|
+
const include = partitionMatchers(lockJob.runsOn ?? []);
|
|
26128
|
+
const exclude = partitionMatchers(lockJob.excludeLabels ?? []);
|
|
26129
|
+
return {
|
|
26130
|
+
runsOnLabels: include.exact,
|
|
26131
|
+
runsOnPatterns: include.regex,
|
|
26132
|
+
excludeLabels: exclude.exact,
|
|
26133
|
+
excludePatterns: exclude.regex
|
|
26134
|
+
};
|
|
26135
|
+
}
|
|
26136
|
+
/**
|
|
26137
|
+
* The generic fan-out tracking fields persisted on `execution_jobs` for a
|
|
26138
|
+
* materialized child: `baseJobName` + `variantKind` + `variantLabel`. Serves
|
|
26139
|
+
* matrix (label = combination suffix) and host (label = hostname) uniformly so
|
|
26140
|
+
* the dashboard groups on real columns instead of string-parsing the name.
|
|
26141
|
+
*/
|
|
26142
|
+
function variantTrackingFields(mat) {
|
|
26143
|
+
if (!mat.variantKind) return {};
|
|
26144
|
+
const variantLabel = mat.variantKind === VariantKind.host ? mat.host : mat.variantValues ? mat.expandedName.slice(mat.baseName.length + 2, -1) : void 0;
|
|
26145
|
+
return {
|
|
26146
|
+
baseJobName: mat.baseName,
|
|
26147
|
+
variantKind: mat.variantKind,
|
|
26148
|
+
...variantLabel && { variantLabel }
|
|
26149
|
+
};
|
|
26150
|
+
}
|
|
26151
|
+
/**
|
|
26152
|
+
* Compute the rolling-wave plan for a materialized job set.
|
|
26153
|
+
*
|
|
26154
|
+
* For each base job declaring `maxParallel` whose fan-out produced more than one
|
|
26155
|
+
* child, children are ordered deterministically by `variant_label` (the matrix
|
|
26156
|
+
* suffix / hostname, via `expandedName`) and every child at index `>=
|
|
26157
|
+
* maxParallel` is held (`wave_gated=true`). The first `maxParallel` dispatch
|
|
26158
|
+
* immediately; held children release one-per-terminal via the wave-scheduler.
|
|
26159
|
+
* Every child of a bounded-wave base — held or not — gets a `policy` entry so
|
|
26160
|
+
* the wave-scheduler can read the width/failFast at terminal time. A non-fan-out
|
|
26161
|
+
* job (single child) or one without `maxParallel` contributes nothing.
|
|
26162
|
+
*/
|
|
26163
|
+
function computeWavePlan(materializedJobs) {
|
|
26164
|
+
const byBase = /* @__PURE__ */ new Map();
|
|
26165
|
+
for (const mat of materializedJobs) {
|
|
26166
|
+
const list = byBase.get(mat.baseName);
|
|
26167
|
+
if (list) list.push(mat);
|
|
26168
|
+
else byBase.set(mat.baseName, [mat]);
|
|
26169
|
+
}
|
|
26170
|
+
const held = /* @__PURE__ */ new Set();
|
|
26171
|
+
const policy = /* @__PURE__ */ new Map();
|
|
26172
|
+
for (const children of byBase.values()) {
|
|
26173
|
+
const maxParallel = children[0]?.lockJob.maxParallel;
|
|
26174
|
+
if (maxParallel === void 0 || children.length <= 1) continue;
|
|
26175
|
+
const failFast = children[0]?.lockJob.failFast ?? false;
|
|
26176
|
+
[...children].sort((a, b) => a.expandedName.localeCompare(b.expandedName)).forEach((mat, i) => {
|
|
26177
|
+
policy.set(mat.expandedName, {
|
|
26178
|
+
maxParallel,
|
|
26179
|
+
failFast
|
|
26180
|
+
});
|
|
26181
|
+
if (i >= maxParallel) held.add(mat.expandedName);
|
|
26182
|
+
});
|
|
26183
|
+
}
|
|
26184
|
+
return {
|
|
26185
|
+
held,
|
|
26186
|
+
policy
|
|
26187
|
+
};
|
|
26188
|
+
}
|
|
26189
|
+
async function materializeStaticJobsSafe(staticJobs, deps) {
|
|
25283
26190
|
const materializedJobs = [];
|
|
25284
26191
|
const expansionMap = /* @__PURE__ */ new Map();
|
|
25285
26192
|
const matrixFailures = [];
|
|
25286
26193
|
for (const lockJob of staticJobs) try {
|
|
25287
|
-
const result = materializeFanout([lockJob]);
|
|
26194
|
+
const result = lockJob.runsOnAll ? materializeResolvedHosts(lockJob, await resolveHostFanoutTargets(lockJob, deps), deps.maxFanoutHosts ?? 1024) : materializeFanout([lockJob]);
|
|
25288
26195
|
materializedJobs.push(...result.jobs);
|
|
25289
26196
|
for (const [k, v] of result.expansionMap) expansionMap.set(k, v);
|
|
25290
26197
|
} catch (err) {
|
|
@@ -25428,7 +26335,7 @@ async function prepareCacheAndBuild(ctx, setup) {
|
|
|
25428
26335
|
}
|
|
25429
26336
|
}
|
|
25430
26337
|
if (!contentHash && deps.sourceCache) logger$38.debug("Workflow missing contentHash, agents will compile from source", { workflow: workflow.name });
|
|
25431
|
-
const { materializedJobs, expansionMap, matrixFailures } = materializeStaticJobsSafe(staticJobs);
|
|
26338
|
+
const { materializedJobs, expansionMap, matrixFailures } = await materializeStaticJobsSafe(staticJobs, deps);
|
|
25432
26339
|
return {
|
|
25433
26340
|
sourceTarUrl,
|
|
25434
26341
|
sourceTarHash,
|
|
@@ -25903,6 +26810,13 @@ function makeBuildJobConfig(args) {
|
|
|
25903
26810
|
name: mat.expandedName,
|
|
25904
26811
|
baseJobName: mat.baseName,
|
|
25905
26812
|
...mat.variantValues && { matrixValues: mat.variantValues },
|
|
26813
|
+
...mat.host && { host: mat.host },
|
|
26814
|
+
...mat.agent && { agent: {
|
|
26815
|
+
host: mat.agent.host,
|
|
26816
|
+
labels: [...mat.agent.labels],
|
|
26817
|
+
...mat.agent.platform && { platform: mat.agent.platform },
|
|
26818
|
+
...mat.agent.arch && { arch: mat.agent.arch }
|
|
26819
|
+
} },
|
|
25906
26820
|
steps: lockJob.steps,
|
|
25907
26821
|
needs: lockJob.needs,
|
|
25908
26822
|
rules: lockJob.rules,
|
|
@@ -25931,15 +26845,17 @@ function makeBuildJobConfig(args) {
|
|
|
25931
26845
|
* single-orch paths).
|
|
25932
26846
|
*/
|
|
25933
26847
|
function buildExecutionJobInput(args) {
|
|
25934
|
-
const { ctx, setup, buildPrep, buildJobConfig, mat,
|
|
26848
|
+
const { ctx, setup, buildPrep, buildJobConfig, mat, selectors } = args;
|
|
25935
26849
|
const lockJob = mat.lockJob;
|
|
25936
26850
|
const { workflow, bundle, repoIdentifier, credentials, event, ref, runId } = ctx;
|
|
25937
26851
|
return {
|
|
25938
26852
|
runId,
|
|
25939
26853
|
workflowName: workflow.name,
|
|
25940
26854
|
jobName: mat.expandedName,
|
|
25941
|
-
runsOnLabels,
|
|
25942
|
-
|
|
26855
|
+
runsOnLabels: selectors.runsOnLabels,
|
|
26856
|
+
runsOnPatterns: selectors.runsOnPatterns,
|
|
26857
|
+
excludeLabels: selectors.excludeLabels,
|
|
26858
|
+
excludePatterns: selectors.excludePatterns,
|
|
25943
26859
|
jobConfig: buildJobConfig(mat),
|
|
25944
26860
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
25945
26861
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -25953,7 +26869,9 @@ function buildExecutionJobInput(args) {
|
|
|
25953
26869
|
depsUrl: buildPrep.depsUrl,
|
|
25954
26870
|
depsHash: buildPrep.depsHash,
|
|
25955
26871
|
requestId: getRequestContext().requestId,
|
|
25956
|
-
...lockJob.resources && { resources: lockJob.resources }
|
|
26872
|
+
...lockJob.resources && { resources: lockJob.resources },
|
|
26873
|
+
...mat.pinnedAgentId && { pinnedAgentId: mat.pinnedAgentId },
|
|
26874
|
+
...mat.connectedInstanceId !== void 0 && { connectedInstanceId: mat.connectedInstanceId }
|
|
25957
26875
|
};
|
|
25958
26876
|
}
|
|
25959
26877
|
/**
|
|
@@ -25967,7 +26885,8 @@ async function holdJobForApproval(args) {
|
|
|
25967
26885
|
const { ctx, setup, buildPrep, buildJobConfig, mat, envData, dispatchedJobs } = args;
|
|
25968
26886
|
const lockJob = mat.lockJob;
|
|
25969
26887
|
const { deps, workflow, runId } = ctx;
|
|
25970
|
-
const
|
|
26888
|
+
const selectors = runsOnSelectorsForLockJob(lockJob);
|
|
26889
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
25971
26890
|
const hold = envData.approvalHold;
|
|
25972
26891
|
if (!hold || !deps.heldRunStore || !deps.db) {
|
|
25973
26892
|
logger$38.info("Job held by protection rules", {
|
|
@@ -25983,8 +26902,7 @@ async function holdJobForApproval(args) {
|
|
|
25983
26902
|
buildPrep,
|
|
25984
26903
|
buildJobConfig,
|
|
25985
26904
|
mat,
|
|
25986
|
-
|
|
25987
|
-
excludeLabels: lockJob._type === "static" && lockJob.excludeLabels ? [...lockJob.excludeLabels] : void 0
|
|
26905
|
+
selectors
|
|
25988
26906
|
});
|
|
25989
26907
|
const heldRow = await deps.heldRunStore.createHold(ctx.resolvedOrgId, {
|
|
25990
26908
|
runId,
|
|
@@ -26072,15 +26990,15 @@ async function preRegisterNonRootJobs(args) {
|
|
|
26072
26990
|
const { deps, workflow, runId } = ctx;
|
|
26073
26991
|
for (const gated of needsGatedJobs) {
|
|
26074
26992
|
const gatedJob = gated.lockJob;
|
|
26075
|
-
const
|
|
26993
|
+
const selectors = runsOnSelectorsForLockJob(gatedJob);
|
|
26994
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
26076
26995
|
const jobInput = buildExecutionJobInput({
|
|
26077
26996
|
ctx,
|
|
26078
26997
|
setup,
|
|
26079
26998
|
buildPrep,
|
|
26080
26999
|
buildJobConfig,
|
|
26081
27000
|
mat: gated,
|
|
26082
|
-
|
|
26083
|
-
excludeLabels: gatedJob._type === "static" && gatedJob.excludeLabels ? [...gatedJob.excludeLabels] : void 0
|
|
27001
|
+
selectors
|
|
26084
27002
|
});
|
|
26085
27003
|
await storePendingJobContext(deps.db, runId, gated.expandedName, {
|
|
26086
27004
|
jobInput,
|
|
@@ -26139,9 +27057,12 @@ async function clusterRouteRootJobs(args) {
|
|
|
26139
27057
|
};
|
|
26140
27058
|
const jobsToRoute = rootDispatchableJobs.map((mj) => {
|
|
26141
27059
|
const j = mj.lockJob;
|
|
27060
|
+
const sel = runsOnSelectorsForLockJob(j);
|
|
26142
27061
|
return {
|
|
26143
27062
|
jobName: mj.expandedName,
|
|
26144
|
-
runsOnLabels: [
|
|
27063
|
+
runsOnLabels: [sel.runsOnLabels],
|
|
27064
|
+
runsOnPatterns: sel.runsOnPatterns,
|
|
27065
|
+
excludePatterns: sel.excludePatterns,
|
|
26145
27066
|
jobConfig: buildJobConfig(mj),
|
|
26146
27067
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26147
27068
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26150,7 +27071,7 @@ async function clusterRouteRootJobs(args) {
|
|
|
26150
27071
|
sourceTarHash: buildPrep.sourceTarHash,
|
|
26151
27072
|
depsUrl: buildPrep.depsUrl,
|
|
26152
27073
|
depsHash: buildPrep.depsHash,
|
|
26153
|
-
excludeLabels:
|
|
27074
|
+
excludeLabels: sel.excludeLabels,
|
|
26154
27075
|
...j.resources ? { resources: j.resources } : {}
|
|
26155
27076
|
};
|
|
26156
27077
|
});
|
|
@@ -26174,6 +27095,8 @@ async function clusterRouteRootJobs(args) {
|
|
|
26174
27095
|
workflowName: workflow.name,
|
|
26175
27096
|
jobName: jtr.jobName,
|
|
26176
27097
|
runsOnLabels: flatLabels,
|
|
27098
|
+
runsOnPatterns: jtr.runsOnPatterns,
|
|
27099
|
+
excludePatterns: jtr.excludePatterns,
|
|
26177
27100
|
excludeLabels: jtr.excludeLabels,
|
|
26178
27101
|
jobConfig: jtr.jobConfig,
|
|
26179
27102
|
repoUrl: jtr.repoUrl,
|
|
@@ -26253,6 +27176,15 @@ async function clusterRouteRootJobs(args) {
|
|
|
26253
27176
|
async function dispatchSingleOrchPath(args) {
|
|
26254
27177
|
const { ctx, setup, buildPrep, buildJobConfig, jobEnvironmentData, dispatchedJobs, rejectedJobs } = args;
|
|
26255
27178
|
const { deps, workflow, runId } = ctx;
|
|
27179
|
+
const wavePlan = computeWavePlan(buildPrep.materializedJobs);
|
|
27180
|
+
/** The wave-policy fields persisted on a bounded-wave child's execution_jobs row. */
|
|
27181
|
+
const wavePolicyFields = (name) => {
|
|
27182
|
+
const p = wavePlan.policy.get(name);
|
|
27183
|
+
return p ? {
|
|
27184
|
+
waveMaxParallel: p.maxParallel,
|
|
27185
|
+
waveFailFast: p.failFast
|
|
27186
|
+
} : {};
|
|
27187
|
+
};
|
|
26256
27188
|
for (const mat of buildPrep.materializedJobs) {
|
|
26257
27189
|
const lockJob = mat.lockJob;
|
|
26258
27190
|
const matrixValues = mat.variantValues;
|
|
@@ -26279,16 +27211,16 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26279
27211
|
continue;
|
|
26280
27212
|
}
|
|
26281
27213
|
if (envData?.pendingInit) continue;
|
|
26282
|
-
const
|
|
26283
|
-
const
|
|
27214
|
+
const selectors = runsOnSelectorsForLockJob(lockJob);
|
|
27215
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
27216
|
+
const excludeLabels = selectors.excludeLabels;
|
|
26284
27217
|
const jobInput = buildExecutionJobInput({
|
|
26285
27218
|
ctx,
|
|
26286
27219
|
setup,
|
|
26287
27220
|
buildPrep,
|
|
26288
27221
|
buildJobConfig,
|
|
26289
27222
|
mat,
|
|
26290
|
-
|
|
26291
|
-
excludeLabels
|
|
27223
|
+
selectors
|
|
26292
27224
|
});
|
|
26293
27225
|
if (!isRootJob(lockJob)) {
|
|
26294
27226
|
await storePendingJobContext(deps.db, runId, mat.expandedName, {
|
|
@@ -26300,6 +27232,7 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26300
27232
|
jobId: syntheticId,
|
|
26301
27233
|
jobName: mat.expandedName,
|
|
26302
27234
|
...matrixValues && { matrixValues },
|
|
27235
|
+
...variantTrackingFields(mat),
|
|
26303
27236
|
runsOnLabels
|
|
26304
27237
|
});
|
|
26305
27238
|
logger$38.info("Job gated by needs scheduler (not dispatched yet)", {
|
|
@@ -26309,6 +27242,29 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26309
27242
|
});
|
|
26310
27243
|
continue;
|
|
26311
27244
|
}
|
|
27245
|
+
if (wavePlan.held.has(mat.expandedName)) {
|
|
27246
|
+
await storePendingJobContext(deps.db, runId, mat.expandedName, {
|
|
27247
|
+
jobInput,
|
|
27248
|
+
runsOnLabels
|
|
27249
|
+
});
|
|
27250
|
+
const syntheticId = `needs-pending-${mat.expandedName}-${randomUUID()}`;
|
|
27251
|
+
dispatchedJobs.push({
|
|
27252
|
+
jobId: syntheticId,
|
|
27253
|
+
jobName: mat.expandedName,
|
|
27254
|
+
...matrixValues && { matrixValues },
|
|
27255
|
+
...variantTrackingFields(mat),
|
|
27256
|
+
...wavePolicyFields(mat.expandedName),
|
|
27257
|
+
runsOnLabels,
|
|
27258
|
+
waveGated: true
|
|
27259
|
+
});
|
|
27260
|
+
logger$38.info("Fan-out child held by rolling wave (maxParallel)", {
|
|
27261
|
+
runId,
|
|
27262
|
+
workflow: workflow.name,
|
|
27263
|
+
job: mat.expandedName,
|
|
27264
|
+
maxParallel: lockJob.maxParallel
|
|
27265
|
+
});
|
|
27266
|
+
continue;
|
|
27267
|
+
}
|
|
26312
27268
|
const result = await setup.dispatcher.dispatch(jobInput);
|
|
26313
27269
|
if (result.status === "rejected") {
|
|
26314
27270
|
const syntheticId = `rejected-${randomUUID()}`;
|
|
@@ -26316,6 +27272,8 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26316
27272
|
jobId: syntheticId,
|
|
26317
27273
|
jobName: mat.expandedName,
|
|
26318
27274
|
...matrixValues && { matrixValues },
|
|
27275
|
+
...variantTrackingFields(mat),
|
|
27276
|
+
...wavePolicyFields(mat.expandedName),
|
|
26319
27277
|
runsOnLabels
|
|
26320
27278
|
});
|
|
26321
27279
|
rejectedJobs.push({
|
|
@@ -26334,6 +27292,8 @@ async function dispatchSingleOrchPath(args) {
|
|
|
26334
27292
|
jobId: result.jobId,
|
|
26335
27293
|
jobName: mat.expandedName,
|
|
26336
27294
|
...matrixValues && { matrixValues },
|
|
27295
|
+
...variantTrackingFields(mat),
|
|
27296
|
+
...wavePolicyFields(mat.expandedName),
|
|
26337
27297
|
runsOnLabels
|
|
26338
27298
|
});
|
|
26339
27299
|
logger$38.info("Job dispatched", {
|
|
@@ -26505,22 +27465,24 @@ async function dispatchExecutionAfterInit(args) {
|
|
|
26505
27465
|
const lockJob = mat.lockJob;
|
|
26506
27466
|
const matrixValues = mat.variantValues;
|
|
26507
27467
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
26508
|
-
const
|
|
26509
|
-
const
|
|
27468
|
+
const selectors = runsOnSelectorsForLockJob(lockJob);
|
|
27469
|
+
const runsOnLabels = selectors.runsOnLabels;
|
|
27470
|
+
const excludeLabels = selectors.excludeLabels;
|
|
26510
27471
|
const jobInput = buildExecutionJobInput({
|
|
26511
27472
|
ctx,
|
|
26512
27473
|
setup,
|
|
26513
27474
|
buildPrep,
|
|
26514
27475
|
buildJobConfig,
|
|
26515
27476
|
mat,
|
|
26516
|
-
|
|
26517
|
-
excludeLabels
|
|
27477
|
+
selectors
|
|
26518
27478
|
});
|
|
26519
27479
|
let dispatchStatus;
|
|
26520
27480
|
if (deps.coordinator && deps.coordinator.hasConnectedPeers()) {
|
|
26521
27481
|
const jobToRoute = {
|
|
26522
27482
|
jobName: mat.expandedName,
|
|
26523
27483
|
runsOnLabels: [runsOnLabels],
|
|
27484
|
+
runsOnPatterns: selectors.runsOnPatterns,
|
|
27485
|
+
excludePatterns: selectors.excludePatterns,
|
|
26524
27486
|
jobConfig: buildJobConfig(mat),
|
|
26525
27487
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26526
27488
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26716,7 +27678,7 @@ function startDeferredInitDispatch(args) {
|
|
|
26716
27678
|
jobId,
|
|
26717
27679
|
jobName: mat.expandedName,
|
|
26718
27680
|
...mat.variantValues && { matrixValues: mat.variantValues },
|
|
26719
|
-
runsOnLabels:
|
|
27681
|
+
runsOnLabels: runsOnSelectorsForLockJob(lockJob).runsOnLabels
|
|
26720
27682
|
}]).catch(() => {});
|
|
26721
27683
|
const carried = err instanceof AgentJobFailedError ? err.initFailure : void 0;
|
|
26722
27684
|
await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
|
|
@@ -26734,14 +27696,15 @@ function startDeferredInitDispatch(args) {
|
|
|
26734
27696
|
}
|
|
26735
27697
|
}
|
|
26736
27698
|
async function dispatchEvalJob(args) {
|
|
26737
|
-
const { ctx, setup, buildPrep, dynamicEntry } = args;
|
|
27699
|
+
const { ctx, setup, buildPrep, dynamicEntry, upstreamSnapshot } = args;
|
|
26738
27700
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
26739
27701
|
const evalJobName = `__dynamic__${workflow.name}__${dynamicEntry.source.index}`;
|
|
26740
27702
|
logger$38.info("Dispatching dynamic eval job", {
|
|
26741
27703
|
runId,
|
|
26742
27704
|
workflow: workflow.name,
|
|
26743
27705
|
evalJob: evalJobName,
|
|
26744
|
-
sourceIndex: dynamicEntry.source.index
|
|
27706
|
+
sourceIndex: dynamicEntry.source.index,
|
|
27707
|
+
resultAware: !!upstreamSnapshot
|
|
26745
27708
|
});
|
|
26746
27709
|
const evalJobInput = {
|
|
26747
27710
|
runId,
|
|
@@ -26759,7 +27722,12 @@ async function dispatchEvalJob(args) {
|
|
|
26759
27722
|
event,
|
|
26760
27723
|
timeoutMs: 12e4,
|
|
26761
27724
|
...workflow.contentHash && { contentHash: workflow.contentHash },
|
|
26762
|
-
...workflow.resolvedHashFiles?.length && { resolvedHashFiles: workflow.resolvedHashFiles }
|
|
27725
|
+
...workflow.resolvedHashFiles?.length && { resolvedHashFiles: workflow.resolvedHashFiles },
|
|
27726
|
+
...upstreamSnapshot && {
|
|
27727
|
+
resultAware: true,
|
|
27728
|
+
declaredNeeds: dynamicEntry.needs ?? [],
|
|
27729
|
+
upstreamSnapshot
|
|
27730
|
+
}
|
|
26763
27731
|
},
|
|
26764
27732
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26765
27733
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26774,38 +27742,65 @@ async function dispatchEvalJob(args) {
|
|
|
26774
27742
|
depsUrl: buildPrep.depsUrl,
|
|
26775
27743
|
depsHash: buildPrep.depsHash
|
|
26776
27744
|
};
|
|
27745
|
+
const replaceSyntheticId = upstreamSnapshot && deps.executionTracker ? await deps.executionTracker.findDynamicEvalSyntheticId(runId, evalJobName) : void 0;
|
|
26777
27746
|
const evalResult = await setup.dispatcher.dispatch(evalJobInput);
|
|
26778
27747
|
if (evalResult.status !== "dispatched" && evalResult.status !== "queued") throw new Error(`Dynamic eval job dispatch rejected: ${evalResult.status}`);
|
|
26779
|
-
if (deps.executionTracker) await deps.executionTracker.addJobsToRun(runId, [{
|
|
26780
|
-
jobId: evalResult.jobId,
|
|
26781
|
-
jobName: evalJobName,
|
|
26782
|
-
runsOnLabels: evalJobInput.runsOnLabels
|
|
26783
|
-
}]);
|
|
26784
27748
|
return {
|
|
26785
27749
|
evalJobId: evalResult.jobId,
|
|
26786
|
-
evalJobLabels: evalJobInput.runsOnLabels
|
|
27750
|
+
evalJobLabels: evalJobInput.runsOnLabels,
|
|
27751
|
+
evalJobName,
|
|
27752
|
+
replaceSyntheticId,
|
|
27753
|
+
runsOnLabels: evalJobInput.runsOnLabels
|
|
26787
27754
|
};
|
|
26788
27755
|
}
|
|
26789
27756
|
/**
|
|
26790
27757
|
* Resolve env/secrets per generated job and build their job configs.
|
|
26791
27758
|
* Skips jobs that fail individual secret resolution.
|
|
26792
27759
|
*/
|
|
27760
|
+
/**
|
|
27761
|
+
* Records a dropped generated-job matrix as a `matrix_expansion` init failure so
|
|
27762
|
+
* the run's dashboard surfaces it, mirroring the static / top-level dynamic-matrix
|
|
27763
|
+
* paths. A no-op when the run has no execution tracker.
|
|
27764
|
+
*/
|
|
27765
|
+
async function recordGeneratedMatrixFailure(deps, runId, err) {
|
|
27766
|
+
if (!deps.executionTracker) return;
|
|
27767
|
+
const jobId = `matrix-failed-${randomUUID()}`;
|
|
27768
|
+
await deps.executionTracker.addJobsToRun(runId, [{
|
|
27769
|
+
jobId,
|
|
27770
|
+
jobName: err.jobName,
|
|
27771
|
+
runsOnLabels: []
|
|
27772
|
+
}]).catch(() => {});
|
|
27773
|
+
await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
|
|
27774
|
+
error: err.message,
|
|
27775
|
+
initFailure: {
|
|
27776
|
+
scope: "job",
|
|
27777
|
+
category: InitFailureCategory.enum.matrix_expansion,
|
|
27778
|
+
message: err.message,
|
|
27779
|
+
jobName: err.jobName
|
|
27780
|
+
}
|
|
27781
|
+
}).catch(() => {});
|
|
27782
|
+
}
|
|
26793
27783
|
async function resolveGeneratedJobConfigs(args) {
|
|
26794
|
-
const { ctx, workflow, fullLockFile, resolvedSecrets, resolvedNamespacedSecrets, runPublicKeyBase64, npmRegistries, installEnvSecrets, generatedJobs, dynamicEntry } = args;
|
|
27784
|
+
const { ctx, workflow, fullLockFile, resolvedSecrets, resolvedNamespacedSecrets, runPublicKeyBase64, npmRegistries, installEnvSecrets, generatedJobs, dynamicEntry, upstreamSnapshot } = args;
|
|
26795
27785
|
const { deps, runId, resolvedOrgId, event } = ctx;
|
|
26796
27786
|
const out = [];
|
|
26797
27787
|
let fanout;
|
|
26798
|
-
|
|
26799
|
-
|
|
27788
|
+
const remaining = [...generatedJobs];
|
|
27789
|
+
for (;;) try {
|
|
27790
|
+
fanout = materializeFanout(remaining);
|
|
27791
|
+
break;
|
|
26800
27792
|
} catch (err) {
|
|
26801
|
-
if (err instanceof FanoutError)
|
|
26802
|
-
|
|
26803
|
-
|
|
26804
|
-
|
|
26805
|
-
|
|
26806
|
-
|
|
26807
|
-
|
|
26808
|
-
|
|
27793
|
+
if (!(err instanceof FanoutError)) throw err;
|
|
27794
|
+
logger$38.error("Dynamic generated job matrix materialization failed", {
|
|
27795
|
+
runId,
|
|
27796
|
+
job: err.jobName,
|
|
27797
|
+
error: err.message
|
|
27798
|
+
});
|
|
27799
|
+
await recordGeneratedMatrixFailure(deps, runId, err);
|
|
27800
|
+
const before = remaining.length;
|
|
27801
|
+
const idx = remaining.findIndex((j) => j.name === err.jobName);
|
|
27802
|
+
if (idx >= 0) remaining.splice(idx, 1);
|
|
27803
|
+
if (remaining.length === before) throw err;
|
|
26809
27804
|
}
|
|
26810
27805
|
const expectedJobNames = [...new Set(fanout.jobs.map((m) => m.baseName))];
|
|
26811
27806
|
const expandNeeds = (needs) => {
|
|
@@ -26883,9 +27878,14 @@ async function resolveGeneratedJobConfigs(args) {
|
|
|
26883
27878
|
dynamicSource: {
|
|
26884
27879
|
index: dynamicEntry.source.index,
|
|
26885
27880
|
event,
|
|
26886
|
-
expectedJobNames
|
|
27881
|
+
expectedJobNames,
|
|
27882
|
+
...upstreamSnapshot && {
|
|
27883
|
+
upstreamSnapshot,
|
|
27884
|
+
declaredNeeds: dynamicEntry.needs ?? []
|
|
27885
|
+
}
|
|
26887
27886
|
}
|
|
26888
27887
|
};
|
|
27888
|
+
const genSel = runsOnSelectorsForLockJob(genJob);
|
|
26889
27889
|
out.push({
|
|
26890
27890
|
genJob: {
|
|
26891
27891
|
...genJob,
|
|
@@ -26893,7 +27893,10 @@ async function resolveGeneratedJobConfigs(args) {
|
|
|
26893
27893
|
needs: expandedNeeds
|
|
26894
27894
|
},
|
|
26895
27895
|
genJobConfig,
|
|
26896
|
-
runsOnLabels:
|
|
27896
|
+
runsOnLabels: genSel.runsOnLabels,
|
|
27897
|
+
runsOnPatterns: genSel.runsOnPatterns,
|
|
27898
|
+
excludeLabels: genSel.excludeLabels,
|
|
27899
|
+
excludePatterns: genSel.excludePatterns,
|
|
26897
27900
|
...envelope.matrixValues && { matrixValues: envelope.matrixValues }
|
|
26898
27901
|
});
|
|
26899
27902
|
} catch (err) {
|
|
@@ -26924,13 +27927,15 @@ async function gateAndStoreNonRootGeneratedJobs(args) {
|
|
|
26924
27927
|
if_failed: need.ifFailed ?? "skip"
|
|
26925
27928
|
});
|
|
26926
27929
|
if (gatedEdgeRows.length > 0) await deps.db.insertInto("execution_job_needs").values(gatedEdgeRows).onConflict((oc) => oc.doNothing()).execute();
|
|
26927
|
-
for (const { genJob, genJobConfig, runsOnLabels, matrixValues } of gatedGeneratedConfigs) {
|
|
27930
|
+
for (const { genJob, genJobConfig, runsOnLabels, runsOnPatterns, excludeLabels, excludePatterns, matrixValues } of gatedGeneratedConfigs) {
|
|
26928
27931
|
const gatedJobInput = {
|
|
26929
27932
|
runId,
|
|
26930
27933
|
workflowName: workflow.name,
|
|
26931
27934
|
jobName: genJob.name,
|
|
26932
27935
|
runsOnLabels,
|
|
26933
|
-
|
|
27936
|
+
runsOnPatterns,
|
|
27937
|
+
excludeLabels,
|
|
27938
|
+
excludePatterns,
|
|
26934
27939
|
jobConfig: genJobConfig,
|
|
26935
27940
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26936
27941
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -26967,13 +27972,15 @@ async function gateAndStoreNonRootGeneratedJobs(args) {
|
|
|
26967
27972
|
async function directDispatchGeneratedJobs(args) {
|
|
26968
27973
|
const { ctx, setup, buildPrep, configs } = args;
|
|
26969
27974
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
26970
|
-
for (const { genJob, genJobConfig, runsOnLabels, matrixValues } of configs) try {
|
|
27975
|
+
for (const { genJob, genJobConfig, runsOnLabels, runsOnPatterns, excludeLabels, excludePatterns, matrixValues } of configs) try {
|
|
26971
27976
|
const genJobInput = {
|
|
26972
27977
|
runId,
|
|
26973
27978
|
workflowName: workflow.name,
|
|
26974
27979
|
jobName: genJob.name,
|
|
26975
27980
|
runsOnLabels,
|
|
26976
|
-
|
|
27981
|
+
runsOnPatterns,
|
|
27982
|
+
excludeLabels,
|
|
27983
|
+
excludePatterns,
|
|
26977
27984
|
jobConfig: genJobConfig,
|
|
26978
27985
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
26979
27986
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -27012,9 +28019,11 @@ async function routeRootGeneratedJobs(args) {
|
|
|
27012
28019
|
const { ctx, setup, buildPrep, rootGeneratedConfigs } = args;
|
|
27013
28020
|
const { deps, workflow, repoIdentifier, credentials, event, ref, runId, bundle } = ctx;
|
|
27014
28021
|
if (rootGeneratedConfigs.length === 0) return;
|
|
27015
|
-
const generatedJobsToRoute = rootGeneratedConfigs.map(({ genJob, genJobConfig, runsOnLabels }) => ({
|
|
28022
|
+
const generatedJobsToRoute = rootGeneratedConfigs.map(({ genJob, genJobConfig, runsOnLabels, runsOnPatterns, excludeLabels, excludePatterns }) => ({
|
|
27016
28023
|
jobName: genJob.name,
|
|
27017
28024
|
runsOnLabels: [runsOnLabels],
|
|
28025
|
+
runsOnPatterns,
|
|
28026
|
+
excludePatterns,
|
|
27018
28027
|
jobConfig: genJobConfig,
|
|
27019
28028
|
repoUrl: bundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
27020
28029
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -27023,7 +28032,7 @@ async function routeRootGeneratedJobs(args) {
|
|
|
27023
28032
|
sourceTarHash: buildPrep.contentHash || void 0,
|
|
27024
28033
|
depsUrl: buildPrep.depsUrl,
|
|
27025
28034
|
depsHash: buildPrep.depsHash,
|
|
27026
|
-
excludeLabels
|
|
28035
|
+
excludeLabels,
|
|
27027
28036
|
...genJob.resources ? { resources: genJob.resources } : {}
|
|
27028
28037
|
}));
|
|
27029
28038
|
if (!(deps.coordinator && deps.coordinator.hasConnectedPeers())) {
|
|
@@ -27216,18 +28225,140 @@ async function recomputeAndDispatchReady(args) {
|
|
|
27216
28225
|
if (skipJobRow) await deps.executionTracker.onJobStatus(runId, skipJobRow.job_id, ExecutionJobStatus.enum.skipped, Date.now(), void 0, { error: result.reason });
|
|
27217
28226
|
}
|
|
27218
28227
|
}
|
|
28228
|
+
/**
|
|
28229
|
+
* Split a result-aware generator's declared needs into static/named upstream job
|
|
28230
|
+
* names and dynamic-group names. Reuses the same normalized lock edge shapes the
|
|
28231
|
+
* static-job `needs` serializer produces.
|
|
28232
|
+
*/
|
|
28233
|
+
function splitDeclaredNeeds(needs) {
|
|
28234
|
+
const jobNames = [];
|
|
28235
|
+
const groupNames = [];
|
|
28236
|
+
for (const need of needs ?? []) if (typeof need === "string") jobNames.push(need);
|
|
28237
|
+
else if ("group" in need) groupNames.push(need.group);
|
|
28238
|
+
else if ("name" in need) jobNames.push(need.name);
|
|
28239
|
+
return {
|
|
28240
|
+
jobNames,
|
|
28241
|
+
groupNames
|
|
28242
|
+
};
|
|
28243
|
+
}
|
|
28244
|
+
/**
|
|
28245
|
+
* Register a result-aware generator's eval job as a deferred, needs-gated DAG
|
|
28246
|
+
* job: insert a synthetic pending execution_jobs row plus its execution_job_needs
|
|
28247
|
+
* edges, so the existing scheduler gates the eval exactly like any other job.
|
|
28248
|
+
* Group needs expand to their member job names (members already carry group_name
|
|
28249
|
+
* from setGroupNameAndResolveEdges on the group's own eval completion).
|
|
28250
|
+
*/
|
|
28251
|
+
async function registerDeferredEvalJob(args) {
|
|
28252
|
+
const { ctx, evalJobName, dynamicEntry } = args;
|
|
28253
|
+
const { deps, runId } = ctx;
|
|
28254
|
+
if (!deps.db) return;
|
|
28255
|
+
const { jobNames, groupNames } = splitDeclaredNeeds(dynamicEntry.needs);
|
|
28256
|
+
const groupMembers = [];
|
|
28257
|
+
for (const groupName of groupNames) {
|
|
28258
|
+
const members = await deps.db.selectFrom("execution_jobs").select("job_name").where("run_id", "=", runId).where("group_name", "=", groupName).execute();
|
|
28259
|
+
for (const m of members) groupMembers.push(m.job_name);
|
|
28260
|
+
}
|
|
28261
|
+
const upstreamNames = [...new Set([...jobNames, ...groupMembers])];
|
|
28262
|
+
const ifFailedByName = /* @__PURE__ */ new Map();
|
|
28263
|
+
for (const need of dynamicEntry.needs ?? []) if (typeof need === "object" && "name" in need) ifFailedByName.set(need.name, need.ifFailed ?? "skip");
|
|
28264
|
+
const groupIfFailed = /* @__PURE__ */ new Map();
|
|
28265
|
+
for (const need of dynamicEntry.needs ?? []) if (typeof need === "object" && "group" in need) groupIfFailed.set(need.group, need.ifFailed ?? "skip");
|
|
28266
|
+
const syntheticId = `dynamic-eval-pending-${evalJobName}-${randomUUID()}`;
|
|
28267
|
+
if (deps.executionTracker) await deps.executionTracker.addJobsToRun(runId, [{
|
|
28268
|
+
jobId: syntheticId,
|
|
28269
|
+
jobName: evalJobName,
|
|
28270
|
+
runsOnLabels: []
|
|
28271
|
+
}]);
|
|
28272
|
+
const edgeRows = upstreamNames.map((upstreamName) => ({
|
|
28273
|
+
run_id: runId,
|
|
28274
|
+
job_name: evalJobName,
|
|
28275
|
+
upstream_name: upstreamName,
|
|
28276
|
+
if_failed: ifFailedByName.get(upstreamName) ?? [...groupIfFailed.values()][0] ?? "skip"
|
|
28277
|
+
}));
|
|
28278
|
+
if (edgeRows.length > 0) await deps.db.insertInto("execution_job_needs").values(edgeRows).onConflict((oc) => oc.doNothing()).execute();
|
|
28279
|
+
logger$38.info("Registered deferred result-aware eval job", {
|
|
28280
|
+
runId,
|
|
28281
|
+
evalJob: evalJobName,
|
|
28282
|
+
upstreams: upstreamNames
|
|
28283
|
+
});
|
|
28284
|
+
const results = await recomputeNeedsSatisfied(deps.db, runId, [evalJobName]);
|
|
28285
|
+
for (const result of results) if (result.action === "dispatch" && deps.executionTracker?.onJobReadyCallback) await deps.executionTracker.onJobReadyCallback(runId, evalJobName);
|
|
28286
|
+
else if (result.action === "skip") {
|
|
28287
|
+
if (deps.executionTracker?.onJobReadyCallback) await deps.executionTracker.onJobReadyCallback(runId, evalJobName);
|
|
28288
|
+
}
|
|
28289
|
+
}
|
|
28290
|
+
/**
|
|
28291
|
+
* Gather the frozen upstream snapshot for a result-aware eval: each declared
|
|
28292
|
+
* job/group-member's stored outputs (the same plain outputs map that backs
|
|
28293
|
+
* jobRef.result), plus group → ordered member names. Captured once, at eval
|
|
28294
|
+
* dispatch, and replayed unchanged on agent-side re-eval.
|
|
28295
|
+
*/
|
|
28296
|
+
async function gatherUpstreamSnapshot(args) {
|
|
28297
|
+
const { ctx, dynamicEntry } = args;
|
|
28298
|
+
const { deps, runId } = ctx;
|
|
28299
|
+
const snapshot = {
|
|
28300
|
+
jobs: {},
|
|
28301
|
+
groups: {}
|
|
28302
|
+
};
|
|
28303
|
+
if (!deps.db) return snapshot;
|
|
28304
|
+
const { jobNames, groupNames } = splitDeclaredNeeds(dynamicEntry.needs);
|
|
28305
|
+
const groupMembers = [];
|
|
28306
|
+
for (const groupName of groupNames) {
|
|
28307
|
+
const memberNames = (await deps.db.selectFrom("execution_jobs").select(["job_name", "ready_at"]).where("run_id", "=", runId).where("group_name", "=", groupName).orderBy("ready_at", "asc").orderBy("job_name", "asc").execute()).map((m) => m.job_name);
|
|
28308
|
+
snapshot.groups[groupName] = memberNames;
|
|
28309
|
+
for (const n of memberNames) groupMembers.push(n);
|
|
28310
|
+
}
|
|
28311
|
+
const allJobNames = [...new Set([...jobNames, ...groupMembers])];
|
|
28312
|
+
if (allJobNames.length > 0) {
|
|
28313
|
+
const rows = await deps.db.selectFrom("execution_jobs").select(["job_name", "outputs"]).where("run_id", "=", runId).where("job_name", "in", allJobNames).execute();
|
|
28314
|
+
for (const row of rows) {
|
|
28315
|
+
const parsed = parseOutputsCell(row.outputs);
|
|
28316
|
+
if (parsed) snapshot.jobs[row.job_name] = parsed;
|
|
28317
|
+
}
|
|
28318
|
+
}
|
|
28319
|
+
return snapshot;
|
|
28320
|
+
}
|
|
27219
28321
|
async function processDynamicEntry(args) {
|
|
27220
28322
|
const { ctx, setup, buildPrep, secrets, dynamicEntry } = args;
|
|
27221
28323
|
const { deps, workflow, fullLockFile, runId } = ctx;
|
|
27222
28324
|
if (!deps.pendingDynamics) return;
|
|
28325
|
+
const evalJobName = `__dynamic__${workflow.name}__${dynamicEntry.source.index}`;
|
|
27223
28326
|
try {
|
|
27224
|
-
|
|
28327
|
+
let upstreamSnapshot;
|
|
28328
|
+
if (dynamicEntry.resultAware) {
|
|
28329
|
+
const gateOpened = trackEvalGate(runId, evalJobName);
|
|
28330
|
+
await registerDeferredEvalJob({
|
|
28331
|
+
ctx,
|
|
28332
|
+
evalJobName,
|
|
28333
|
+
dynamicEntry
|
|
28334
|
+
});
|
|
28335
|
+
await gateOpened;
|
|
28336
|
+
upstreamSnapshot = await gatherUpstreamSnapshot({
|
|
28337
|
+
ctx,
|
|
28338
|
+
dynamicEntry
|
|
28339
|
+
});
|
|
28340
|
+
logger$38.info("Result-aware eval gate opened, dispatching eval with snapshot", {
|
|
28341
|
+
runId,
|
|
28342
|
+
workflow: workflow.name,
|
|
28343
|
+
evalJob: evalJobName,
|
|
28344
|
+
snapshotJobs: Object.keys(upstreamSnapshot.jobs).length,
|
|
28345
|
+
snapshotGroups: Object.keys(upstreamSnapshot.groups).length
|
|
28346
|
+
});
|
|
28347
|
+
}
|
|
28348
|
+
const { evalJobId, replaceSyntheticId, runsOnLabels } = await dispatchEvalJob({
|
|
27225
28349
|
ctx,
|
|
27226
28350
|
setup,
|
|
27227
28351
|
buildPrep,
|
|
27228
|
-
dynamicEntry
|
|
28352
|
+
dynamicEntry,
|
|
28353
|
+
upstreamSnapshot
|
|
27229
28354
|
});
|
|
27230
|
-
const
|
|
28355
|
+
const generatedJobsPromise = deps.pendingDynamics.track(evalJobId);
|
|
28356
|
+
if (deps.executionTracker) await deps.executionTracker.addJobsToRun(runId, [{
|
|
28357
|
+
jobId: evalJobId,
|
|
28358
|
+
jobName: evalJobName,
|
|
28359
|
+
runsOnLabels
|
|
28360
|
+
}], void 0, replaceSyntheticId);
|
|
28361
|
+
const generatedJobs = await generatedJobsPromise;
|
|
27231
28362
|
logger$38.info("Dynamic eval completed, dispatching generated jobs", {
|
|
27232
28363
|
runId,
|
|
27233
28364
|
workflow: workflow.name,
|
|
@@ -27244,7 +28375,8 @@ async function processDynamicEntry(args) {
|
|
|
27244
28375
|
npmRegistries: secrets.npmRegistries,
|
|
27245
28376
|
installEnvSecrets: secrets.installEnvSecrets,
|
|
27246
28377
|
generatedJobs,
|
|
27247
|
-
dynamicEntry
|
|
28378
|
+
dynamicEntry,
|
|
28379
|
+
upstreamSnapshot
|
|
27248
28380
|
});
|
|
27249
28381
|
const rootGeneratedConfigs = generatedJobConfigs.filter((c) => isRootJob(c.genJob));
|
|
27250
28382
|
await gateAndStoreNonRootGeneratedJobs({
|
|
@@ -27562,6 +28694,8 @@ async function dispatchMatchedWorkflow(ctx, opts = {}) {
|
|
|
27562
28694
|
}
|
|
27563
28695
|
var logger$38;
|
|
27564
28696
|
var init_dispatch_matched_workflow = __esmMin((() => {
|
|
28697
|
+
init_host_roster();
|
|
28698
|
+
init_orchestrator_core();
|
|
27565
28699
|
init_agent_job_failed_error();
|
|
27566
28700
|
init_pipeline();
|
|
27567
28701
|
init_environment_store();
|
|
@@ -28217,7 +29351,9 @@ function buildGlobalWorkflowJobInputs(args) {
|
|
|
28217
29351
|
const materialized = materializeFanout(globalWorkflow.jobs.filter(isLockStaticJob)).jobs;
|
|
28218
29352
|
for (const mat of materialized) {
|
|
28219
29353
|
const lockJob = mat.lockJob;
|
|
28220
|
-
const
|
|
29354
|
+
const runsOnParts = partitionMatchers(lockJob.runsOn ?? []);
|
|
29355
|
+
const excludeParts = partitionMatchers(lockJob.excludeLabels ?? []);
|
|
29356
|
+
const flatLabels = runsOnParts.exact;
|
|
28221
29357
|
const jobConfig = {
|
|
28222
29358
|
source: globalWorkflow.source ?? reg.lockEntry.source,
|
|
28223
29359
|
workflowName: globalWorkflow.name,
|
|
@@ -28240,7 +29376,9 @@ function buildGlobalWorkflowJobInputs(args) {
|
|
|
28240
29376
|
workflowName: globalWorkflow.name,
|
|
28241
29377
|
jobName: mat.expandedName,
|
|
28242
29378
|
runsOnLabels: flatLabels,
|
|
28243
|
-
|
|
29379
|
+
runsOnPatterns: runsOnParts.regex,
|
|
29380
|
+
excludeLabels: excludeParts.exact,
|
|
29381
|
+
excludePatterns: excludeParts.regex,
|
|
28244
29382
|
jobConfig,
|
|
28245
29383
|
repoUrl: dispatchBundle.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
28246
29384
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
@@ -28845,6 +29983,36 @@ var init_process_webhook = __esmMin((() => {
|
|
|
28845
29983
|
* Decision traces are forwarded to Platform via platformClient.send() (which buffers
|
|
28846
29984
|
* internally when disconnected -- the caller does NOT check connection state).
|
|
28847
29985
|
*/
|
|
29986
|
+
function evalGateKey(runId, evalJobName) {
|
|
29987
|
+
return `${runId}:${evalJobName}`;
|
|
29988
|
+
}
|
|
29989
|
+
/**
|
|
29990
|
+
* Register an eval gate and return a promise that resolves when the scheduler
|
|
29991
|
+
* opens it (the eval job's upstream needs are all satisfied).
|
|
29992
|
+
*/
|
|
29993
|
+
function trackEvalGate(runId, evalJobName) {
|
|
29994
|
+
return new Promise((resolve) => {
|
|
29995
|
+
pendingEvalGates.set(evalGateKey(runId, evalJobName), resolve);
|
|
29996
|
+
});
|
|
29997
|
+
}
|
|
29998
|
+
/**
|
|
29999
|
+
* Open a registered eval gate, unblocking the deferred dispatch task. Returns
|
|
30000
|
+
* true if a gate was registered for this eval job (so the scheduler knows it
|
|
30001
|
+
* handled the ready signal itself and must not run the normal dispatch path).
|
|
30002
|
+
*/
|
|
30003
|
+
function openEvalGate(runId, evalJobName) {
|
|
30004
|
+
const key = evalGateKey(runId, evalJobName);
|
|
30005
|
+
const resolve = pendingEvalGates.get(key);
|
|
30006
|
+
if (!resolve) return false;
|
|
30007
|
+
pendingEvalGates.delete(key);
|
|
30008
|
+
resolve();
|
|
30009
|
+
return true;
|
|
30010
|
+
}
|
|
30011
|
+
/** Clear all eval gates for a run (called on run completion / cleanup). */
|
|
30012
|
+
function clearEvalGatesForRun(runId) {
|
|
30013
|
+
const prefix = `${runId}:`;
|
|
30014
|
+
for (const key of pendingEvalGates.keys()) if (key.startsWith(prefix)) pendingEvalGates.delete(key);
|
|
30015
|
+
}
|
|
28848
30016
|
/**
|
|
28849
30017
|
* Store a pending dispatch context for a job that will be dispatched later
|
|
28850
30018
|
* by the needs scheduler. The key is `${runId}:${jobName}`.
|
|
@@ -29294,11 +30462,12 @@ function summarizeDecision(decision) {
|
|
|
29294
30462
|
checksCount: decision.checks.length
|
|
29295
30463
|
};
|
|
29296
30464
|
}
|
|
29297
|
-
var logger$36, pendingJobContexts;
|
|
30465
|
+
var logger$36, pendingJobContexts, pendingEvalGates;
|
|
29298
30466
|
var init_processor = __esmMin((() => {
|
|
29299
30467
|
init_process_webhook();
|
|
29300
30468
|
logger$36 = createLogger({ prefix: "pipeline" });
|
|
29301
30469
|
pendingJobContexts = /* @__PURE__ */ new Map();
|
|
30470
|
+
pendingEvalGates = /* @__PURE__ */ new Map();
|
|
29302
30471
|
}));
|
|
29303
30472
|
//#endregion
|
|
29304
30473
|
//#region src/concurrency/waiters.ts
|
|
@@ -29712,6 +30881,7 @@ var init_admin_event_dlq = __esmMin((() => {
|
|
|
29712
30881
|
* @returns Object with Hono app instance and injectWebSocket function
|
|
29713
30882
|
*/
|
|
29714
30883
|
function createApp(deps) {
|
|
30884
|
+
registerOrchestratorMetrics();
|
|
29715
30885
|
const app = new Hono().basePath(deps.config.basePath);
|
|
29716
30886
|
const { injectWebSocket, upgradeWebSocket, wss } = createNodeWebSocket({ app });
|
|
29717
30887
|
configureSecureWsServer(wss);
|
|
@@ -30224,7 +31394,11 @@ function createApp(deps) {
|
|
|
30224
31394
|
eventLog: deps.eventLogWriter,
|
|
30225
31395
|
eventLogSource: "direct",
|
|
30226
31396
|
contributorCache: deps.contributorCache,
|
|
30227
|
-
accessLogWriter: deps.accessLogWriter
|
|
31397
|
+
accessLogWriter: deps.accessLogWriter,
|
|
31398
|
+
hostRosterStore: deps.hostRosterStore,
|
|
31399
|
+
instanceId: deps.config.instanceId,
|
|
31400
|
+
rosterGraceMs: deps.config.rosterGraceMs,
|
|
31401
|
+
maxFanoutHosts: deps.config.maxFanoutHosts
|
|
30228
31402
|
});
|
|
30229
31403
|
} catch (err) {
|
|
30230
31404
|
if (deps.eventLogWriter) try {
|
|
@@ -30556,7 +31730,7 @@ async function createDebugBundle(options) {
|
|
|
30556
31730
|
const dir = path$1.dirname(outputPath);
|
|
30557
31731
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
30558
31732
|
const output = fs.createWriteStream(outputPath);
|
|
30559
|
-
const archive =
|
|
31733
|
+
const archive = new ZipArchive({ zlib: { level: 6 } });
|
|
30560
31734
|
const finalized = new Promise((resolve, reject) => {
|
|
30561
31735
|
output.on("close", resolve);
|
|
30562
31736
|
archive.on("error", reject);
|
|
@@ -30632,7 +31806,7 @@ function classifyError(err) {
|
|
|
30632
31806
|
};
|
|
30633
31807
|
}
|
|
30634
31808
|
async function collectFleetSubtree(opts, deps) {
|
|
30635
|
-
const archive =
|
|
31809
|
+
const archive = new ZipArchive({ zlib: { level: 6 } });
|
|
30636
31810
|
const out = [];
|
|
30637
31811
|
archive.on("data", (d) => out.push(d));
|
|
30638
31812
|
const done = new Promise((res, rej) => {
|
|
@@ -37028,6 +38202,61 @@ var init_log_storage = __esmMin((() => {
|
|
|
37028
38202
|
init_s3_log_storage();
|
|
37029
38203
|
}));
|
|
37030
38204
|
//#endregion
|
|
38205
|
+
//#region src/pipeline/wave-scheduler.ts
|
|
38206
|
+
/**
|
|
38207
|
+
* Decide what happens after a fan-out child of `baseJobName` reaches terminal.
|
|
38208
|
+
*
|
|
38209
|
+
* The wave policy (`maxParallel` / `failFast`) is read from the base group's
|
|
38210
|
+
* own rows — every child of a bounded wave carries the same stamped
|
|
38211
|
+
* `wave_max_parallel` / `wave_fail_fast`, so the just-completed child's slot
|
|
38212
|
+
* being re-inserted without the policy on release does not break the chain
|
|
38213
|
+
* (the still-held siblings carry it). If no sibling carries a policy, this is
|
|
38214
|
+
* not a bounded wave → `noop`.
|
|
38215
|
+
*
|
|
38216
|
+
* - `failFast` + a child failure → `skip-remaining` every still-held sibling.
|
|
38217
|
+
* - in-flight count `< maxParallel` AND a held sibling exists → `release` the
|
|
38218
|
+
* next held sibling (lowest `variant_label`).
|
|
38219
|
+
* - otherwise → `noop`.
|
|
38220
|
+
*
|
|
38221
|
+
* "In-flight" = a non-terminal, non-`wave_gated` child (it has been dispatched
|
|
38222
|
+
* and not yet completed). The just-completed child is terminal, so it does not
|
|
38223
|
+
* count against the window — its slot is the one we are filling.
|
|
38224
|
+
*/
|
|
38225
|
+
async function evaluateWave(db, evaluation) {
|
|
38226
|
+
const { runId, baseJobName, completedStatus } = evaluation;
|
|
38227
|
+
const children = await db.selectFrom("execution_jobs").select([
|
|
38228
|
+
"job_name",
|
|
38229
|
+
"status",
|
|
38230
|
+
"wave_gated",
|
|
38231
|
+
"variant_label",
|
|
38232
|
+
"wave_max_parallel",
|
|
38233
|
+
"wave_fail_fast"
|
|
38234
|
+
]).where("run_id", "=", runId).where("base_job_name", "=", baseJobName).execute();
|
|
38235
|
+
const policyRow = children.find((c) => c.wave_max_parallel != null);
|
|
38236
|
+
if (!policyRow || policyRow.wave_max_parallel == null) return { action: "noop" };
|
|
38237
|
+
const maxParallel = policyRow.wave_max_parallel;
|
|
38238
|
+
const failFast = policyRow.wave_fail_fast ?? false;
|
|
38239
|
+
const heldSiblings = children.filter((c) => c.wave_gated).sort((a, b) => (a.variant_label ?? a.job_name).localeCompare(b.variant_label ?? b.job_name));
|
|
38240
|
+
const isFailure = completedStatus !== ExecutionJobStatus.enum.success;
|
|
38241
|
+
if (failFast && isFailure) {
|
|
38242
|
+
if (heldSiblings.length === 0) return { action: "noop" };
|
|
38243
|
+
return {
|
|
38244
|
+
action: "skip-remaining",
|
|
38245
|
+
jobNames: heldSiblings.map((c) => c.job_name)
|
|
38246
|
+
};
|
|
38247
|
+
}
|
|
38248
|
+
if (heldSiblings.length === 0) return { action: "noop" };
|
|
38249
|
+
if (children.filter((c) => !c.wave_gated && !TERMINAL_JOB_STATES.has(c.status)).length >= maxParallel) return { action: "noop" };
|
|
38250
|
+
return {
|
|
38251
|
+
action: "release",
|
|
38252
|
+
jobName: heldSiblings[0].job_name,
|
|
38253
|
+
baseJobName,
|
|
38254
|
+
maxParallel,
|
|
38255
|
+
failFast
|
|
38256
|
+
};
|
|
38257
|
+
}
|
|
38258
|
+
var init_wave_scheduler = __esmMin((() => {}));
|
|
38259
|
+
//#endregion
|
|
37031
38260
|
//#region src/reporting/execution-tracker.ts
|
|
37032
38261
|
/**
|
|
37033
38262
|
* Execution state tracker with write-through DB persistence.
|
|
@@ -37046,6 +38275,7 @@ var logger$24, PRUNE_DELAY_MS, ExecutionTracker;
|
|
|
37046
38275
|
var init_execution_tracker = __esmMin((() => {
|
|
37047
38276
|
init_prometheus();
|
|
37048
38277
|
init_needs_scheduler();
|
|
38278
|
+
init_wave_scheduler();
|
|
37049
38279
|
logger$24 = createLogger({ prefix: "execution-tracker" });
|
|
37050
38280
|
PRUNE_DELAY_MS = 300 * 1e3;
|
|
37051
38281
|
ExecutionTracker = class {
|
|
@@ -37177,10 +38407,22 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37177
38407
|
job_name: job.jobName,
|
|
37178
38408
|
routing_key: routingKey ?? null,
|
|
37179
38409
|
matrix_values: job.matrixValues ? JSON.stringify(job.matrixValues) : null,
|
|
38410
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38411
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38412
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
38413
|
+
...job.waveGated && { wave_gated: true },
|
|
38414
|
+
...job.waveMaxParallel !== void 0 && { wave_max_parallel: job.waveMaxParallel },
|
|
38415
|
+
...job.waveFailFast !== void 0 && { wave_fail_fast: job.waveFailFast },
|
|
37180
38416
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37181
38417
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37182
38418
|
}).onConflict((oc) => oc.columns(["run_id", "job_id"]).doUpdateSet({
|
|
37183
38419
|
job_name: job.jobName,
|
|
38420
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38421
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38422
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
38423
|
+
...job.waveGated && { wave_gated: true },
|
|
38424
|
+
...job.waveMaxParallel !== void 0 && { wave_max_parallel: job.waveMaxParallel },
|
|
38425
|
+
...job.waveFailFast !== void 0 && { wave_fail_fast: job.waveFailFast },
|
|
37184
38426
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37185
38427
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37186
38428
|
})).execute();
|
|
@@ -37235,6 +38477,20 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37235
38477
|
return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
|
|
37236
38478
|
}
|
|
37237
38479
|
/**
|
|
38480
|
+
* Find the synthetic deferred-eval placeholder job ID for a result-aware
|
|
38481
|
+
* dynamic generator's eval job. Mirrors {@link findSyntheticJobId} but keys on
|
|
38482
|
+
* the `dynamic-eval-pending-<evalJobName>-` prefix that registerDeferredEvalJob
|
|
38483
|
+
* uses, so dispatchEvalJob can swap it for the real eval job id.
|
|
38484
|
+
*/
|
|
38485
|
+
async findDynamicEvalSyntheticId(runId, evalJobName) {
|
|
38486
|
+
const run = this.runs.get(runId);
|
|
38487
|
+
const prefix = `dynamic-eval-pending-${evalJobName}-`;
|
|
38488
|
+
if (run) {
|
|
38489
|
+
for (const key of run.jobs.keys()) if (key.startsWith(prefix)) return key;
|
|
38490
|
+
}
|
|
38491
|
+
return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", evalJobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
|
|
38492
|
+
}
|
|
38493
|
+
/**
|
|
37238
38494
|
* Run `fn` while holding a per-run lock, serializing the run-mutating methods
|
|
37239
38495
|
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|
|
37240
38496
|
* synthetic→real job swap and wedge the run in `running`.
|
|
@@ -37317,10 +38573,16 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37317
38573
|
job_name: job.jobName,
|
|
37318
38574
|
routing_key: run.routingKey ?? null,
|
|
37319
38575
|
matrix_values: job.matrixValues ? JSON.stringify(job.matrixValues) : null,
|
|
38576
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38577
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38578
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
37320
38579
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37321
38580
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37322
38581
|
}).onConflict((oc) => oc.columns(["run_id", "job_id"]).doUpdateSet({
|
|
37323
38582
|
job_name: job.jobName,
|
|
38583
|
+
...job.baseJobName && { base_job_name: job.baseJobName },
|
|
38584
|
+
...job.variantKind && { variant_kind: job.variantKind },
|
|
38585
|
+
...job.variantLabel && { variant_label: job.variantLabel },
|
|
37324
38586
|
...runsOnLabelsJson && { runs_on_labels: runsOnLabelsJson },
|
|
37325
38587
|
...dispatchedContexts?.length && { dispatched_contexts: JSON.stringify(dispatchedContexts) }
|
|
37326
38588
|
})).execute();
|
|
@@ -37421,7 +38683,10 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37421
38683
|
workflowName: run.workflowName
|
|
37422
38684
|
});
|
|
37423
38685
|
if (TERMINAL_JOB_STATES.has(state) && data?.droppedJobs && Array.isArray(data.droppedJobs)) await this.handleDriftDroppedJobs(runId, data.droppedJobs);
|
|
37424
|
-
if (TERMINAL_JOB_STATES.has(state) && job)
|
|
38686
|
+
if (TERMINAL_JOB_STATES.has(state) && job) {
|
|
38687
|
+
await this.runSchedulerHook(runId, jobId, job.name, state);
|
|
38688
|
+
await this.runWaveSchedulerHook(runId, jobId, state);
|
|
38689
|
+
}
|
|
37425
38690
|
if (TERMINAL_JOB_STATES.has(state) && run && this.isRunComplete(runId)) {
|
|
37426
38691
|
if (await this.enforceSchedulerInvariantOrFail(runId)) return;
|
|
37427
38692
|
}
|
|
@@ -37725,6 +38990,60 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
37725
38990
|
}
|
|
37726
38991
|
}
|
|
37727
38992
|
/**
|
|
38993
|
+
* Rolling-wave hook: fires beside the needs-scheduler when a fan-out child of
|
|
38994
|
+
* a bounded wave (`maxParallel` set) reaches terminal. Reads the completed
|
|
38995
|
+
* child's row to recover the base + wave policy, asks {@link evaluateWave}
|
|
38996
|
+
* what to do, then performs it:
|
|
38997
|
+
*
|
|
38998
|
+
* - `release`: clear the next held sibling's `wave_gated` flag and fire the
|
|
38999
|
+
* onJobReady callback (the existing ready→dispatch path).
|
|
39000
|
+
* - `skip-remaining`: mark every still-held sibling `skipped` (failFast).
|
|
39001
|
+
* - `noop`: nothing — a later terminal will free the next slot.
|
|
39002
|
+
*/
|
|
39003
|
+
async runWaveSchedulerHook(runId, jobId, state) {
|
|
39004
|
+
try {
|
|
39005
|
+
const row = await this.db.selectFrom("execution_jobs").select(["base_job_name"]).where("run_id", "=", runId).where("job_id", "=", jobId).executeTakeFirst();
|
|
39006
|
+
if (!row?.base_job_name) return;
|
|
39007
|
+
const result = await evaluateWave(this.db, {
|
|
39008
|
+
runId,
|
|
39009
|
+
baseJobName: row.base_job_name,
|
|
39010
|
+
completedStatus: state
|
|
39011
|
+
});
|
|
39012
|
+
if (result.action === "release") {
|
|
39013
|
+
await this.db.updateTable("execution_jobs").set({ wave_gated: false }).where("run_id", "=", runId).where("job_name", "=", result.jobName).execute();
|
|
39014
|
+
if (this.onJobReadyCallback) await this.onJobReadyCallback(runId, result.jobName);
|
|
39015
|
+
await this.db.updateTable("execution_jobs").set({
|
|
39016
|
+
base_job_name: result.baseJobName,
|
|
39017
|
+
wave_max_parallel: result.maxParallel,
|
|
39018
|
+
wave_fail_fast: result.failFast
|
|
39019
|
+
}).where("run_id", "=", runId).where("job_name", "=", result.jobName).execute();
|
|
39020
|
+
logger$24.info("Rolling wave released next child", {
|
|
39021
|
+
runId,
|
|
39022
|
+
baseJobName: row.base_job_name,
|
|
39023
|
+
released: result.jobName
|
|
39024
|
+
});
|
|
39025
|
+
} else if (result.action === "skip-remaining") {
|
|
39026
|
+
logger$24.info("Rolling wave halting (failFast): skipping held remainder", {
|
|
39027
|
+
runId,
|
|
39028
|
+
baseJobName: row.base_job_name,
|
|
39029
|
+
skipped: result.jobNames
|
|
39030
|
+
});
|
|
39031
|
+
for (const jobName of result.jobNames) {
|
|
39032
|
+
const heldRow = await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).executeTakeFirst();
|
|
39033
|
+
if (!heldRow) continue;
|
|
39034
|
+
await this.db.updateTable("execution_jobs").set({ wave_gated: false }).where("run_id", "=", runId).where("job_id", "=", heldRow.job_id).execute();
|
|
39035
|
+
await this.onJobStatus(runId, heldRow.job_id, ExecutionJobStatus.enum.skipped, Date.now(), void 0, { error: "fan-out halted by failFast" });
|
|
39036
|
+
}
|
|
39037
|
+
}
|
|
39038
|
+
} catch (e) {
|
|
39039
|
+
logger$24.error("Wave scheduler hook failed", {
|
|
39040
|
+
runId,
|
|
39041
|
+
jobId,
|
|
39042
|
+
error: e
|
|
39043
|
+
});
|
|
39044
|
+
}
|
|
39045
|
+
}
|
|
39046
|
+
/**
|
|
37728
39047
|
* Phase 9: stuck-jobs invariant check ( Layer 3).
|
|
37729
39048
|
* Before declaring a run complete, verify no stuck jobs exist. If any are
|
|
37730
39049
|
* found, fail them via recursive onJobStatus calls and signal the caller to
|
|
@@ -40691,6 +42010,8 @@ var init_pg_secret_store = __esmMin((() => {
|
|
|
40691
42010
|
async renameScope(orgId, oldScope, newScope) {
|
|
40692
42011
|
await this.db.transaction().execute(async (trx) => {
|
|
40693
42012
|
const rows = await trx.selectFrom("scoped_secrets").selectAll().where("org_id", "=", orgId).where("scope", "=", oldScope).execute();
|
|
42013
|
+
const bindings = await trx.selectFrom("environment_bindings").select("id").where("org_id", "=", orgId).where("scope_pattern", "=", oldScope).execute();
|
|
42014
|
+
if (rows.length === 0 && bindings.length === 0) throw new Error(`Secret scope '${oldScope}' not found`);
|
|
40694
42015
|
for (const row of rows) {
|
|
40695
42016
|
const oldAad = `${orgId}:${oldScope}:${row.key}`;
|
|
40696
42017
|
const newAad = `${orgId}:${newScope}:${row.key}`;
|
|
@@ -44723,9 +46044,12 @@ var init_disk_guard = __esmMin((() => {
|
|
|
44723
46044
|
*/
|
|
44724
46045
|
var orchestrator_core_exports = /* @__PURE__ */ __exportAll({
|
|
44725
46046
|
bootstrapOrchestrator: () => bootstrapOrchestrator$1,
|
|
46047
|
+
buildHostOutputsEnvelope: () => buildHostOutputsEnvelope,
|
|
44726
46048
|
buildMatrixOutputsEnvelope: () => buildMatrixOutputsEnvelope,
|
|
44727
46049
|
buildUpstreamOutputsByBase: () => buildUpstreamOutputsByBase,
|
|
46050
|
+
internalJobRunsOnSelectors: () => internalJobRunsOnSelectors,
|
|
44728
46051
|
mergeUpstreamOutputs: () => mergeUpstreamOutputs,
|
|
46052
|
+
parseOutputsCell: () => parseOutputsCell,
|
|
44729
46053
|
upstreamBaseNamesFromNeeds: () => upstreamBaseNamesFromNeeds
|
|
44730
46054
|
});
|
|
44731
46055
|
async function initializeScaler(config, db, tokenStore, onScalerEvent) {
|
|
@@ -45054,6 +46378,23 @@ function upstreamBaseNamesFromNeeds(needs) {
|
|
|
45054
46378
|
}
|
|
45055
46379
|
return names;
|
|
45056
46380
|
}
|
|
46381
|
+
/**
|
|
46382
|
+
* Partition a lock job's `runsOn` / `excludeLabels` matchers into exact label
|
|
46383
|
+
* strings and regex patterns for internal-event (cron / `ctx.emit`) dispatch.
|
|
46384
|
+
* Lock jobs carry `runsOn` as `LabelMatcher[]`; the coordinator routing and the
|
|
46385
|
+
* direct dispatcher both need exact labels for the indexed/SQL fast path and
|
|
46386
|
+
* regex patterns as a separate JS post-filter — never the raw matcher objects.
|
|
46387
|
+
*/
|
|
46388
|
+
function internalJobRunsOnSelectors(job) {
|
|
46389
|
+
const include = partitionMatchers(job.runsOn ?? []);
|
|
46390
|
+
const exclude = partitionMatchers(job.excludeLabels ?? []);
|
|
46391
|
+
return {
|
|
46392
|
+
runsOnLabels: include.exact,
|
|
46393
|
+
runsOnPatterns: include.regex,
|
|
46394
|
+
excludeLabels: exclude.exact,
|
|
46395
|
+
excludePatterns: exclude.regex
|
|
46396
|
+
};
|
|
46397
|
+
}
|
|
45057
46398
|
/** Escape SQL LIKE wildcards (`%`, `_`) so a literal base name matches exactly. */
|
|
45058
46399
|
function escapeLikePattern(value) {
|
|
45059
46400
|
return value.replace(/[\\%_]/g, (c) => `\\${c}`);
|
|
@@ -45081,6 +46422,16 @@ function parseOutputsCell(outputs) {
|
|
|
45081
46422
|
function buildUpstreamOutputsByBase(baseNames, rows) {
|
|
45082
46423
|
let result;
|
|
45083
46424
|
for (const base of baseNames) {
|
|
46425
|
+
const hostChildren = rows.filter((r) => r.variant_kind === VariantKind.host && r.job_name.startsWith(`${base} (`)).map((r) => ({
|
|
46426
|
+
host: r.variant_label ?? r.job_name.slice(base.length + 2, -1),
|
|
46427
|
+
status: r.status ?? null,
|
|
46428
|
+
parsed: parseOutputsCell(r.outputs) ?? {}
|
|
46429
|
+
}));
|
|
46430
|
+
if (hostChildren.length > 0) {
|
|
46431
|
+
if (!result) result = {};
|
|
46432
|
+
result[base] = buildHostOutputsEnvelope(hostChildren);
|
|
46433
|
+
continue;
|
|
46434
|
+
}
|
|
45084
46435
|
const exact = rows.find((r) => r.job_name === base && !r.matrix_values);
|
|
45085
46436
|
const children = rows.filter((r) => r.matrix_values && r.job_name.startsWith(`${base} (`)).map((r) => ({
|
|
45086
46437
|
job_name: r.job_name,
|
|
@@ -45100,6 +46451,34 @@ function buildUpstreamOutputsByBase(baseNames, rows) {
|
|
|
45100
46451
|
return result;
|
|
45101
46452
|
}
|
|
45102
46453
|
/**
|
|
46454
|
+
* Fold a `runsOnAll` upstream's host children into the `byHost` envelope
|
|
46455
|
+
* `{ byHost: { '<host>': outputs }, summary: { succeededHosts, failedHosts, outputs } }`.
|
|
46456
|
+
* Unlike the matrix envelope, `summary.outputs[key]` is an array view across hosts
|
|
46457
|
+
* (host order), never a last-write-wins scalar; `succeededHosts`/`failedHosts`
|
|
46458
|
+
* record each host's terminal outcome.
|
|
46459
|
+
*/
|
|
46460
|
+
function buildHostOutputsEnvelope(children) {
|
|
46461
|
+
const byHost = {};
|
|
46462
|
+
const succeededHosts = [];
|
|
46463
|
+
const failedHosts = [];
|
|
46464
|
+
const outputs = {};
|
|
46465
|
+
const ordered = [...children].sort((a, b) => a.host.localeCompare(b.host));
|
|
46466
|
+
for (const child of ordered) {
|
|
46467
|
+
byHost[child.host] = child.parsed;
|
|
46468
|
+
if (child.status === ExecutionJobStatus.enum.success) succeededHosts.push(child.host);
|
|
46469
|
+
else if (child.status === ExecutionJobStatus.enum.failed) failedHosts.push(child.host);
|
|
46470
|
+
for (const [key, value] of Object.entries(child.parsed)) (outputs[key] ??= []).push(value);
|
|
46471
|
+
}
|
|
46472
|
+
return {
|
|
46473
|
+
byHost,
|
|
46474
|
+
summary: {
|
|
46475
|
+
succeededHosts,
|
|
46476
|
+
failedHosts,
|
|
46477
|
+
outputs
|
|
46478
|
+
}
|
|
46479
|
+
};
|
|
46480
|
+
}
|
|
46481
|
+
/**
|
|
45103
46482
|
* Group an upstream's child rows into the matrix outputs envelope
|
|
45104
46483
|
* `{ byMatrix: { '<suffix>': outputs }, merged: <last-write-wins> }`. The suffix
|
|
45105
46484
|
* is the text inside the `(...)` of each expanded child name; children are
|
|
@@ -45150,7 +46529,10 @@ async function mergeUpstreamOutputs(db, runId, jobName, needs, dispatchSecrets,
|
|
|
45150
46529
|
"job_id",
|
|
45151
46530
|
"job_name",
|
|
45152
46531
|
"outputs",
|
|
45153
|
-
"matrix_values"
|
|
46532
|
+
"matrix_values",
|
|
46533
|
+
"variant_kind",
|
|
46534
|
+
"variant_label",
|
|
46535
|
+
"status"
|
|
45154
46536
|
]).where("run_id", "=", runId);
|
|
45155
46537
|
query = query.where((eb) => eb.or(baseNames.flatMap((base) => [eb("job_name", "=", base), eb("job_name", "like", `${escapeLikePattern(base)} (%`)])));
|
|
45156
46538
|
const upstreamJobs = await query.execute();
|
|
@@ -45365,15 +46747,21 @@ function buildOnSecretOutputs(config, db) {
|
|
|
45365
46747
|
*/
|
|
45366
46748
|
async function routeInternalJobsViaCoordinator(coordinator, dispatcher, runId, workflow, staticJobs, ctx, buildInternalJobConfig) {
|
|
45367
46749
|
const dispatchedJobs = [];
|
|
45368
|
-
const jobsToRoute = staticJobs.map((job) =>
|
|
45369
|
-
|
|
45370
|
-
|
|
45371
|
-
|
|
45372
|
-
|
|
45373
|
-
|
|
45374
|
-
|
|
45375
|
-
|
|
45376
|
-
|
|
46750
|
+
const jobsToRoute = staticJobs.map((job) => {
|
|
46751
|
+
const sel = internalJobRunsOnSelectors(job);
|
|
46752
|
+
return {
|
|
46753
|
+
jobName: job.name,
|
|
46754
|
+
runsOnLabels: [sel.runsOnLabels],
|
|
46755
|
+
runsOnPatterns: sel.runsOnPatterns,
|
|
46756
|
+
excludeLabels: sel.excludeLabels,
|
|
46757
|
+
excludePatterns: sel.excludePatterns,
|
|
46758
|
+
jobConfig: buildInternalJobConfig(job),
|
|
46759
|
+
repoUrl: ctx.repoUrl,
|
|
46760
|
+
ref: "",
|
|
46761
|
+
sha: ctx.cronCommitSha,
|
|
46762
|
+
...job.resources && { resources: job.resources }
|
|
46763
|
+
};
|
|
46764
|
+
});
|
|
45377
46765
|
const runCtx = {
|
|
45378
46766
|
runId,
|
|
45379
46767
|
deliveryId: ctx.event.id,
|
|
@@ -45425,11 +46813,15 @@ async function dispatchInternalJobsDirect(dispatcher, runId, workflow, staticJob
|
|
|
45425
46813
|
const dispatchedJobs = [];
|
|
45426
46814
|
const buildJobConfig = (job) => buildInternalJobConfigForWorkflow(workflow, job);
|
|
45427
46815
|
for (const job of staticJobs) {
|
|
46816
|
+
const sel = internalJobRunsOnSelectors(job);
|
|
45428
46817
|
const result = await dispatcher.dispatch({
|
|
45429
46818
|
runId,
|
|
45430
46819
|
workflowName: workflow.name,
|
|
45431
46820
|
jobName: job.name,
|
|
45432
|
-
runsOnLabels:
|
|
46821
|
+
runsOnLabels: sel.runsOnLabels,
|
|
46822
|
+
runsOnPatterns: sel.runsOnPatterns,
|
|
46823
|
+
excludeLabels: sel.excludeLabels,
|
|
46824
|
+
excludePatterns: sel.excludePatterns,
|
|
45433
46825
|
jobConfig: buildJobConfig(job),
|
|
45434
46826
|
repoUrl: ctx.repoUrl,
|
|
45435
46827
|
ref: "",
|
|
@@ -45858,7 +47250,11 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
45858
47250
|
const tokenStore = new AgentTokenStore(db);
|
|
45859
47251
|
if (config.agentAuth === "none") logger$3.warn("Agent authentication disabled (KICI_AGENT_AUTH=none). All agents will be accepted without tokens.");
|
|
45860
47252
|
else logger$3.info("Agent authentication enabled (token mode)");
|
|
45861
|
-
const
|
|
47253
|
+
const hostRosterStore = new HostRosterStore(db);
|
|
47254
|
+
const agentRegistry = new AgentRegistry({
|
|
47255
|
+
rosterStore: hostRosterStore,
|
|
47256
|
+
instanceId: config.instanceId
|
|
47257
|
+
});
|
|
45862
47258
|
const fleetAgentCollector = new FleetAgentCollector({ timeoutMs: FLEET_NODE_TIMEOUT_MS });
|
|
45863
47259
|
const queue = new JobQueue(db, {
|
|
45864
47260
|
maxDepth: config.queueMaxDepth,
|
|
@@ -45918,6 +47314,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
45918
47314
|
error: toErrorMessage(err)
|
|
45919
47315
|
});
|
|
45920
47316
|
});
|
|
47317
|
+
clearEvalGatesForRun(runId);
|
|
45921
47318
|
const [owner, repo] = context.repoIdentifier.split("/");
|
|
45922
47319
|
checkRunReporter.updateWorkflowStatus({
|
|
45923
47320
|
provider: context.provider,
|
|
@@ -46005,6 +47402,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46005
47402
|
});
|
|
46006
47403
|
executionTrackerRef = executionTracker;
|
|
46007
47404
|
executionTracker.setOnJobReadyCallback(async (runId, jobName) => {
|
|
47405
|
+
if (openEvalGate(runId, jobName)) return;
|
|
46008
47406
|
await dispatchReadyJob(runId, jobName, dispatcher, executionTracker, cluster.coordinator, db);
|
|
46009
47407
|
});
|
|
46010
47408
|
logger$3.info("Execution reporting initialized", { logStorageType: config.storage?.type ?? "filesystem" });
|
|
@@ -46165,8 +47563,21 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46165
47563
|
eventStore,
|
|
46166
47564
|
config: eventRouterConfig
|
|
46167
47565
|
});
|
|
46168
|
-
|
|
46169
|
-
|
|
47566
|
+
const hostRosterReaper = new HostRosterReaper({
|
|
47567
|
+
store: hostRosterStore,
|
|
47568
|
+
ttlMs: config.rosterTtlMs,
|
|
47569
|
+
graceMs: config.rosterGraceMs,
|
|
47570
|
+
scanIntervalMs: 6e4,
|
|
47571
|
+
setUnreachableGauge: setDeclaredHostsUnreachable
|
|
47572
|
+
});
|
|
47573
|
+
eventRetryScannerRef.onBecomeLeader = () => {
|
|
47574
|
+
eventRetryScanner.onBecomeLeader();
|
|
47575
|
+
hostRosterReaper.onBecomeLeader();
|
|
47576
|
+
};
|
|
47577
|
+
eventRetryScannerRef.onLoseLeadership = () => {
|
|
47578
|
+
eventRetryScanner.onLoseLeadership();
|
|
47579
|
+
hostRosterReaper.onLoseLeadership();
|
|
47580
|
+
};
|
|
46170
47581
|
const cronStore = new CronStore(db);
|
|
46171
47582
|
const cronScheduler = new CronScheduler({
|
|
46172
47583
|
db,
|
|
@@ -46275,6 +47686,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46275
47686
|
pool,
|
|
46276
47687
|
providerRegistry,
|
|
46277
47688
|
agentRegistry,
|
|
47689
|
+
hostRosterStore,
|
|
46278
47690
|
dispatcher,
|
|
46279
47691
|
queue,
|
|
46280
47692
|
scalerManager,
|
|
@@ -46485,6 +47897,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46485
47897
|
db,
|
|
46486
47898
|
pool,
|
|
46487
47899
|
registry: agentRegistry,
|
|
47900
|
+
hostRosterStore,
|
|
46488
47901
|
dispatcher,
|
|
46489
47902
|
jobQueue: queue,
|
|
46490
47903
|
dedup,
|
|
@@ -46578,13 +47991,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46578
47991
|
scanIntervalMs: config.staleDetectorScanIntervalMs,
|
|
46579
47992
|
heldRunStore: modeResult.appDepsExtras?.heldRunStore,
|
|
46580
47993
|
stepApprovalBridge: modeResult.appDepsExtras?.stepApprovalBridge,
|
|
46581
|
-
failRun: (runId, reason) =>
|
|
46582
|
-
db,
|
|
46583
|
-
jobQueue: queue,
|
|
46584
|
-
dispatcher,
|
|
46585
|
-
registry: agentRegistry,
|
|
46586
|
-
executionTracker
|
|
46587
|
-
}, runId, reason).then(() => void 0),
|
|
47994
|
+
failRun: (runId, reason) => executionTracker.failRun(runId, reason).then(() => void 0),
|
|
46588
47995
|
onWorkflowRelease: modeResult.appDepsExtras?.onWorkflowRelease,
|
|
46589
47996
|
accessLogWriter
|
|
46590
47997
|
});
|
|
@@ -46755,6 +48162,10 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
46755
48162
|
name: "Stopping event retry scanner",
|
|
46756
48163
|
fn: () => eventRetryScanner.stop()
|
|
46757
48164
|
},
|
|
48165
|
+
{
|
|
48166
|
+
name: "Stopping host roster reaper",
|
|
48167
|
+
fn: () => hostRosterReaper.stop()
|
|
48168
|
+
},
|
|
46758
48169
|
{
|
|
46759
48170
|
name: "Stopping timers, cleanup, and reloader",
|
|
46760
48171
|
fn: () => {
|
|
@@ -46794,6 +48205,8 @@ var init_orchestrator_core = __esmMin((() => {
|
|
|
46794
48205
|
init_resolver();
|
|
46795
48206
|
init_client();
|
|
46796
48207
|
init_registry();
|
|
48208
|
+
init_host_roster();
|
|
48209
|
+
init_host_roster_reaper();
|
|
46797
48210
|
init_job_queue();
|
|
46798
48211
|
init_cleanup$1();
|
|
46799
48212
|
init_bootstrap();
|
|
@@ -47003,6 +48416,9 @@ function matchesGate(job, agentLabels, agentMandatoryLabels) {
|
|
|
47003
48416
|
const runsOnSet = new Set(job.runsOnLabels);
|
|
47004
48417
|
if (!agentMandatoryLabels.every((m) => runsOnSet.has(m))) return false;
|
|
47005
48418
|
}
|
|
48419
|
+
const labelSet = new Set(agentLabels);
|
|
48420
|
+
if (!job.runsOnPatterns.every((p) => matcherSatisfiedBy(p, labelSet))) return false;
|
|
48421
|
+
if (job.excludePatterns.some((p) => matcherSatisfiedBy(p, labelSet))) return false;
|
|
47006
48422
|
return true;
|
|
47007
48423
|
}
|
|
47008
48424
|
var InMemoryJobQueue;
|
|
@@ -47039,6 +48455,8 @@ var init_in_memory_job_queue = __esmMin((() => {
|
|
|
47039
48455
|
depsHash: input.depsHash,
|
|
47040
48456
|
requestId: input.requestId,
|
|
47041
48457
|
excludeLabels: input.excludeLabels ?? [],
|
|
48458
|
+
runsOnPatterns: input.runsOnPatterns ?? [],
|
|
48459
|
+
excludePatterns: input.excludePatterns ?? [],
|
|
47042
48460
|
routingKey: input.routingKey
|
|
47043
48461
|
});
|
|
47044
48462
|
return id;
|
|
@@ -47786,6 +49204,8 @@ async function bootstrapWorker(config, _opts) {
|
|
|
47786
49204
|
jobName: msg.jobName,
|
|
47787
49205
|
runsOnLabels: flatLabels,
|
|
47788
49206
|
excludeLabels: msg.excludeLabels,
|
|
49207
|
+
runsOnPatterns: msg.runsOnPatterns,
|
|
49208
|
+
excludePatterns: msg.excludePatterns,
|
|
47789
49209
|
jobConfig,
|
|
47790
49210
|
repoUrl: msg.repoUrl ?? "",
|
|
47791
49211
|
ref: msg.ref ?? "",
|
|
@@ -48079,14 +49499,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
48079
49499
|
init_fleet_wiring();
|
|
48080
49500
|
init_worker_status();
|
|
48081
49501
|
init_agent_heartbeat();
|
|
48082
|
-
ORCHESTRATOR_VERSION$1 = "0.1.
|
|
48083
|
-
WORKER_BUILD_COMMIT = "
|
|
48084
|
-
WORKER_SDK_VERSION = "0.1.
|
|
48085
|
-
WORKER_SDK_BUNDLE_HASH = "
|
|
48086
|
-
WORKER_SHARED_VERSION = "0.1.
|
|
48087
|
-
WORKER_SHARED_BUNDLE_HASH = "
|
|
48088
|
-
WORKER_ENGINE_VERSION = "0.1.
|
|
48089
|
-
WORKER_ENGINE_BUNDLE_HASH = "
|
|
49502
|
+
ORCHESTRATOR_VERSION$1 = "0.1.18";
|
|
49503
|
+
WORKER_BUILD_COMMIT = "d8cff38bb";
|
|
49504
|
+
WORKER_SDK_VERSION = "0.1.18";
|
|
49505
|
+
WORKER_SDK_BUNDLE_HASH = "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f";
|
|
49506
|
+
WORKER_SHARED_VERSION = "0.1.18";
|
|
49507
|
+
WORKER_SHARED_BUNDLE_HASH = "5f2c220f24d166b0f13d0620a2e80d19689ac8b683a12154daef44e938fd46a7";
|
|
49508
|
+
WORKER_ENGINE_VERSION = "0.1.18";
|
|
49509
|
+
WORKER_ENGINE_BUNDLE_HASH = "79ce14640d1798eaaa7cb3aa6c4bc325da3bff1e9f4716936e19614eabb1d858";
|
|
48090
49510
|
logger$2 = createLogger({ prefix: "worker" });
|
|
48091
49511
|
DRAIN_TIMEOUT_MS = 3e5;
|
|
48092
49512
|
}));
|
|
@@ -48110,14 +49530,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
48110
49530
|
* Graceful shutdown:
|
|
48111
49531
|
* agent WS -> heartbeat -> HTTP -> DB
|
|
48112
49532
|
*/
|
|
48113
|
-
const ORCHESTRATOR_VERSION = "0.1.
|
|
48114
|
-
const BUILD_COMMIT = "
|
|
48115
|
-
const SDK_VERSION = "0.1.
|
|
48116
|
-
const SDK_BUNDLE_HASH = "
|
|
48117
|
-
const SHARED_VERSION = "0.1.
|
|
48118
|
-
const SHARED_BUNDLE_HASH = "
|
|
48119
|
-
const ENGINE_VERSION = "0.1.
|
|
48120
|
-
const ENGINE_BUNDLE_HASH = "
|
|
49533
|
+
const ORCHESTRATOR_VERSION = "0.1.18";
|
|
49534
|
+
const BUILD_COMMIT = "d8cff38bb";
|
|
49535
|
+
const SDK_VERSION = "0.1.18";
|
|
49536
|
+
const SDK_BUNDLE_HASH = "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f";
|
|
49537
|
+
const SHARED_VERSION = "0.1.18";
|
|
49538
|
+
const SHARED_BUNDLE_HASH = "5f2c220f24d166b0f13d0620a2e80d19689ac8b683a12154daef44e938fd46a7";
|
|
49539
|
+
const ENGINE_VERSION = "0.1.18";
|
|
49540
|
+
const ENGINE_BUNDLE_HASH = "79ce14640d1798eaaa7cb3aa6c4bc325da3bff1e9f4716936e19614eabb1d858";
|
|
48121
49541
|
const otelSdk = initTelemetry({
|
|
48122
49542
|
serviceName: "kici-orchestrator",
|
|
48123
49543
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|