@kici-dev/orchestrator 0.1.13 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/service/index.d.ts +1 -1
- package/dist/cli/service/instance/manifest.d.ts +9 -0
- package/dist/cli.js +179 -103
- package/dist/db/migrations/025_init_failure.d.ts +16 -0
- package/dist/db/types.d.ts +15 -0
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/checks/scaler.d.ts +13 -0
- package/dist/diagnostics/types.d.ts +5 -2
- package/dist/reporting/execution-tracker.d.ts +77 -6
- package/dist/scaler/failure-tracker.d.ts +46 -0
- package/dist/scaler/manager.d.ts +8 -0
- package/dist/server.js +570 -168
- package/dist/standalone.js +547 -156
- package/package.json +4 -4
- package/sbom.spdx.json +91 -36
package/dist/standalone.js
CHANGED
|
@@ -9,7 +9,7 @@ import { z } from "zod";
|
|
|
9
9
|
import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
|
|
10
10
|
import WebSocket from "ws";
|
|
11
11
|
import { chmodSync, closeSync, createReadStream, createWriteStream, mkdirSync, mkdtempSync, openSync, promises, readFileSync, rmSync, statSync, unlinkSync, unwatchFile, watchFile, writeFileSync } from "node:fs";
|
|
12
|
-
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, testEventSchema } from "@kici-dev/engine";
|
|
12
|
+
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, testEventSchema } from "@kici-dev/engine";
|
|
13
13
|
import { access, appendFile, chmod, constants, copyFile, link, mkdir, mkdtemp, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
14
14
|
import { dirname, join, relative, resolve } from "node:path";
|
|
15
15
|
import { sql } from "kysely";
|
|
@@ -47,6 +47,7 @@ import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCom
|
|
|
47
47
|
import { Upload } from "@aws-sdk/lib-storage";
|
|
48
48
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
49
49
|
import stripAnsi from "strip-ansi";
|
|
50
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
50
51
|
import Vault from "hashi-vault-js";
|
|
51
52
|
import { Cron } from "croner";
|
|
52
53
|
var __defProp = Object.defineProperty;
|
|
@@ -1040,7 +1041,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
1040
1041
|
init_peer_crypto();
|
|
1041
1042
|
init_peer_credentials();
|
|
1042
1043
|
logger$86 = createLogger({ prefix: "peer-client" });
|
|
1043
|
-
SOFTWARE_VERSION$1 = "0.1.
|
|
1044
|
+
SOFTWARE_VERSION$1 = "0.1.14";
|
|
1044
1045
|
PeerClient$1 = class {
|
|
1045
1046
|
ws = null;
|
|
1046
1047
|
_state = "disconnected";
|
|
@@ -2506,7 +2507,7 @@ var init_peer_handler = __esmMin((() => {
|
|
|
2506
2507
|
init_peer_crypto();
|
|
2507
2508
|
init_join_token();
|
|
2508
2509
|
logger$85 = createLogger({ prefix: "peer-handler" });
|
|
2509
|
-
SOFTWARE_VERSION = "0.1.
|
|
2510
|
+
SOFTWARE_VERSION = "0.1.14";
|
|
2510
2511
|
RATE_LIMIT_MAX = 5;
|
|
2511
2512
|
RATE_LIMIT_WINDOW_MS = 6e4;
|
|
2512
2513
|
}));
|
|
@@ -13608,10 +13609,10 @@ var init_admin_sources = __esmMin((() => {
|
|
|
13608
13609
|
//#endregion
|
|
13609
13610
|
//#region src/db/migrations/001_initial.ts
|
|
13610
13611
|
var _001_initial_exports = /* @__PURE__ */ __exportAll({
|
|
13611
|
-
down: () => down$
|
|
13612
|
-
up: () => up$
|
|
13612
|
+
down: () => down$24,
|
|
13613
|
+
up: () => up$24
|
|
13613
13614
|
});
|
|
13614
|
-
async function up$
|
|
13615
|
+
async function up$24(db) {
|
|
13615
13616
|
for (const stmt of DDL_STATEMENTS) await sql.raw(stmt).execute(db);
|
|
13616
13617
|
await sql`
|
|
13617
13618
|
INSERT INTO cluster_meta (key, value)
|
|
@@ -13633,7 +13634,7 @@ async function up$23(db) {
|
|
|
13633
13634
|
* Rollback drops everything created above. Uses CASCADE on table drops to cut
|
|
13634
13635
|
* through the FK graph without relying on exact topological order.
|
|
13635
13636
|
*/
|
|
13636
|
-
async function down$
|
|
13637
|
+
async function down$24(db) {
|
|
13637
13638
|
for (const [trig, tbl] of [["source_secrets_change_trigger", "scoped_secrets"], ["sources_change_trigger", "sources"]]) await sql.raw(`DROP TRIGGER IF EXISTS ${trig} ON public.${tbl}`).execute(db);
|
|
13638
13639
|
for (const table of [
|
|
13639
13640
|
"workflow_registrations",
|
|
@@ -14353,8 +14354,8 @@ var init__001_initial = __esmMin((() => {
|
|
|
14353
14354
|
//#endregion
|
|
14354
14355
|
//#region src/db/migrations/002_config_versions_key_version.ts
|
|
14355
14356
|
var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
14356
|
-
down: () => down$
|
|
14357
|
-
up: () => up$
|
|
14357
|
+
down: () => down$23,
|
|
14358
|
+
up: () => up$23
|
|
14358
14359
|
});
|
|
14359
14360
|
/**
|
|
14360
14361
|
* Add key_version column to config_versions so that sensitive-field encryption
|
|
@@ -14368,13 +14369,13 @@ var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
|
14368
14369
|
* No index is needed: rotation does a full-table scan; reads are by
|
|
14369
14370
|
* `version` primary key and never filter on `key_version`.
|
|
14370
14371
|
*/
|
|
14371
|
-
async function up$
|
|
14372
|
+
async function up$23(db) {
|
|
14372
14373
|
await sql`
|
|
14373
14374
|
ALTER TABLE public.config_versions
|
|
14374
14375
|
ADD COLUMN key_version integer NOT NULL DEFAULT 1
|
|
14375
14376
|
`.execute(db);
|
|
14376
14377
|
}
|
|
14377
|
-
async function down$
|
|
14378
|
+
async function down$23(db) {
|
|
14378
14379
|
await sql`
|
|
14379
14380
|
ALTER TABLE public.config_versions
|
|
14380
14381
|
DROP COLUMN key_version
|
|
@@ -14384,8 +14385,8 @@ var init__002_config_versions_key_version = __esmMin((() => {}));
|
|
|
14384
14385
|
//#endregion
|
|
14385
14386
|
//#region src/db/migrations/003_access_log.ts
|
|
14386
14387
|
var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
14387
|
-
down: () => down$
|
|
14388
|
-
up: () => up$
|
|
14388
|
+
down: () => down$22,
|
|
14389
|
+
up: () => up$22
|
|
14389
14390
|
});
|
|
14390
14391
|
/**
|
|
14391
14392
|
* Access log: one row per read or orchestrator-admin mutation attributable
|
|
@@ -14409,7 +14410,7 @@ var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
|
14409
14410
|
* Retention is TTL-based via expires_at; packages/orchestrator/src/queue/
|
|
14410
14411
|
* cleanup.ts picks up the prune pass.
|
|
14411
14412
|
*/
|
|
14412
|
-
async function up$
|
|
14413
|
+
async function up$22(db) {
|
|
14413
14414
|
await sql`
|
|
14414
14415
|
CREATE TABLE public.access_log (
|
|
14415
14416
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
@@ -14447,21 +14448,21 @@ async function up$21(db) {
|
|
|
14447
14448
|
ON public.access_log (actor_type, actor_id, created_at DESC)
|
|
14448
14449
|
`.execute(db);
|
|
14449
14450
|
}
|
|
14450
|
-
async function down$
|
|
14451
|
+
async function down$22(db) {
|
|
14451
14452
|
await sql`DROP TABLE IF EXISTS public.access_log`.execute(db);
|
|
14452
14453
|
}
|
|
14453
14454
|
var init__003_access_log = __esmMin((() => {}));
|
|
14454
14455
|
//#endregion
|
|
14455
14456
|
//#region src/db/migrations/004_rename_bundle_to_source.ts
|
|
14456
14457
|
var _004_rename_bundle_to_source_exports = /* @__PURE__ */ __exportAll({
|
|
14457
|
-
down: () => down$
|
|
14458
|
-
up: () => up$
|
|
14458
|
+
down: () => down$21,
|
|
14459
|
+
up: () => up$21
|
|
14459
14460
|
});
|
|
14460
|
-
async function up$
|
|
14461
|
+
async function up$21(db) {
|
|
14461
14462
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_url", "source_tar_url").execute();
|
|
14462
14463
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_hash", "source_tar_hash").execute();
|
|
14463
14464
|
}
|
|
14464
|
-
async function down$
|
|
14465
|
+
async function down$21(db) {
|
|
14465
14466
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_url", "bundle_url").execute();
|
|
14466
14467
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_hash", "bundle_hash").execute();
|
|
14467
14468
|
}
|
|
@@ -14469,8 +14470,8 @@ var init__004_rename_bundle_to_source = __esmMin((() => {}));
|
|
|
14469
14470
|
//#endregion
|
|
14470
14471
|
//#region src/db/migrations/005_cold_store_chunk_counter.ts
|
|
14471
14472
|
var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
14472
|
-
down: () => down$
|
|
14473
|
-
up: () => up$
|
|
14473
|
+
down: () => down$20,
|
|
14474
|
+
up: () => up$20
|
|
14474
14475
|
});
|
|
14475
14476
|
/**
|
|
14476
14477
|
* Cold-store chunk counter table.
|
|
@@ -14487,7 +14488,7 @@ var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
|
14487
14488
|
*
|
|
14488
14489
|
* sections 5 and 8.
|
|
14489
14490
|
*/
|
|
14490
|
-
async function up$
|
|
14491
|
+
async function up$20(db) {
|
|
14491
14492
|
await sql`
|
|
14492
14493
|
CREATE TABLE public.cold_store_chunk_counts (
|
|
14493
14494
|
db TEXT NOT NULL,
|
|
@@ -14505,15 +14506,15 @@ async function up$19(db) {
|
|
|
14505
14506
|
ON public.cold_store_chunk_counts (db, table_name)
|
|
14506
14507
|
`.execute(db);
|
|
14507
14508
|
}
|
|
14508
|
-
async function down$
|
|
14509
|
+
async function down$20(db) {
|
|
14509
14510
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunk_counts`.execute(db);
|
|
14510
14511
|
}
|
|
14511
14512
|
var init__005_cold_store_chunk_counter = __esmMin((() => {}));
|
|
14512
14513
|
//#endregion
|
|
14513
14514
|
//#region src/db/migrations/006_runs_jobs_steps_archived_at.ts
|
|
14514
14515
|
var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14515
|
-
down: () => down$
|
|
14516
|
-
up: () => up$
|
|
14516
|
+
down: () => down$19,
|
|
14517
|
+
up: () => up$19
|
|
14517
14518
|
});
|
|
14518
14519
|
/**
|
|
14519
14520
|
* `execution_runs` / `execution_jobs` / `execution_steps` cold-store
|
|
@@ -14549,7 +14550,7 @@ var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
14549
14550
|
* - `idx_execution_jobs_routing_key_created (routing_key, created_at)`
|
|
14550
14551
|
* - `idx_execution_steps_routing_key_created (routing_key, created_at)`
|
|
14551
14552
|
*/
|
|
14552
|
-
async function up$
|
|
14553
|
+
async function up$19(db) {
|
|
14553
14554
|
await sql`
|
|
14554
14555
|
ALTER TABLE public.execution_runs
|
|
14555
14556
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -14594,7 +14595,7 @@ async function up$18(db) {
|
|
|
14594
14595
|
ON public.execution_steps (routing_key, created_at)
|
|
14595
14596
|
`.execute(db);
|
|
14596
14597
|
}
|
|
14597
|
-
async function down$
|
|
14598
|
+
async function down$19(db) {
|
|
14598
14599
|
await sql`DROP INDEX IF EXISTS public.idx_execution_steps_routing_key_created`.execute(db);
|
|
14599
14600
|
await sql`
|
|
14600
14601
|
ALTER TABLE public.execution_steps
|
|
@@ -14620,8 +14621,8 @@ var init__006_runs_jobs_steps_archived_at = __esmMin((() => {}));
|
|
|
14620
14621
|
//#endregion
|
|
14621
14622
|
//#region src/db/migrations/007_audit_logs_archived_at.ts
|
|
14622
14623
|
var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14623
|
-
down: () => down$
|
|
14624
|
-
up: () => up$
|
|
14624
|
+
down: () => down$18,
|
|
14625
|
+
up: () => up$18
|
|
14625
14626
|
});
|
|
14626
14627
|
/**
|
|
14627
14628
|
* `secret_audit_log` and `access_log` cold-store schema additions, plus
|
|
@@ -14661,7 +14662,7 @@ var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
14661
14662
|
* `down()` would not have meaningful retention bounds. Acceptable for
|
|
14662
14663
|
* staging.
|
|
14663
14664
|
*/
|
|
14664
|
-
async function up$
|
|
14665
|
+
async function up$18(db) {
|
|
14665
14666
|
await sql`
|
|
14666
14667
|
ALTER TABLE public.secret_audit_log
|
|
14667
14668
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -14682,7 +14683,7 @@ async function up$17(db) {
|
|
|
14682
14683
|
DROP COLUMN IF EXISTS expires_at
|
|
14683
14684
|
`.execute(db);
|
|
14684
14685
|
}
|
|
14685
|
-
async function down$
|
|
14686
|
+
async function down$18(db) {
|
|
14686
14687
|
await sql`
|
|
14687
14688
|
ALTER TABLE public.access_log
|
|
14688
14689
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '90 days')
|
|
@@ -14711,8 +14712,8 @@ var init__007_audit_logs_archived_at = __esmMin((() => {}));
|
|
|
14711
14712
|
//#endregion
|
|
14712
14713
|
//#region src/db/migrations/008_event_log_archived_at.ts
|
|
14713
14714
|
var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
14714
|
-
down: () => down$
|
|
14715
|
-
up: () => up$
|
|
14715
|
+
down: () => down$17,
|
|
14716
|
+
up: () => up$17
|
|
14716
14717
|
});
|
|
14717
14718
|
/**
|
|
14718
14719
|
* `event_log` cold-store schema additions plus removal of the
|
|
@@ -14750,7 +14751,7 @@ var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
14750
14751
|
* — best effort; rows inserted between `up()` and a hypothetical
|
|
14751
14752
|
* `down()` would not have meaningful retention bounds.
|
|
14752
14753
|
*/
|
|
14753
|
-
async function up$
|
|
14754
|
+
async function up$17(db) {
|
|
14754
14755
|
await sql`
|
|
14755
14756
|
ALTER TABLE public.event_log
|
|
14756
14757
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -14766,7 +14767,7 @@ async function up$16(db) {
|
|
|
14766
14767
|
DROP COLUMN IF EXISTS expires_at
|
|
14767
14768
|
`.execute(db);
|
|
14768
14769
|
}
|
|
14769
|
-
async function down$
|
|
14770
|
+
async function down$17(db) {
|
|
14770
14771
|
await sql`
|
|
14771
14772
|
ALTER TABLE public.event_log
|
|
14772
14773
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '30 days')
|
|
@@ -14786,8 +14787,8 @@ var init__008_event_log_archived_at = __esmMin((() => {}));
|
|
|
14786
14787
|
//#endregion
|
|
14787
14788
|
//#region src/db/migrations/009_access_log_trigram.ts
|
|
14788
14789
|
var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
14789
|
-
down: () => down$
|
|
14790
|
-
up: () => up$
|
|
14790
|
+
down: () => down$16,
|
|
14791
|
+
up: () => up$16
|
|
14791
14792
|
});
|
|
14792
14793
|
/**
|
|
14793
14794
|
* Trigram (pg_trgm) index on access_log.error_message for the federated
|
|
@@ -14800,7 +14801,7 @@ var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
|
14800
14801
|
* EXISTS` are both safe to re-run. No CONCURRENTLY because Kysely runs
|
|
14801
14802
|
* migrations inside a transaction; the lock is brief on a sampled table.
|
|
14802
14803
|
*/
|
|
14803
|
-
async function up$
|
|
14804
|
+
async function up$16(db) {
|
|
14804
14805
|
await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
|
|
14805
14806
|
await sql`
|
|
14806
14807
|
CREATE INDEX IF NOT EXISTS access_log_error_message_trgm_idx
|
|
@@ -14809,15 +14810,15 @@ async function up$15(db) {
|
|
|
14809
14810
|
WHERE error_message IS NOT NULL
|
|
14810
14811
|
`.execute(db);
|
|
14811
14812
|
}
|
|
14812
|
-
async function down$
|
|
14813
|
+
async function down$16(db) {
|
|
14813
14814
|
await sql`DROP INDEX IF EXISTS public.access_log_error_message_trgm_idx`.execute(db);
|
|
14814
14815
|
}
|
|
14815
14816
|
var init__009_access_log_trigram = __esmMin((() => {}));
|
|
14816
14817
|
//#endregion
|
|
14817
14818
|
//#region src/db/migrations/010_cold_store_chunks.ts
|
|
14818
14819
|
var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
14819
|
-
down: () => down$
|
|
14820
|
-
up: () => up$
|
|
14820
|
+
down: () => down$15,
|
|
14821
|
+
up: () => up$15
|
|
14821
14822
|
});
|
|
14822
14823
|
/**
|
|
14823
14824
|
* Cold-store chunk index — Phase 2 (cold-store purge).
|
|
@@ -14849,7 +14850,7 @@ var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
|
14849
14850
|
* forever. Adapters that don't opt into per-bucket archival via
|
|
14850
14851
|
* `coldTtlDays` don't insert here either.
|
|
14851
14852
|
*/
|
|
14852
|
-
async function up$
|
|
14853
|
+
async function up$15(db) {
|
|
14853
14854
|
await sql`
|
|
14854
14855
|
CREATE TABLE public.cold_store_chunks (
|
|
14855
14856
|
db TEXT NOT NULL,
|
|
@@ -14876,15 +14877,15 @@ async function up$14(db) {
|
|
|
14876
14877
|
ON public.cold_store_chunks (db, table_name, tenant_id, archived_at DESC)
|
|
14877
14878
|
`.execute(db);
|
|
14878
14879
|
}
|
|
14879
|
-
async function down$
|
|
14880
|
+
async function down$15(db) {
|
|
14880
14881
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunks`.execute(db);
|
|
14881
14882
|
}
|
|
14882
14883
|
var init__010_cold_store_chunks = __esmMin((() => {}));
|
|
14883
14884
|
//#endregion
|
|
14884
14885
|
//#region src/db/migrations/011_drop_source_secrets_notify.ts
|
|
14885
14886
|
var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
14886
|
-
down: () => down$
|
|
14887
|
-
up: () => up$
|
|
14887
|
+
down: () => down$14,
|
|
14888
|
+
up: () => up$14
|
|
14888
14889
|
});
|
|
14889
14890
|
/**
|
|
14890
14891
|
* Drop the `source_secrets_change_trigger` and the
|
|
@@ -14903,11 +14904,11 @@ var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
14903
14904
|
* in `001_initial.ts`. They wake up no consumer until the `WebhookSecretManager`
|
|
14904
14905
|
* is restored, so this migration is safe to roll back.
|
|
14905
14906
|
*/
|
|
14906
|
-
async function up$
|
|
14907
|
+
async function up$14(db) {
|
|
14907
14908
|
await sql`DROP TRIGGER IF EXISTS source_secrets_change_trigger ON public.scoped_secrets`.execute(db);
|
|
14908
14909
|
await sql`DROP FUNCTION IF EXISTS public.notify_source_secrets_change() CASCADE`.execute(db);
|
|
14909
14910
|
}
|
|
14910
|
-
async function down$
|
|
14911
|
+
async function down$14(db) {
|
|
14911
14912
|
await sql`
|
|
14912
14913
|
CREATE OR REPLACE FUNCTION public.notify_source_secrets_change() RETURNS trigger
|
|
14913
14914
|
LANGUAGE plpgsql
|
|
@@ -14947,8 +14948,8 @@ var init__011_drop_source_secrets_notify = __esmMin((() => {}));
|
|
|
14947
14948
|
//#endregion
|
|
14948
14949
|
//#region src/db/migrations/012_peer_credentials_active_uniq.ts
|
|
14949
14950
|
var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
14950
|
-
down: () => down$
|
|
14951
|
-
up: () => up$
|
|
14951
|
+
down: () => down$13,
|
|
14952
|
+
up: () => up$13
|
|
14952
14953
|
});
|
|
14953
14954
|
/**
|
|
14954
14955
|
* Add a partial unique index on `peer_credentials (instance_id) WHERE
|
|
@@ -14974,7 +14975,7 @@ var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
|
14974
14975
|
* `down()` only drops the index; it does NOT undo the dedupe (there's no
|
|
14975
14976
|
* safe way to recreate revoked rows, and the dedupe is monotonic).
|
|
14976
14977
|
*/
|
|
14977
|
-
async function up$
|
|
14978
|
+
async function up$13(db) {
|
|
14978
14979
|
await sql`
|
|
14979
14980
|
UPDATE public.peer_credentials
|
|
14980
14981
|
SET revoked_at = NOW()
|
|
@@ -14992,15 +14993,15 @@ async function up$12(db) {
|
|
|
14992
14993
|
WHERE revoked_at IS NULL
|
|
14993
14994
|
`.execute(db);
|
|
14994
14995
|
}
|
|
14995
|
-
async function down$
|
|
14996
|
+
async function down$13(db) {
|
|
14996
14997
|
await sql`DROP INDEX IF EXISTS public.peer_credentials_active_uniq`.execute(db);
|
|
14997
14998
|
}
|
|
14998
14999
|
var init__012_peer_credentials_active_uniq = __esmMin((() => {}));
|
|
14999
15000
|
//#endregion
|
|
15000
15001
|
//#region src/db/migrations/013_execution_log_bytes.ts
|
|
15001
15002
|
var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
15002
|
-
down: () => down$
|
|
15003
|
-
up: () => up$
|
|
15003
|
+
down: () => down$12,
|
|
15004
|
+
up: () => up$12
|
|
15004
15005
|
});
|
|
15005
15006
|
/**
|
|
15006
15007
|
* Add `log_bytes BIGINT NOT NULL DEFAULT 0` columns to `execution_runs` and
|
|
@@ -15019,7 +15020,7 @@ var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
|
15019
15020
|
*
|
|
15020
15021
|
* Idempotent (`ADD COLUMN IF NOT EXISTS`).
|
|
15021
15022
|
*/
|
|
15022
|
-
async function up$
|
|
15023
|
+
async function up$12(db) {
|
|
15023
15024
|
await sql`
|
|
15024
15025
|
ALTER TABLE public.execution_runs
|
|
15025
15026
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
@@ -15029,7 +15030,7 @@ async function up$11(db) {
|
|
|
15029
15030
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
15030
15031
|
`.execute(db);
|
|
15031
15032
|
}
|
|
15032
|
-
async function down$
|
|
15033
|
+
async function down$12(db) {
|
|
15033
15034
|
await sql`ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
15034
15035
|
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
15035
15036
|
}
|
|
@@ -15037,8 +15038,8 @@ var init__013_execution_log_bytes = __esmMin((() => {}));
|
|
|
15037
15038
|
//#endregion
|
|
15038
15039
|
//#region src/db/migrations/014_kici_events_lease_retry.ts
|
|
15039
15040
|
var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
15040
|
-
down: () => down$
|
|
15041
|
-
up: () => up$
|
|
15041
|
+
down: () => down$11,
|
|
15042
|
+
up: () => up$11
|
|
15042
15043
|
});
|
|
15043
15044
|
/**
|
|
15044
15045
|
* Add lease + retry + DLQ columns to `kici_events` so the EventRouter can
|
|
@@ -15072,7 +15073,7 @@ var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
|
15072
15073
|
*
|
|
15073
15074
|
* Idempotent (`ADD COLUMN IF NOT EXISTS` + `CREATE INDEX IF NOT EXISTS`).
|
|
15074
15075
|
*/
|
|
15075
|
-
async function up$
|
|
15076
|
+
async function up$11(db) {
|
|
15076
15077
|
await sql`
|
|
15077
15078
|
ALTER TABLE public.kici_events
|
|
15078
15079
|
ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ,
|
|
@@ -15103,7 +15104,7 @@ async function up$10(db) {
|
|
|
15103
15104
|
WHERE dlq_at IS NOT NULL
|
|
15104
15105
|
`.execute(db);
|
|
15105
15106
|
}
|
|
15106
|
-
async function down$
|
|
15107
|
+
async function down$11(db) {
|
|
15107
15108
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_dlq`.execute(db);
|
|
15108
15109
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_lease_expired`.execute(db);
|
|
15109
15110
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_retry_due`.execute(db);
|
|
@@ -15122,8 +15123,8 @@ var init__014_kici_events_lease_retry = __esmMin((() => {}));
|
|
|
15122
15123
|
//#endregion
|
|
15123
15124
|
//#region src/db/migrations/015_org_settings_customer_scoped.ts
|
|
15124
15125
|
var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
15125
|
-
down: () => down$
|
|
15126
|
-
up: () => up$
|
|
15126
|
+
down: () => down$10,
|
|
15127
|
+
up: () => up$10
|
|
15127
15128
|
});
|
|
15128
15129
|
/**
|
|
15129
15130
|
* Org-scope `org_settings` and qualify each glob entry by source.
|
|
@@ -15151,7 +15152,7 @@ var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
|
15151
15152
|
* Idempotent: a re-run on an already-migrated DB sees `customer_id` exists
|
|
15152
15153
|
* and the list columns are already jsonb, so it is a no-op.
|
|
15153
15154
|
*/
|
|
15154
|
-
async function up$
|
|
15155
|
+
async function up$10(db) {
|
|
15155
15156
|
if ((await sql`
|
|
15156
15157
|
SELECT EXISTS (
|
|
15157
15158
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15276,7 +15277,7 @@ async function up$9(db) {
|
|
|
15276
15277
|
await sql`DROP TABLE _org_settings_merged`.execute(db);
|
|
15277
15278
|
await sql`DROP TABLE _org_settings_stage`.execute(db);
|
|
15278
15279
|
}
|
|
15279
|
-
async function down$
|
|
15280
|
+
async function down$10(db) {
|
|
15280
15281
|
if (!(await sql`
|
|
15281
15282
|
SELECT EXISTS (
|
|
15282
15283
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15302,8 +15303,8 @@ var init__015_org_settings_customer_scoped = __esmMin((() => {}));
|
|
|
15302
15303
|
//#endregion
|
|
15303
15304
|
//#region src/db/migrations/016_org_settings_allow_http_npm.ts
|
|
15304
15305
|
var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
15305
|
-
down: () => down$
|
|
15306
|
-
up: () => up$
|
|
15306
|
+
down: () => down$9,
|
|
15307
|
+
up: () => up$9
|
|
15307
15308
|
});
|
|
15308
15309
|
/**
|
|
15309
15310
|
* Add `org_settings.allow_http_npm_registries boolean NOT NULL DEFAULT false`.
|
|
@@ -15316,7 +15317,7 @@ var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
|
15316
15317
|
*
|
|
15317
15318
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
15318
15319
|
*/
|
|
15319
|
-
async function up$
|
|
15320
|
+
async function up$9(db) {
|
|
15320
15321
|
if ((await sql`
|
|
15321
15322
|
SELECT EXISTS (
|
|
15322
15323
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15330,7 +15331,7 @@ async function up$8(db) {
|
|
|
15330
15331
|
ADD COLUMN allow_http_npm_registries boolean NOT NULL DEFAULT false
|
|
15331
15332
|
`.execute(db);
|
|
15332
15333
|
}
|
|
15333
|
-
async function down$
|
|
15334
|
+
async function down$9(db) {
|
|
15334
15335
|
await sql`
|
|
15335
15336
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_http_npm_registries
|
|
15336
15337
|
`.execute(db);
|
|
@@ -15339,13 +15340,13 @@ var init__016_org_settings_allow_http_npm = __esmMin((() => {}));
|
|
|
15339
15340
|
//#endregion
|
|
15340
15341
|
//#region src/db/migrations/017_org_id_widen.ts
|
|
15341
15342
|
var _017_org_id_widen_exports = /* @__PURE__ */ __exportAll({
|
|
15342
|
-
down: () => down$
|
|
15343
|
-
up: () => up$
|
|
15343
|
+
down: () => down$8,
|
|
15344
|
+
up: () => up$8
|
|
15344
15345
|
});
|
|
15345
|
-
async function up$
|
|
15346
|
+
async function up$8(db) {
|
|
15346
15347
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(16)`).execute(db);
|
|
15347
15348
|
}
|
|
15348
|
-
async function down$
|
|
15349
|
+
async function down$8(db) {
|
|
15349
15350
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(12)`).execute(db);
|
|
15350
15351
|
}
|
|
15351
15352
|
var ORG_ID_TABLES$1;
|
|
@@ -15364,14 +15365,14 @@ var init__017_org_id_widen = __esmMin((() => {
|
|
|
15364
15365
|
//#endregion
|
|
15365
15366
|
//#region src/db/migrations/018_org_id_prefix_backfill.ts
|
|
15366
15367
|
var _018_org_id_prefix_backfill_exports = /* @__PURE__ */ __exportAll({
|
|
15367
|
-
down: () => down$
|
|
15368
|
-
up: () => up$
|
|
15368
|
+
down: () => down$7,
|
|
15369
|
+
up: () => up$7
|
|
15369
15370
|
});
|
|
15370
|
-
async function up$
|
|
15371
|
+
async function up$7(db) {
|
|
15371
15372
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = 'org_' || org_id WHERE org_id <> 'kici-admin' AND org_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15372
15373
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = 'org_' || customer_id WHERE customer_id <> 'kici-admin' AND customer_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15373
15374
|
}
|
|
15374
|
-
async function down$
|
|
15375
|
+
async function down$7(db) {
|
|
15375
15376
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = substring(customer_id from 5) WHERE customer_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15376
15377
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = substring(org_id from 5) WHERE org_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
15377
15378
|
}
|
|
@@ -15397,8 +15398,8 @@ var init__018_org_id_prefix_backfill = __esmMin((() => {
|
|
|
15397
15398
|
//#endregion
|
|
15398
15399
|
//#region src/db/migrations/019_generic_sources_change_notify.ts
|
|
15399
15400
|
var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
15400
|
-
down: () => down$
|
|
15401
|
-
up: () => up$
|
|
15401
|
+
down: () => down$6,
|
|
15402
|
+
up: () => up$6
|
|
15402
15403
|
});
|
|
15403
15404
|
/**
|
|
15404
15405
|
* Add a Postgres trigger on `generic_webhook_sources` that emits
|
|
@@ -15421,7 +15422,7 @@ var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
15421
15422
|
* (`notify_sources_change()` + `sources_change_trigger`, defined in
|
|
15422
15423
|
* `001_initial.ts`).
|
|
15423
15424
|
*/
|
|
15424
|
-
async function up$
|
|
15425
|
+
async function up$6(db) {
|
|
15425
15426
|
await sql`
|
|
15426
15427
|
CREATE FUNCTION public.notify_generic_sources_change() RETURNS trigger
|
|
15427
15428
|
LANGUAGE plpgsql
|
|
@@ -15442,7 +15443,7 @@ async function up$5(db) {
|
|
|
15442
15443
|
FOR EACH ROW EXECUTE FUNCTION public.notify_generic_sources_change()
|
|
15443
15444
|
`.execute(db);
|
|
15444
15445
|
}
|
|
15445
|
-
async function down$
|
|
15446
|
+
async function down$6(db) {
|
|
15446
15447
|
await sql`DROP TRIGGER IF EXISTS generic_sources_change_trigger ON public.generic_webhook_sources`.execute(db);
|
|
15447
15448
|
await sql`DROP FUNCTION IF EXISTS public.notify_generic_sources_change()`.execute(db);
|
|
15448
15449
|
}
|
|
@@ -15450,8 +15451,8 @@ var init__019_generic_sources_change_notify = __esmMin((() => {}));
|
|
|
15450
15451
|
//#endregion
|
|
15451
15452
|
//#region src/db/migrations/020_org_settings_dashboard_write_policy.ts
|
|
15452
15453
|
var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportAll({
|
|
15453
|
-
down: () => down$
|
|
15454
|
-
up: () => up$
|
|
15454
|
+
down: () => down$5,
|
|
15455
|
+
up: () => up$5
|
|
15455
15456
|
});
|
|
15456
15457
|
/**
|
|
15457
15458
|
* Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
|
|
@@ -15466,7 +15467,7 @@ var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportA
|
|
|
15466
15467
|
*
|
|
15467
15468
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
15468
15469
|
*/
|
|
15469
|
-
async function up$
|
|
15470
|
+
async function up$5(db) {
|
|
15470
15471
|
if ((await sql`
|
|
15471
15472
|
SELECT EXISTS (
|
|
15472
15473
|
SELECT 1 FROM information_schema.columns
|
|
@@ -15480,7 +15481,7 @@ async function up$4(db) {
|
|
|
15480
15481
|
ADD COLUMN dashboard_write_policy jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
15481
15482
|
`.execute(db);
|
|
15482
15483
|
}
|
|
15483
|
-
async function down$
|
|
15484
|
+
async function down$5(db) {
|
|
15484
15485
|
await sql`
|
|
15485
15486
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dashboard_write_policy
|
|
15486
15487
|
`.execute(db);
|
|
@@ -15489,8 +15490,8 @@ var init__020_org_settings_dashboard_write_policy = __esmMin((() => {}));
|
|
|
15489
15490
|
//#endregion
|
|
15490
15491
|
//#region src/db/migrations/021_check_run_tracking.ts
|
|
15491
15492
|
var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
15492
|
-
down: () => down$
|
|
15493
|
-
up: () => up$
|
|
15493
|
+
down: () => down$4,
|
|
15494
|
+
up: () => up$4
|
|
15494
15495
|
});
|
|
15495
15496
|
/**
|
|
15496
15497
|
* Add `check_run_tracking` table for HA-safe check-run state persistence.
|
|
@@ -15514,7 +15515,7 @@ var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
|
15514
15515
|
*
|
|
15515
15516
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
15516
15517
|
*/
|
|
15517
|
-
async function up$
|
|
15518
|
+
async function up$4(db) {
|
|
15518
15519
|
if ((await sql`
|
|
15519
15520
|
SELECT EXISTS (
|
|
15520
15521
|
SELECT 1 FROM information_schema.tables
|
|
@@ -15545,15 +15546,15 @@ async function up$3(db) {
|
|
|
15545
15546
|
WHERE run_id IS NOT NULL
|
|
15546
15547
|
`.execute(db);
|
|
15547
15548
|
}
|
|
15548
|
-
async function down$
|
|
15549
|
+
async function down$4(db) {
|
|
15549
15550
|
await sql`DROP TABLE IF EXISTS public.check_run_tracking`.execute(db);
|
|
15550
15551
|
}
|
|
15551
15552
|
var init__021_check_run_tracking = __esmMin((() => {}));
|
|
15552
15553
|
//#endregion
|
|
15553
15554
|
//#region src/db/migrations/022_scaler_manager_state.ts
|
|
15554
15555
|
var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
15555
|
-
down: () => down$
|
|
15556
|
-
up: () => up$
|
|
15556
|
+
down: () => down$3,
|
|
15557
|
+
up: () => up$3
|
|
15557
15558
|
});
|
|
15558
15559
|
/**
|
|
15559
15560
|
* Add three tables persisting `ScalerManager` per-coord state:
|
|
@@ -15580,7 +15581,7 @@ var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
|
15580
15581
|
* Idempotent: a re-run on a DB that already has any of these tables
|
|
15581
15582
|
* leaves the existing one alone.
|
|
15582
15583
|
*/
|
|
15583
|
-
async function up$
|
|
15584
|
+
async function up$3(db) {
|
|
15584
15585
|
const tableExists = async (name) => {
|
|
15585
15586
|
return (await sql`
|
|
15586
15587
|
SELECT EXISTS (
|
|
@@ -15630,7 +15631,7 @@ async function up$2(db) {
|
|
|
15630
15631
|
`.execute(db);
|
|
15631
15632
|
}
|
|
15632
15633
|
}
|
|
15633
|
-
async function down$
|
|
15634
|
+
async function down$3(db) {
|
|
15634
15635
|
await sql`DROP TABLE IF EXISTS public.scaler_reservations`.execute(db);
|
|
15635
15636
|
await sql`DROP TABLE IF EXISTS public.scaler_agent_jobs`.execute(db);
|
|
15636
15637
|
await sql`DROP TABLE IF EXISTS public.scaler_spawning_agents`.execute(db);
|
|
@@ -15639,8 +15640,8 @@ var init__022_scaler_manager_state = __esmMin((() => {}));
|
|
|
15639
15640
|
//#endregion
|
|
15640
15641
|
//#region src/db/migrations/023_dispatch_queue_recovery_deadline.ts
|
|
15641
15642
|
var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
15642
|
-
down: () => down$
|
|
15643
|
-
up: () => up$
|
|
15643
|
+
down: () => down$2,
|
|
15644
|
+
up: () => up$2
|
|
15644
15645
|
});
|
|
15645
15646
|
/**
|
|
15646
15647
|
* Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
|
|
@@ -15664,7 +15665,7 @@ var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll(
|
|
|
15664
15665
|
* Idempotent: re-running on a DB that already has either column is a
|
|
15665
15666
|
* no-op.
|
|
15666
15667
|
*/
|
|
15667
|
-
async function up$
|
|
15668
|
+
async function up$2(db) {
|
|
15668
15669
|
const colExists = async (name) => {
|
|
15669
15670
|
return (await sql`
|
|
15670
15671
|
SELECT EXISTS (
|
|
@@ -15689,7 +15690,7 @@ async function up$1(db) {
|
|
|
15689
15690
|
WHERE recovery_deadline IS NOT NULL
|
|
15690
15691
|
`.execute(db);
|
|
15691
15692
|
}
|
|
15692
|
-
async function down$
|
|
15693
|
+
async function down$2(db) {
|
|
15693
15694
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_recovery_deadline`.execute(db);
|
|
15694
15695
|
await sql`
|
|
15695
15696
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS recovery_agent_id
|
|
@@ -15702,8 +15703,8 @@ var init__023_dispatch_queue_recovery_deadline = __esmMin((() => {}));
|
|
|
15702
15703
|
//#endregion
|
|
15703
15704
|
//#region src/db/migrations/024_dispatch_queue_provisioning_error.ts
|
|
15704
15705
|
var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll({
|
|
15705
|
-
down: () => down,
|
|
15706
|
-
up: () => up
|
|
15706
|
+
down: () => down$1,
|
|
15707
|
+
up: () => up$1
|
|
15707
15708
|
});
|
|
15708
15709
|
/**
|
|
15709
15710
|
* Add `dispatch_queue.last_provisioning_error TEXT` recording the most
|
|
@@ -15719,7 +15720,7 @@ var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll
|
|
|
15719
15720
|
*
|
|
15720
15721
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
15721
15722
|
*/
|
|
15722
|
-
async function up(db) {
|
|
15723
|
+
async function up$1(db) {
|
|
15723
15724
|
const colExists = async (name) => {
|
|
15724
15725
|
return (await sql`
|
|
15725
15726
|
SELECT EXISTS (
|
|
@@ -15735,13 +15736,60 @@ async function up(db) {
|
|
|
15735
15736
|
ADD COLUMN last_provisioning_error TEXT
|
|
15736
15737
|
`.execute(db);
|
|
15737
15738
|
}
|
|
15738
|
-
async function down(db) {
|
|
15739
|
+
async function down$1(db) {
|
|
15739
15740
|
await sql`
|
|
15740
15741
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS last_provisioning_error
|
|
15741
15742
|
`.execute(db);
|
|
15742
15743
|
}
|
|
15743
15744
|
var init__024_dispatch_queue_provisioning_error = __esmMin((() => {}));
|
|
15744
15745
|
//#endregion
|
|
15746
|
+
//#region src/db/migrations/025_init_failure.ts
|
|
15747
|
+
var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
15748
|
+
down: () => down,
|
|
15749
|
+
up: () => up
|
|
15750
|
+
});
|
|
15751
|
+
/**
|
|
15752
|
+
* Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
|
|
15753
|
+
*
|
|
15754
|
+
* Presence of this column on a row means the run/job never executed a step
|
|
15755
|
+
* because of an init-phase failure; absence (NULL) means a normal run.
|
|
15756
|
+
* Shape on the wire is `InitFailure` from `@kici-dev/engine`. The dashboard
|
|
15757
|
+
* reads this column directly so it can render the right banner without
|
|
15758
|
+
* round-tripping to the orchestrator (which may be offline).
|
|
15759
|
+
*
|
|
15760
|
+
* Idempotent: re-running on a DB that already has either column is a no-op
|
|
15761
|
+
* for that column.
|
|
15762
|
+
*/
|
|
15763
|
+
async function up(db) {
|
|
15764
|
+
const colExists = async (table, name) => {
|
|
15765
|
+
return (await sql`
|
|
15766
|
+
SELECT EXISTS (
|
|
15767
|
+
SELECT 1 FROM information_schema.columns
|
|
15768
|
+
WHERE table_schema = 'public'
|
|
15769
|
+
AND table_name = ${table}
|
|
15770
|
+
AND column_name = ${name}
|
|
15771
|
+
) AS exists
|
|
15772
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
15773
|
+
};
|
|
15774
|
+
if (!await colExists("execution_runs", "init_failure")) await sql`
|
|
15775
|
+
ALTER TABLE public.execution_runs
|
|
15776
|
+
ADD COLUMN init_failure JSONB DEFAULT NULL
|
|
15777
|
+
`.execute(db);
|
|
15778
|
+
if (!await colExists("execution_jobs", "init_failure")) await sql`
|
|
15779
|
+
ALTER TABLE public.execution_jobs
|
|
15780
|
+
ADD COLUMN init_failure JSONB DEFAULT NULL
|
|
15781
|
+
`.execute(db);
|
|
15782
|
+
}
|
|
15783
|
+
async function down(db) {
|
|
15784
|
+
await sql`
|
|
15785
|
+
ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS init_failure
|
|
15786
|
+
`.execute(db);
|
|
15787
|
+
await sql`
|
|
15788
|
+
ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS init_failure
|
|
15789
|
+
`.execute(db);
|
|
15790
|
+
}
|
|
15791
|
+
var init__025_init_failure = __esmMin((() => {}));
|
|
15792
|
+
//#endregion
|
|
15745
15793
|
//#region src/db/migration-provider.ts
|
|
15746
15794
|
function createMigrationProvider() {
|
|
15747
15795
|
return { async getMigrations() {
|
|
@@ -15769,7 +15817,8 @@ function createMigrationProvider() {
|
|
|
15769
15817
|
"021_check_run_tracking": _021_check_run_tracking_exports,
|
|
15770
15818
|
"022_scaler_manager_state": _022_scaler_manager_state_exports,
|
|
15771
15819
|
"023_dispatch_queue_recovery_deadline": _023_dispatch_queue_recovery_deadline_exports,
|
|
15772
|
-
"024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports
|
|
15820
|
+
"024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports,
|
|
15821
|
+
"025_init_failure": _025_init_failure_exports
|
|
15773
15822
|
};
|
|
15774
15823
|
} };
|
|
15775
15824
|
}
|
|
@@ -15798,6 +15847,7 @@ var init_migration_provider = __esmMin((() => {
|
|
|
15798
15847
|
init__022_scaler_manager_state();
|
|
15799
15848
|
init__023_dispatch_queue_recovery_deadline();
|
|
15800
15849
|
init__024_dispatch_queue_provisioning_error();
|
|
15850
|
+
init__025_init_failure();
|
|
15801
15851
|
}));
|
|
15802
15852
|
//#endregion
|
|
15803
15853
|
//#region src/db/migrator.ts
|
|
@@ -21704,15 +21754,15 @@ var init_admin_config = __esmMin((() => {
|
|
|
21704
21754
|
function createHealthRoutes$1(deps = {}) {
|
|
21705
21755
|
return createHealthRoutes({
|
|
21706
21756
|
livenessInfo: () => ({
|
|
21707
|
-
version: "0.1.
|
|
21708
|
-
buildDate: "2026-05-
|
|
21709
|
-
buildCommit: "
|
|
21710
|
-
sdkVersion: "0.1.
|
|
21757
|
+
version: "0.1.14",
|
|
21758
|
+
buildDate: "2026-05-30T03:04:39.672Z",
|
|
21759
|
+
buildCommit: "0fc7a43c9",
|
|
21760
|
+
sdkVersion: "0.1.14",
|
|
21711
21761
|
sdkBundleHash: "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128",
|
|
21712
|
-
sharedVersion: "0.1.
|
|
21713
|
-
sharedBundleHash: "
|
|
21714
|
-
engineVersion: "0.1.
|
|
21715
|
-
engineBundleHash: "
|
|
21762
|
+
sharedVersion: "0.1.14",
|
|
21763
|
+
sharedBundleHash: "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4",
|
|
21764
|
+
engineVersion: "0.1.14",
|
|
21765
|
+
engineBundleHash: "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932"
|
|
21716
21766
|
}),
|
|
21717
21767
|
readinessCheck: deps.db ? async () => {
|
|
21718
21768
|
const checks = {};
|
|
@@ -21746,7 +21796,7 @@ function createCapabilitiesRoutes() {
|
|
|
21746
21796
|
const app = new Hono();
|
|
21747
21797
|
app.get("/api/v1/capabilities", (c) => {
|
|
21748
21798
|
const manifest = {
|
|
21749
|
-
orchestratorVersion: "0.1.
|
|
21799
|
+
orchestratorVersion: "0.1.14",
|
|
21750
21800
|
protocolVersion: PROTOCOL_VERSION,
|
|
21751
21801
|
minProtocolVersion: MIN_PROTOCOL_VERSION
|
|
21752
21802
|
};
|
|
@@ -22048,6 +22098,61 @@ var init_certs = __esmMin((() => {
|
|
|
22048
22098
|
WARN_DAYS = 30;
|
|
22049
22099
|
}));
|
|
22050
22100
|
//#endregion
|
|
22101
|
+
//#region src/diagnostics/checks/scaler.ts
|
|
22102
|
+
function truncate(s) {
|
|
22103
|
+
return s.length > MAX_DETAIL_LEN ? `${s.slice(0, MAX_DETAIL_LEN - 1)}…` : s;
|
|
22104
|
+
}
|
|
22105
|
+
async function checkScalerProvisioning(deps) {
|
|
22106
|
+
const start = Date.now();
|
|
22107
|
+
const mgr = deps.scalerManager;
|
|
22108
|
+
const backends = mgr?.getStatus().backends ?? [];
|
|
22109
|
+
if (!mgr || backends.length === 0) return [{
|
|
22110
|
+
name: "scaler",
|
|
22111
|
+
status: "pass",
|
|
22112
|
+
message: "No scaler backends configured",
|
|
22113
|
+
durationMs: Date.now() - start
|
|
22114
|
+
}];
|
|
22115
|
+
const recent = mgr.recentSpawnFailures(SCALER_FAILURE_WINDOW_MS, Date.now());
|
|
22116
|
+
return backends.map((backend) => {
|
|
22117
|
+
const summary = recent.get(backend.name);
|
|
22118
|
+
const durationMs = Date.now() - start;
|
|
22119
|
+
if (!summary || summary.boundCount + summary.unboundCount === 0) return {
|
|
22120
|
+
name: `scaler:${backend.name}`,
|
|
22121
|
+
status: "pass",
|
|
22122
|
+
message: "0 spawn failures in last 5m",
|
|
22123
|
+
details: {
|
|
22124
|
+
windowMs: SCALER_FAILURE_WINDOW_MS,
|
|
22125
|
+
backendType: backend.type,
|
|
22126
|
+
boundCount: 0,
|
|
22127
|
+
unboundCount: 0
|
|
22128
|
+
},
|
|
22129
|
+
durationMs
|
|
22130
|
+
};
|
|
22131
|
+
const total = summary.boundCount + summary.unboundCount;
|
|
22132
|
+
const status = summary.boundCount > 0 ? "fail" : "warn";
|
|
22133
|
+
const message = `${total} spawn failures in last 5m (${summary.boundCount} bound, ${summary.unboundCount} warm-pool; last: ${truncate(summary.lastError)})`;
|
|
22134
|
+
return {
|
|
22135
|
+
name: `scaler:${backend.name}`,
|
|
22136
|
+
status,
|
|
22137
|
+
message,
|
|
22138
|
+
details: {
|
|
22139
|
+
windowMs: SCALER_FAILURE_WINDOW_MS,
|
|
22140
|
+
backendType: summary.backendType,
|
|
22141
|
+
boundCount: summary.boundCount,
|
|
22142
|
+
unboundCount: summary.unboundCount,
|
|
22143
|
+
lastError: summary.lastError,
|
|
22144
|
+
lastAtMs: summary.lastAtMs
|
|
22145
|
+
},
|
|
22146
|
+
durationMs
|
|
22147
|
+
};
|
|
22148
|
+
});
|
|
22149
|
+
}
|
|
22150
|
+
var SCALER_FAILURE_WINDOW_MS, MAX_DETAIL_LEN;
|
|
22151
|
+
var init_scaler$1 = __esmMin((() => {
|
|
22152
|
+
SCALER_FAILURE_WINDOW_MS = 300 * 1e3;
|
|
22153
|
+
MAX_DETAIL_LEN = 120;
|
|
22154
|
+
}));
|
|
22155
|
+
//#endregion
|
|
22051
22156
|
//#region src/diagnostics/checks/index.ts
|
|
22052
22157
|
var defaultChecks;
|
|
22053
22158
|
var init_checks = __esmMin((() => {
|
|
@@ -22057,13 +22162,15 @@ var init_checks = __esmMin((() => {
|
|
|
22057
22162
|
init_disk();
|
|
22058
22163
|
init_config$2();
|
|
22059
22164
|
init_certs();
|
|
22165
|
+
init_scaler$1();
|
|
22060
22166
|
defaultChecks = [
|
|
22061
22167
|
checkDbConnectivity,
|
|
22062
22168
|
checkWsToPlatform,
|
|
22063
22169
|
checkAgentConnectivity,
|
|
22064
22170
|
checkDiskSpace,
|
|
22065
22171
|
checkConfigValidity,
|
|
22066
|
-
checkCertificateExpiry
|
|
22172
|
+
checkCertificateExpiry,
|
|
22173
|
+
checkScalerProvisioning
|
|
22067
22174
|
];
|
|
22068
22175
|
}));
|
|
22069
22176
|
//#endregion
|
|
@@ -22079,14 +22186,14 @@ var init_checks = __esmMin((() => {
|
|
|
22079
22186
|
async function runDiagnostics(deps, checks, options) {
|
|
22080
22187
|
const checksToRun = checks ?? defaultChecks;
|
|
22081
22188
|
const timeoutMs = options?.timeoutMs ?? 5e3;
|
|
22082
|
-
return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).
|
|
22083
|
-
if (result.status === "fulfilled") return result.value;
|
|
22084
|
-
return {
|
|
22189
|
+
return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).flatMap((result, index) => {
|
|
22190
|
+
if (result.status === "fulfilled") return Array.isArray(result.value) ? result.value : [result.value];
|
|
22191
|
+
return [{
|
|
22085
22192
|
name: `Check ${index + 1}`,
|
|
22086
22193
|
status: "fail",
|
|
22087
22194
|
message: `Unexpected error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
|
|
22088
22195
|
durationMs: 0
|
|
22089
|
-
};
|
|
22196
|
+
}];
|
|
22090
22197
|
});
|
|
22091
22198
|
}
|
|
22092
22199
|
/**
|
|
@@ -23477,7 +23584,11 @@ async function runBuildJob(args) {
|
|
|
23477
23584
|
jobName: buildJobName,
|
|
23478
23585
|
runsOnLabels: buildJobInput.runsOnLabels
|
|
23479
23586
|
}], setup.info.routingKey, void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
|
|
23480
|
-
await deps.executionTracker.failRun(runId, reason
|
|
23587
|
+
await deps.executionTracker.failRun(runId, reason, {
|
|
23588
|
+
scope: "run",
|
|
23589
|
+
category: InitFailureCategory.enum.build_coordination,
|
|
23590
|
+
message: reason
|
|
23591
|
+
});
|
|
23481
23592
|
}
|
|
23482
23593
|
return;
|
|
23483
23594
|
}
|
|
@@ -23530,8 +23641,14 @@ async function recordBuildFailure(args) {
|
|
|
23530
23641
|
});
|
|
23531
23642
|
}
|
|
23532
23643
|
if (deps.executionTracker) try {
|
|
23533
|
-
|
|
23534
|
-
|
|
23644
|
+
const buildFailureReason = toErrorMessage(err);
|
|
23645
|
+
const buildInitFailure = {
|
|
23646
|
+
scope: "run",
|
|
23647
|
+
category: InitFailureCategory.enum.build_coordination,
|
|
23648
|
+
message: buildFailureReason
|
|
23649
|
+
};
|
|
23650
|
+
if (buildJobTrackedEarly) await deps.executionTracker.onBuildFailed(runId, buildInitFailure);
|
|
23651
|
+
else await deps.executionTracker.onBuildFailedBeforeTracking(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, setup.info.routingKey, buildTriggerEvent(event.type, event.action), extractCommitMessage(setup.info.event, setup.info.payload), buildFailureReason, buildInitFailure);
|
|
23535
23652
|
} catch (cleanupErr) {
|
|
23536
23653
|
logger$37.warn("Failed to mark run as failed after build error", {
|
|
23537
23654
|
runId,
|
|
@@ -23741,11 +23858,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
|
|
|
23741
23858
|
const declaredContexts = workflow.contexts ?? [];
|
|
23742
23859
|
if (declaredContexts.length > 0) {
|
|
23743
23860
|
if (!deps.secretResolver) {
|
|
23744
|
-
|
|
23861
|
+
const reason = "Workflow declares secret contexts but secrets subsystem is not configured (KICI_SECRET_KEY missing)";
|
|
23862
|
+
logger$37.error(reason, {
|
|
23745
23863
|
workflow: workflow.name,
|
|
23746
23864
|
contexts: declaredContexts
|
|
23747
23865
|
});
|
|
23748
|
-
return {
|
|
23866
|
+
return {
|
|
23867
|
+
skipDispatch: true,
|
|
23868
|
+
reason
|
|
23869
|
+
};
|
|
23749
23870
|
}
|
|
23750
23871
|
try {
|
|
23751
23872
|
const mergedSecrets = {};
|
|
@@ -23760,11 +23881,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
|
|
|
23760
23881
|
resolvedNamespacedSecrets = mergedNamespaced;
|
|
23761
23882
|
}
|
|
23762
23883
|
} catch (err) {
|
|
23884
|
+
const errMessage = toErrorMessage(err);
|
|
23763
23885
|
logger$37.error("Secret resolution failed, skipping workflow", {
|
|
23764
23886
|
workflow: workflow.name,
|
|
23765
|
-
error:
|
|
23887
|
+
error: errMessage
|
|
23766
23888
|
});
|
|
23767
|
-
return {
|
|
23889
|
+
return {
|
|
23890
|
+
skipDispatch: true,
|
|
23891
|
+
reason: `Secret resolution failed: ${errMessage}`
|
|
23892
|
+
};
|
|
23768
23893
|
}
|
|
23769
23894
|
}
|
|
23770
23895
|
let runPublicKeyBase64;
|
|
@@ -23842,7 +23967,10 @@ async function resolveWorkflowInstallSecrets(ctx, secrets) {
|
|
|
23842
23967
|
workflow: workflow.name,
|
|
23843
23968
|
reason: result.reason
|
|
23844
23969
|
});
|
|
23845
|
-
return {
|
|
23970
|
+
return {
|
|
23971
|
+
skipDispatch: true,
|
|
23972
|
+
reason: `Workflow install-secrets resolution rejected dispatch: ${result.reason}`
|
|
23973
|
+
};
|
|
23846
23974
|
}
|
|
23847
23975
|
if (result.contributorStripped) logger$37.warn("Skipping registries:/installEnv: resolution for untrusted contributor — install will fail naturally if private deps are required", {
|
|
23848
23976
|
runId,
|
|
@@ -24295,6 +24423,7 @@ async function clusterRouteRootJobs(args) {
|
|
|
24295
24423
|
});
|
|
24296
24424
|
rejectedJobs.push({
|
|
24297
24425
|
jobId: syntheticId,
|
|
24426
|
+
jobName: jtr.jobName,
|
|
24298
24427
|
reason: result.reason
|
|
24299
24428
|
});
|
|
24300
24429
|
} else if (result.status === "queued-no-backend") logger$37.warn("Job has no matching backend (cluster fallback), skipping", {
|
|
@@ -24400,6 +24529,7 @@ async function dispatchSingleOrchPath(args) {
|
|
|
24400
24529
|
});
|
|
24401
24530
|
rejectedJobs.push({
|
|
24402
24531
|
jobId: syntheticId,
|
|
24532
|
+
jobName: jobOrFactory.name,
|
|
24403
24533
|
reason: result.reason
|
|
24404
24534
|
});
|
|
24405
24535
|
} else if (result.status === "queued-no-backend") logger$37.warn("Job has no matching backend, skipping execution tracking", {
|
|
@@ -24504,6 +24634,11 @@ async function recordRunStart(args) {
|
|
|
24504
24634
|
});
|
|
24505
24635
|
});
|
|
24506
24636
|
}
|
|
24637
|
+
function categorizeRejectReason(reason) {
|
|
24638
|
+
const lower = reason.toLowerCase();
|
|
24639
|
+
if (/no\s+agent|no\s+matching\s+backend/.test(lower)) return InitFailureCategory.enum.no_agent;
|
|
24640
|
+
return InitFailureCategory.enum.environment_rules;
|
|
24641
|
+
}
|
|
24507
24642
|
async function insertEdgesAndMarkRejected(args) {
|
|
24508
24643
|
const { ctx, staticJobs, dispatchedJobs, rejectedJobs } = args;
|
|
24509
24644
|
const { deps, runId } = ctx;
|
|
@@ -24517,7 +24652,15 @@ async function insertEdgesAndMarkRejected(args) {
|
|
|
24517
24652
|
}
|
|
24518
24653
|
if (deps.executionTracker && rejectedJobs.length > 0) {
|
|
24519
24654
|
const now = Date.now();
|
|
24520
|
-
for (const { jobId, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, {
|
|
24655
|
+
for (const { jobId, jobName, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, {
|
|
24656
|
+
error: reason,
|
|
24657
|
+
initFailure: {
|
|
24658
|
+
scope: "job",
|
|
24659
|
+
category: categorizeRejectReason(reason),
|
|
24660
|
+
message: reason,
|
|
24661
|
+
jobName
|
|
24662
|
+
}
|
|
24663
|
+
}).catch((err) => {
|
|
24521
24664
|
logger$37.error("Failed to mark rejected job as failed", {
|
|
24522
24665
|
runId,
|
|
24523
24666
|
jobId,
|
|
@@ -24698,17 +24841,30 @@ function startDeferredInitDispatch(args) {
|
|
|
24698
24841
|
lockJob
|
|
24699
24842
|
});
|
|
24700
24843
|
} catch (err) {
|
|
24844
|
+
const errMsg = toErrorMessage(err);
|
|
24701
24845
|
logger$37.error("Deferred init job failed", {
|
|
24702
24846
|
runId,
|
|
24703
24847
|
workflow: workflow.name,
|
|
24704
24848
|
job: lockJob.name,
|
|
24705
|
-
error:
|
|
24849
|
+
error: errMsg
|
|
24706
24850
|
});
|
|
24707
|
-
if (deps.executionTracker)
|
|
24708
|
-
jobId
|
|
24709
|
-
|
|
24710
|
-
|
|
24711
|
-
|
|
24851
|
+
if (deps.executionTracker) {
|
|
24852
|
+
const jobId = `init-failed-${lockJob.name}`;
|
|
24853
|
+
await deps.executionTracker.addJobsToRun(runId, [{
|
|
24854
|
+
jobId,
|
|
24855
|
+
jobName: lockJob.name,
|
|
24856
|
+
runsOnLabels: Array.isArray(lockJob.runsOn) ? [...lockJob.runsOn] : [lockJob.runsOn]
|
|
24857
|
+
}]).catch(() => {});
|
|
24858
|
+
await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
|
|
24859
|
+
error: errMsg,
|
|
24860
|
+
initFailure: {
|
|
24861
|
+
scope: "job",
|
|
24862
|
+
category: InitFailureCategory.enum.dynamic_eval,
|
|
24863
|
+
message: errMsg,
|
|
24864
|
+
jobName: lockJob.name
|
|
24865
|
+
}
|
|
24866
|
+
}).catch(() => {});
|
|
24867
|
+
}
|
|
24712
24868
|
}
|
|
24713
24869
|
})();
|
|
24714
24870
|
}
|
|
@@ -25250,6 +25406,39 @@ async function ensureExecutionRunForDeferred(args) {
|
|
|
25250
25406
|
await deps.executionTracker.onExecutionStarted(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, summarizeDecision(decision), [], setup.info.routingKey, declaredContexts.length > 0 ? [...declaredContexts] : void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
|
|
25251
25407
|
}
|
|
25252
25408
|
/**
|
|
25409
|
+
* Persist a failed `execution_runs` row for a pre-dispatch early-exit so the
|
|
25410
|
+
* dashboard's Runs view surfaces secret_resolution / install_secrets /
|
|
25411
|
+
* environment_rules rejections instead of leaving the run with zero trace.
|
|
25412
|
+
*
|
|
25413
|
+
* Called from each of the three early-exit sites in `dispatchMatchedWorkflow`
|
|
25414
|
+
* (workflow secrets, install secrets, all-jobs-rejected). The helper is a
|
|
25415
|
+
* no-op when no `executionTracker` is wired into deps (test / minimal
|
|
25416
|
+
* deployments).
|
|
25417
|
+
*/
|
|
25418
|
+
async function recordInitFailureFromSkip(args) {
|
|
25419
|
+
const { ctx, setup, category, reason } = args;
|
|
25420
|
+
const { deps, workflow, repoIdentifier, credentials, event, ref, runId } = ctx;
|
|
25421
|
+
if (!deps.executionTracker) return;
|
|
25422
|
+
await deps.executionTracker.recordInitFailureRun({
|
|
25423
|
+
runId,
|
|
25424
|
+
workflowName: workflow.name,
|
|
25425
|
+
provider: setup.info.provider,
|
|
25426
|
+
repoIdentifier,
|
|
25427
|
+
ref: event.sourceBranch ?? event.targetBranch,
|
|
25428
|
+
sha: ref,
|
|
25429
|
+
deliveryId: setup.effectiveDeliveryId,
|
|
25430
|
+
providerContext: credentials ?? {},
|
|
25431
|
+
routingKey: setup.info.routingKey,
|
|
25432
|
+
initFailure: {
|
|
25433
|
+
scope: "run",
|
|
25434
|
+
category,
|
|
25435
|
+
message: reason
|
|
25436
|
+
},
|
|
25437
|
+
triggerEvent: buildTriggerEvent(event.type, event.action),
|
|
25438
|
+
commitMessage: extractCommitMessage(setup.info.event, setup.info.payload)
|
|
25439
|
+
});
|
|
25440
|
+
}
|
|
25441
|
+
/**
|
|
25253
25442
|
* Dispatch a single matched workflow.
|
|
25254
25443
|
*
|
|
25255
25444
|
* Narrative orchestrator: each phase is a typed helper that returns a typed
|
|
@@ -25269,9 +25458,26 @@ async function dispatchMatchedWorkflow(ctx) {
|
|
|
25269
25458
|
const buildPrep = await prepareCacheAndBuild(ctx, setup);
|
|
25270
25459
|
if (buildPrep.abort) return { dispatchedJobCount: 0 };
|
|
25271
25460
|
const secretsResult = await resolveWorkflowSecretsAndKey(ctx);
|
|
25272
|
-
if ("skipDispatch" in secretsResult)
|
|
25461
|
+
if ("skipDispatch" in secretsResult) {
|
|
25462
|
+
await recordInitFailureFromSkip({
|
|
25463
|
+
ctx,
|
|
25464
|
+
setup,
|
|
25465
|
+
category: InitFailureCategory.enum.secret_resolution,
|
|
25466
|
+
reason: secretsResult.reason
|
|
25467
|
+
});
|
|
25468
|
+
return { dispatchedJobCount: 0 };
|
|
25469
|
+
}
|
|
25273
25470
|
const secrets = secretsResult;
|
|
25274
|
-
|
|
25471
|
+
const installResult = await resolveWorkflowInstallSecrets(ctx, secrets);
|
|
25472
|
+
if (installResult.skipDispatch) {
|
|
25473
|
+
await recordInitFailureFromSkip({
|
|
25474
|
+
ctx,
|
|
25475
|
+
setup,
|
|
25476
|
+
category: InitFailureCategory.enum.install_secrets,
|
|
25477
|
+
reason: installResult.reason
|
|
25478
|
+
});
|
|
25479
|
+
return { dispatchedJobCount: 0 };
|
|
25480
|
+
}
|
|
25275
25481
|
const evalResult = await evaluateJobEnvironments({
|
|
25276
25482
|
ctx,
|
|
25277
25483
|
setup,
|
|
@@ -25318,6 +25524,12 @@ async function dispatchMatchedWorkflow(ctx) {
|
|
|
25318
25524
|
dispatchedJobs,
|
|
25319
25525
|
rejectedJobs
|
|
25320
25526
|
});
|
|
25527
|
+
if (dispatchedJobs.length === 0 && evalResult.deferredInitJobs.length === 0 && buildPrep.dynamicEntries.length === 0 && rejectedJobs.length > 0) await recordInitFailureFromSkip({
|
|
25528
|
+
ctx,
|
|
25529
|
+
setup,
|
|
25530
|
+
category: InitFailureCategory.enum.environment_rules,
|
|
25531
|
+
reason: rejectedJobs[0].reason
|
|
25532
|
+
});
|
|
25321
25533
|
if (evalResult.deferredInitJobs.length > 0) await ensureExecutionRunForDeferred({
|
|
25322
25534
|
ctx,
|
|
25323
25535
|
setup,
|
|
@@ -28039,7 +28251,8 @@ function createApp(deps) {
|
|
|
28039
28251
|
platformUrl: deps.config.platformUrl,
|
|
28040
28252
|
agentRegistry: deps.registry,
|
|
28041
28253
|
config: deps.config,
|
|
28042
|
-
tlsCertPath: deps.config.tlsCertPath
|
|
28254
|
+
tlsCertPath: deps.config.tlsCertPath,
|
|
28255
|
+
scalerManager: deps.scalerManager
|
|
28043
28256
|
}));
|
|
28044
28257
|
{
|
|
28045
28258
|
const peerHandlerRef = deps.peerHandler;
|
|
@@ -31199,6 +31412,55 @@ var init_machine_ledger = __esmMin((() => {
|
|
|
31199
31412
|
};
|
|
31200
31413
|
}));
|
|
31201
31414
|
//#endregion
|
|
31415
|
+
//#region src/scaler/failure-tracker.ts
|
|
31416
|
+
var DEFAULT_MAX_ENTRIES, ScalerFailureTracker;
|
|
31417
|
+
var init_failure_tracker = __esmMin((() => {
|
|
31418
|
+
DEFAULT_MAX_ENTRIES = 256;
|
|
31419
|
+
ScalerFailureTracker = class {
|
|
31420
|
+
records = [];
|
|
31421
|
+
maxEntries;
|
|
31422
|
+
constructor(maxEntries = DEFAULT_MAX_ENTRIES) {
|
|
31423
|
+
this.maxEntries = maxEntries;
|
|
31424
|
+
}
|
|
31425
|
+
/** Record a failure, evicting the oldest entry when over capacity. */
|
|
31426
|
+
record(rec) {
|
|
31427
|
+
this.records.push(rec);
|
|
31428
|
+
if (this.records.length > this.maxEntries) this.records.shift();
|
|
31429
|
+
}
|
|
31430
|
+
/**
|
|
31431
|
+
* Group failures newer than `nowMs - windowMs` by backend instance name.
|
|
31432
|
+
* `nowMs` is injected so callers control the clock (and tests are
|
|
31433
|
+
* deterministic).
|
|
31434
|
+
*/
|
|
31435
|
+
recentByBackend(windowMs, nowMs) {
|
|
31436
|
+
const cutoff = nowMs - windowMs;
|
|
31437
|
+
const out = /* @__PURE__ */ new Map();
|
|
31438
|
+
for (const rec of this.records) {
|
|
31439
|
+
if (rec.timestampMs < cutoff) continue;
|
|
31440
|
+
const existing = out.get(rec.backendName);
|
|
31441
|
+
if (!existing) {
|
|
31442
|
+
out.set(rec.backendName, {
|
|
31443
|
+
backendType: rec.backendType,
|
|
31444
|
+
boundCount: rec.bound ? 1 : 0,
|
|
31445
|
+
unboundCount: rec.bound ? 0 : 1,
|
|
31446
|
+
lastError: rec.detail,
|
|
31447
|
+
lastAtMs: rec.timestampMs
|
|
31448
|
+
});
|
|
31449
|
+
continue;
|
|
31450
|
+
}
|
|
31451
|
+
if (rec.bound) existing.boundCount += 1;
|
|
31452
|
+
else existing.unboundCount += 1;
|
|
31453
|
+
if (rec.timestampMs >= existing.lastAtMs) {
|
|
31454
|
+
existing.lastError = rec.detail;
|
|
31455
|
+
existing.lastAtMs = rec.timestampMs;
|
|
31456
|
+
existing.backendType = rec.backendType;
|
|
31457
|
+
}
|
|
31458
|
+
}
|
|
31459
|
+
return out;
|
|
31460
|
+
}
|
|
31461
|
+
};
|
|
31462
|
+
}));
|
|
31463
|
+
//#endregion
|
|
31202
31464
|
//#region src/scaler/manager.ts
|
|
31203
31465
|
/**
|
|
31204
31466
|
* ScalerManager: central coordinator for all scaler backends.
|
|
@@ -31262,10 +31524,13 @@ var init_manager = __esmMin((() => {
|
|
|
31262
31524
|
init_machine_ledger();
|
|
31263
31525
|
init_prometheus();
|
|
31264
31526
|
init_types$1();
|
|
31527
|
+
init_failure_tracker();
|
|
31265
31528
|
logger$27 = createLogger({ prefix: "scaler" });
|
|
31266
31529
|
ScalerManager = class {
|
|
31267
31530
|
backends = /* @__PURE__ */ new Map();
|
|
31268
31531
|
backendRoles = /* @__PURE__ */ new Map();
|
|
31532
|
+
/** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
|
|
31533
|
+
failureTracker = new ScalerFailureTracker();
|
|
31269
31534
|
globalMaxAgents;
|
|
31270
31535
|
/** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
|
|
31271
31536
|
resourceCaps = /* @__PURE__ */ new Map();
|
|
@@ -31907,6 +32172,13 @@ var init_manager = __esmMin((() => {
|
|
|
31907
32172
|
/**
|
|
31908
32173
|
* Return status summary for metrics and health endpoints.
|
|
31909
32174
|
*/
|
|
32175
|
+
/**
|
|
32176
|
+
* Recent scaler spawn failures grouped per backend instance, for the
|
|
32177
|
+
* diagnose scaler check. `nowMs` is injected by the caller.
|
|
32178
|
+
*/
|
|
32179
|
+
recentSpawnFailures(windowMs, nowMs) {
|
|
32180
|
+
return this.failureTracker.recentByBackend(windowMs, nowMs);
|
|
32181
|
+
}
|
|
31910
32182
|
getStatus() {
|
|
31911
32183
|
const backendStatuses = [];
|
|
31912
32184
|
const enrichedEntries = this.getEnrichedScalerEntries();
|
|
@@ -32193,6 +32465,13 @@ var init_manager = __esmMin((() => {
|
|
|
32193
32465
|
jobId: jobId ?? null,
|
|
32194
32466
|
detail: event.detail
|
|
32195
32467
|
});
|
|
32468
|
+
this.failureTracker.record({
|
|
32469
|
+
backendName: backendName ?? "unknown",
|
|
32470
|
+
backendType: backend,
|
|
32471
|
+
bound: Boolean(jobId),
|
|
32472
|
+
detail: event.detail ?? "",
|
|
32473
|
+
timestampMs: event.timestampMs
|
|
32474
|
+
});
|
|
32196
32475
|
}
|
|
32197
32476
|
if (runId && jobId && this.onScalerEvent) this.onScalerEvent(runId, jobId, event);
|
|
32198
32477
|
else if (this.onScalerEvent) {
|
|
@@ -33948,6 +34227,22 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
33948
34227
|
orgId;
|
|
33949
34228
|
jobQueue;
|
|
33950
34229
|
runs = /* @__PURE__ */ new Map();
|
|
34230
|
+
/**
|
|
34231
|
+
* Per-run async-mutex chain. `onJobStatus` and `addJobsToRun` mutate the same
|
|
34232
|
+
* `run.jobs` Map / `execution_jobs` row; without serialization a job-status
|
|
34233
|
+
* reply that lands mid synthetic→real swap (`addJobsToRun`) clobbers the swap
|
|
34234
|
+
* and the run hangs in `running` forever (see `withRunLock`). The map value is
|
|
34235
|
+
* the tail of the chain for that runId; entries are GC'd when the last holder
|
|
34236
|
+
* releases.
|
|
34237
|
+
*/
|
|
34238
|
+
runLockTails = /* @__PURE__ */ new Map();
|
|
34239
|
+
/**
|
|
34240
|
+
* Tracks which runIds the current async context already holds in
|
|
34241
|
+
* `withRunLock`, so reentrant calls (e.g. onJobStatus → scheduler hook →
|
|
34242
|
+
* dispatchReadyJob → addJobsToRun, all the same runId) bypass re-acquisition
|
|
34243
|
+
* instead of deadlocking. Propagates across awaits via AsyncLocalStorage.
|
|
34244
|
+
*/
|
|
34245
|
+
heldRunLocks = new AsyncLocalStorage();
|
|
33951
34246
|
/** Tracks which runs are test runs for observer broadcasting. */
|
|
33952
34247
|
testRunIds = /* @__PURE__ */ new Set();
|
|
33953
34248
|
/**
|
|
@@ -34102,7 +34397,44 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34102
34397
|
}
|
|
34103
34398
|
return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
|
|
34104
34399
|
}
|
|
34400
|
+
/**
|
|
34401
|
+
* Run `fn` while holding a per-run lock, serializing the run-mutating methods
|
|
34402
|
+
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|
|
34403
|
+
* synthetic→real job swap and wedge the run in `running`.
|
|
34404
|
+
*
|
|
34405
|
+
* The lock is **reentrant**: three paths re-enter a locked method within the
|
|
34406
|
+
* same async context for the same runId — onJobStatus → scheduler hook →
|
|
34407
|
+
* dispatchReadyJob → addJobsToRun; onJobStatus → enforceSchedulerInvariant →
|
|
34408
|
+
* onJobStatus; runSchedulerHook → onJobStatus (skip). A non-reentrant mutex
|
|
34409
|
+
* would deadlock on these, so a context that already holds the runId's lock
|
|
34410
|
+
* (tracked via `heldRunLocks`) runs `fn` inline. A genuinely concurrent caller
|
|
34411
|
+
* for the same runId (a separate WS message) is a different async context and
|
|
34412
|
+
* correctly waits. All reentrant paths are same-runId, so there is no
|
|
34413
|
+
* cross-run lock-ordering deadlock.
|
|
34414
|
+
*/
|
|
34415
|
+
async withRunLock(runId, fn) {
|
|
34416
|
+
if (this.heldRunLocks.getStore()?.has(runId)) return fn();
|
|
34417
|
+
const prev = this.runLockTails.get(runId) ?? Promise.resolve();
|
|
34418
|
+
let release;
|
|
34419
|
+
const mine = new Promise((resolve) => {
|
|
34420
|
+
release = resolve;
|
|
34421
|
+
});
|
|
34422
|
+
const chained = prev.then(() => mine);
|
|
34423
|
+
this.runLockTails.set(runId, chained);
|
|
34424
|
+
await prev.catch(() => {});
|
|
34425
|
+
const nextHeld = new Set(this.heldRunLocks.getStore() ?? []);
|
|
34426
|
+
nextHeld.add(runId);
|
|
34427
|
+
try {
|
|
34428
|
+
return await this.heldRunLocks.run(nextHeld, fn);
|
|
34429
|
+
} finally {
|
|
34430
|
+
release();
|
|
34431
|
+
if (this.runLockTails.get(runId) === chained) this.runLockTails.delete(runId);
|
|
34432
|
+
}
|
|
34433
|
+
}
|
|
34105
34434
|
async addJobsToRun(runId, jobs, dispatchedContexts, replaceSyntheticId) {
|
|
34435
|
+
return this.withRunLock(runId, () => this.addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId));
|
|
34436
|
+
}
|
|
34437
|
+
async addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId) {
|
|
34106
34438
|
const run = this.runs.get(runId);
|
|
34107
34439
|
if (!run) {
|
|
34108
34440
|
logger$23.warn("addJobsToRun: run not found in memory", { runId });
|
|
@@ -34197,6 +34529,9 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34197
34529
|
* fires the onExecutionComplete callback.
|
|
34198
34530
|
*/
|
|
34199
34531
|
async onJobStatus(runId, jobId, state, timestamp, agentId, data) {
|
|
34532
|
+
return this.withRunLock(runId, () => this.onJobStatusImpl(runId, jobId, state, timestamp, agentId, data));
|
|
34533
|
+
}
|
|
34534
|
+
async onJobStatusImpl(runId, jobId, state, timestamp, agentId, data) {
|
|
34200
34535
|
let run = this.runs.get(runId);
|
|
34201
34536
|
let job = run?.jobs.get(jobId);
|
|
34202
34537
|
if (!run) {
|
|
@@ -34317,6 +34652,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34317
34652
|
* run's in-memory map. Returns the freshly-inserted job entry.
|
|
34318
34653
|
*/
|
|
34319
34654
|
async recoverJobFromDispatchQueue(run, runId, jobId) {
|
|
34655
|
+
const existing = run.jobs.get(jobId);
|
|
34656
|
+
if (existing) return existing;
|
|
34320
34657
|
const jobName = (await this.db.selectFrom("dispatch_queue").select(["job_name"]).where("id", "=", jobId).executeTakeFirst())?.job_name ?? jobId;
|
|
34321
34658
|
run.jobs.set(jobId, {
|
|
34322
34659
|
name: jobName,
|
|
@@ -34351,6 +34688,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34351
34688
|
updateValues.duration_ms = timestamp - job.startedAt;
|
|
34352
34689
|
}
|
|
34353
34690
|
if (data?.error) updateValues.error_message = String(data.error);
|
|
34691
|
+
if (data?.initFailure) updateValues.init_failure = JSON.stringify(data.initFailure);
|
|
34354
34692
|
if (data?.outputs) updateValues.outputs = JSON.stringify(data.outputs);
|
|
34355
34693
|
jobLogBytesTotal = this.jobLogBytes.get(runId)?.get(jobId) ?? 0;
|
|
34356
34694
|
updateValues.log_bytes = jobLogBytesTotal;
|
|
@@ -34469,7 +34807,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34469
34807
|
const completedAtMs = isTerminal ? timestamp : void 0;
|
|
34470
34808
|
const duration = isTerminal && job.startedAt ? timestamp - job.startedAt : void 0;
|
|
34471
34809
|
const errorMsg = isTerminal && data?.error ? String(data.error) : void 0;
|
|
34472
|
-
|
|
34810
|
+
const initFailure = isTerminal && data?.initFailure ? data.initFailure : void 0;
|
|
34811
|
+
this.onJobStatusChange?.(runId, jobId, job.name, state, timestamp, startedAtMs, completedAtMs, duration, errorMsg, job.agentId, job.runsOnLabels, jobLogBytesTotal, initFailure);
|
|
34473
34812
|
}
|
|
34474
34813
|
/**
|
|
34475
34814
|
* Phase 6: reconcile orphaned steps. If a job reaches terminal state but
|
|
@@ -34666,13 +35005,14 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34666
35005
|
* onExecutionStarted but the build subsequently fails. Without this,
|
|
34667
35006
|
* the execution_runs row would stay in a non-terminal state.
|
|
34668
35007
|
*/
|
|
34669
|
-
async onBuildFailed(runId) {
|
|
35008
|
+
async onBuildFailed(runId, initFailure) {
|
|
34670
35009
|
const now = /* @__PURE__ */ new Date();
|
|
34671
|
-
const failureReason = "Build job failed";
|
|
35010
|
+
const failureReason = initFailure?.message ?? "Build job failed";
|
|
34672
35011
|
await this.db.updateTable("execution_runs").set({
|
|
34673
35012
|
status: ExecutionRunStatus.enum.failed,
|
|
34674
35013
|
completed_at: now,
|
|
34675
|
-
failure_reason: failureReason
|
|
35014
|
+
failure_reason: failureReason,
|
|
35015
|
+
...initFailure && { init_failure: JSON.stringify(initFailure) }
|
|
34676
35016
|
}).where("run_id", "=", runId).execute();
|
|
34677
35017
|
await this.db.updateTable("execution_jobs").set({
|
|
34678
35018
|
status: ExecutionJobStatus.enum.failed,
|
|
@@ -34695,7 +35035,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34695
35035
|
parentRunId: run.parentRunId,
|
|
34696
35036
|
originalRunId: run.originalRunId,
|
|
34697
35037
|
triggeredBy: run.triggeredBy
|
|
34698
|
-
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason);
|
|
35038
|
+
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason, void 0, initFailure);
|
|
34699
35039
|
this.runs.delete(runId);
|
|
34700
35040
|
this.jobLogBytes.delete(runId);
|
|
34701
35041
|
}
|
|
@@ -34708,7 +35048,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34708
35048
|
* Inserts a minimal execution_runs row with status='failed' directly so the E2E
|
|
34709
35049
|
* test (and dashboard) can observe the failure instead of a missing run.
|
|
34710
35050
|
*/
|
|
34711
|
-
async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason) {
|
|
35051
|
+
async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason, initFailure) {
|
|
34712
35052
|
const now = /* @__PURE__ */ new Date();
|
|
34713
35053
|
const reason = failureReason ?? "Build job timed out before execution tracking started";
|
|
34714
35054
|
await this.db.insertInto("execution_runs").values({
|
|
@@ -34725,7 +35065,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34725
35065
|
started_at: now,
|
|
34726
35066
|
completed_at: now,
|
|
34727
35067
|
status: ExecutionRunStatus.enum.failed,
|
|
34728
|
-
failure_reason: reason
|
|
35068
|
+
failure_reason: reason,
|
|
35069
|
+
...initFailure && { init_failure: JSON.stringify(initFailure) }
|
|
34729
35070
|
}).execute();
|
|
34730
35071
|
logger$23.info("Created failed execution run (build failed before tracking)", {
|
|
34731
35072
|
runId,
|
|
@@ -34734,6 +35075,55 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34734
35075
|
});
|
|
34735
35076
|
}
|
|
34736
35077
|
/**
|
|
35078
|
+
* Insert a `failed` execution_runs row directly for an init failure that
|
|
35079
|
+
* occurred BEFORE onExecutionStarted ran (so no in-memory state exists
|
|
35080
|
+
* and no jobs were dispatched). Also writes the structured init_failure
|
|
35081
|
+
* signal and fires onExecutionStatusChange so Platform's projection picks
|
|
35082
|
+
* it up via the normal forward path. Idempotent: if a row already exists
|
|
35083
|
+
* for this runId, the insert is a no-op (ON CONFLICT DO NOTHING).
|
|
35084
|
+
*
|
|
35085
|
+
* Closes the silent pre-run-failure gap — without this helper, secret /
|
|
35086
|
+
* install-secret / all-jobs-rejected early-exits in dispatch-matched-workflow
|
|
35087
|
+
* leave no trace on the dashboard.
|
|
35088
|
+
*/
|
|
35089
|
+
async recordInitFailureRun(args) {
|
|
35090
|
+
const now = /* @__PURE__ */ new Date();
|
|
35091
|
+
await this.db.insertInto("execution_runs").values({
|
|
35092
|
+
run_id: args.runId,
|
|
35093
|
+
routing_key: args.routingKey,
|
|
35094
|
+
workflow_name: args.workflowName,
|
|
35095
|
+
provider: args.provider,
|
|
35096
|
+
repo_identifier: args.repoIdentifier,
|
|
35097
|
+
ref: args.ref,
|
|
35098
|
+
sha: args.sha,
|
|
35099
|
+
delivery_id: args.deliveryId,
|
|
35100
|
+
trigger_decision: null,
|
|
35101
|
+
provider_context: JSON.stringify(args.providerContext),
|
|
35102
|
+
started_at: now,
|
|
35103
|
+
completed_at: now,
|
|
35104
|
+
status: ExecutionRunStatus.enum.failed,
|
|
35105
|
+
failure_reason: args.initFailure.message,
|
|
35106
|
+
init_failure: JSON.stringify(args.initFailure)
|
|
35107
|
+
}).onConflict((oc) => oc.column("run_id").doNothing()).execute();
|
|
35108
|
+
executionsTotal.add(1, { status: ExecutionRunStatus.enum.failed });
|
|
35109
|
+
this.onExecutionStatusChange?.(args.runId, ExecutionRunStatus.enum.failed, {
|
|
35110
|
+
workflowName: args.workflowName,
|
|
35111
|
+
provider: args.provider,
|
|
35112
|
+
repoIdentifier: args.repoIdentifier,
|
|
35113
|
+
sha: args.sha,
|
|
35114
|
+
routingKey: args.routingKey,
|
|
35115
|
+
ref: args.ref,
|
|
35116
|
+
triggerEvent: args.triggerEvent,
|
|
35117
|
+
commitMessage: args.commitMessage
|
|
35118
|
+
}, 0, now.getTime(), now.getTime(), 0, args.initFailure.message, 0, args.initFailure);
|
|
35119
|
+
logger$23.info("Recorded init-failure execution run", {
|
|
35120
|
+
runId: args.runId,
|
|
35121
|
+
workflowName: args.workflowName,
|
|
35122
|
+
category: args.initFailure.category,
|
|
35123
|
+
scope: args.initFailure.scope
|
|
35124
|
+
});
|
|
35125
|
+
}
|
|
35126
|
+
/**
|
|
34737
35127
|
* Mark a run as failed immediately with a reason message.
|
|
34738
35128
|
*
|
|
34739
35129
|
* Used when no agents are available to dispatch any jobs (e.g. cron-triggered
|
|
@@ -34741,7 +35131,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34741
35131
|
* Instead of leaving the run in 'running' for OrphanRecovery to catch after 5 min,
|
|
34742
35132
|
* this fails it right away.
|
|
34743
35133
|
*/
|
|
34744
|
-
async failRun(runId, reason) {
|
|
35134
|
+
async failRun(runId, reason, initFailure) {
|
|
34745
35135
|
const now = /* @__PURE__ */ new Date();
|
|
34746
35136
|
const run = this.runs.get(runId);
|
|
34747
35137
|
const durationMs = run ? now.getTime() - run.startedAt : 0;
|
|
@@ -34749,7 +35139,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34749
35139
|
status: ExecutionRunStatus.enum.failed,
|
|
34750
35140
|
completed_at: now,
|
|
34751
35141
|
duration_ms: durationMs,
|
|
34752
|
-
failure_reason: reason
|
|
35142
|
+
failure_reason: reason,
|
|
35143
|
+
...initFailure && { init_failure: JSON.stringify(initFailure) }
|
|
34753
35144
|
}).where("run_id", "=", runId).execute();
|
|
34754
35145
|
await this.db.updateTable("execution_jobs").set({
|
|
34755
35146
|
status: ExecutionJobStatus.enum.failed,
|
|
@@ -34773,7 +35164,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
34773
35164
|
parentRunId: run.parentRunId,
|
|
34774
35165
|
originalRunId: run.originalRunId,
|
|
34775
35166
|
triggeredBy: run.triggeredBy
|
|
34776
|
-
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason);
|
|
35167
|
+
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason, void 0, initFailure);
|
|
34777
35168
|
this.runs.delete(runId);
|
|
34778
35169
|
this.jobLogBytes.delete(runId);
|
|
34779
35170
|
}
|
|
@@ -44028,14 +44419,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
44028
44419
|
init_agent_handler();
|
|
44029
44420
|
init_worker_status();
|
|
44030
44421
|
init_agent_heartbeat();
|
|
44031
|
-
ORCHESTRATOR_VERSION$1 = "0.1.
|
|
44032
|
-
WORKER_BUILD_COMMIT = "
|
|
44033
|
-
WORKER_SDK_VERSION = "0.1.
|
|
44422
|
+
ORCHESTRATOR_VERSION$1 = "0.1.14";
|
|
44423
|
+
WORKER_BUILD_COMMIT = "0fc7a43c9";
|
|
44424
|
+
WORKER_SDK_VERSION = "0.1.14";
|
|
44034
44425
|
WORKER_SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
|
|
44035
|
-
WORKER_SHARED_VERSION = "0.1.
|
|
44036
|
-
WORKER_SHARED_BUNDLE_HASH = "
|
|
44037
|
-
WORKER_ENGINE_VERSION = "0.1.
|
|
44038
|
-
WORKER_ENGINE_BUNDLE_HASH = "
|
|
44426
|
+
WORKER_SHARED_VERSION = "0.1.14";
|
|
44427
|
+
WORKER_SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
|
|
44428
|
+
WORKER_ENGINE_VERSION = "0.1.14";
|
|
44429
|
+
WORKER_ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
|
|
44039
44430
|
logger$2 = createLogger({ prefix: "worker" });
|
|
44040
44431
|
DRAIN_TIMEOUT_MS = 3e5;
|
|
44041
44432
|
}));
|
|
@@ -44059,14 +44450,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
44059
44450
|
* Graceful shutdown:
|
|
44060
44451
|
* agent WS -> heartbeat -> HTTP -> DB
|
|
44061
44452
|
*/
|
|
44062
|
-
const ORCHESTRATOR_VERSION = "0.1.
|
|
44063
|
-
const BUILD_COMMIT = "
|
|
44064
|
-
const SDK_VERSION = "0.1.
|
|
44453
|
+
const ORCHESTRATOR_VERSION = "0.1.14";
|
|
44454
|
+
const BUILD_COMMIT = "0fc7a43c9";
|
|
44455
|
+
const SDK_VERSION = "0.1.14";
|
|
44065
44456
|
const SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
|
|
44066
|
-
const SHARED_VERSION = "0.1.
|
|
44067
|
-
const SHARED_BUNDLE_HASH = "
|
|
44068
|
-
const ENGINE_VERSION = "0.1.
|
|
44069
|
-
const ENGINE_BUNDLE_HASH = "
|
|
44457
|
+
const SHARED_VERSION = "0.1.14";
|
|
44458
|
+
const SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
|
|
44459
|
+
const ENGINE_VERSION = "0.1.14";
|
|
44460
|
+
const ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
|
|
44070
44461
|
const otelSdk = initTelemetry({
|
|
44071
44462
|
serviceName: "kici-orchestrator",
|
|
44072
44463
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|