@kici-dev/orchestrator 0.1.12 → 0.1.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/cli/commands/agent-service/install.d.ts +7 -0
  2. package/dist/cli/commands/agent-service/logs.d.ts +8 -3
  3. package/dist/cli/commands/agent-service/restart.d.ts +5 -1
  4. package/dist/cli/commands/agent-service/start.d.ts +5 -1
  5. package/dist/cli/commands/agent-service/status.d.ts +5 -1
  6. package/dist/cli/commands/agent-service/stop.d.ts +5 -1
  7. package/dist/cli/commands/agent-service/uninstall.d.ts +6 -2
  8. package/dist/cli/commands/agent-service/upgrade.d.ts +13 -3
  9. package/dist/cli/commands/orchestrator-service/logs.d.ts +4 -0
  10. package/dist/cli/commands/orchestrator-service/restart.d.ts +5 -1
  11. package/dist/cli/commands/orchestrator-service/start.d.ts +5 -1
  12. package/dist/cli/commands/orchestrator-service/status.d.ts +4 -0
  13. package/dist/cli/commands/orchestrator-service/stop.d.ts +5 -1
  14. package/dist/cli/commands/orchestrator-service/uninstall.d.ts +6 -2
  15. package/dist/cli/commands/orchestrator-service/upgrade.d.ts +9 -0
  16. package/dist/cli/commands/shared/versioned-upgrade.d.ts +54 -2
  17. package/dist/cli/service/compose.d.ts +3 -2
  18. package/dist/cli/service/index.d.ts +7 -1
  19. package/dist/cli/service/instance/index-file.d.ts +30 -0
  20. package/dist/cli/service/instance/manifest.d.ts +30 -0
  21. package/dist/cli/service/instance/resolve.d.ts +59 -0
  22. package/dist/cli/service/instance/types.d.ts +55 -0
  23. package/dist/cli/service/launchd.d.ts +19 -1
  24. package/dist/cli/service/platform-detect.d.ts +29 -9
  25. package/dist/cli/service/systemd.d.ts +2 -1
  26. package/dist/cli/service/types.d.ts +28 -0
  27. package/dist/cli/service/windows.d.ts +2 -1
  28. package/dist/cli.js +1719 -1002
  29. package/dist/db/migrations/025_init_failure.d.ts +16 -0
  30. package/dist/db/types.d.ts +15 -0
  31. package/dist/diagnostics/checks/index.d.ts +2 -1
  32. package/dist/diagnostics/checks/scaler.d.ts +13 -0
  33. package/dist/diagnostics/types.d.ts +5 -2
  34. package/dist/metrics/prometheus.d.ts +32 -4
  35. package/dist/reporting/execution-tracker.d.ts +77 -6
  36. package/dist/scaler/failure-tracker.d.ts +46 -0
  37. package/dist/scaler/manager.d.ts +8 -0
  38. package/dist/server.js +570 -168
  39. package/dist/standalone.js +547 -156
  40. package/package.json +4 -4
  41. package/sbom.spdx.json +91 -36
@@ -9,7 +9,7 @@ import { z } from "zod";
9
9
  import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
10
10
  import WebSocket from "ws";
11
11
  import { chmodSync, closeSync, createReadStream, createWriteStream, mkdirSync, mkdtempSync, openSync, promises, readFileSync, rmSync, statSync, unlinkSync, unwatchFile, watchFile, writeFileSync } from "node:fs";
12
- import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, testEventSchema } from "@kici-dev/engine";
12
+ import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, matchAllWorkflows, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, resolveRoleLabels, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, testEventSchema } from "@kici-dev/engine";
13
13
  import { access, appendFile, chmod, constants, copyFile, link, mkdir, mkdtemp, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
14
14
  import { dirname, join, relative, resolve } from "node:path";
15
15
  import { sql } from "kysely";
@@ -47,6 +47,7 @@ import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCom
47
47
  import { Upload } from "@aws-sdk/lib-storage";
48
48
  import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
49
49
  import stripAnsi from "strip-ansi";
50
+ import { AsyncLocalStorage } from "node:async_hooks";
50
51
  import Vault from "hashi-vault-js";
51
52
  import { Cron } from "croner";
52
53
  var __defProp = Object.defineProperty;
@@ -1040,7 +1041,7 @@ var init_peer_client = __esmMin((() => {
1040
1041
  init_peer_crypto();
1041
1042
  init_peer_credentials();
1042
1043
  logger$86 = createLogger({ prefix: "peer-client" });
1043
- SOFTWARE_VERSION$1 = "0.1.12";
1044
+ SOFTWARE_VERSION$1 = "0.1.14";
1044
1045
  PeerClient$1 = class {
1045
1046
  ws = null;
1046
1047
  _state = "disconnected";
@@ -2506,7 +2507,7 @@ var init_peer_handler = __esmMin((() => {
2506
2507
  init_peer_crypto();
2507
2508
  init_join_token();
2508
2509
  logger$85 = createLogger({ prefix: "peer-handler" });
2509
- SOFTWARE_VERSION = "0.1.12";
2510
+ SOFTWARE_VERSION = "0.1.14";
2510
2511
  RATE_LIMIT_MAX = 5;
2511
2512
  RATE_LIMIT_WINDOW_MS = 6e4;
2512
2513
  }));
@@ -13608,10 +13609,10 @@ var init_admin_sources = __esmMin((() => {
13608
13609
  //#endregion
13609
13610
  //#region src/db/migrations/001_initial.ts
13610
13611
  var _001_initial_exports = /* @__PURE__ */ __exportAll({
13611
- down: () => down$23,
13612
- up: () => up$23
13612
+ down: () => down$24,
13613
+ up: () => up$24
13613
13614
  });
13614
- async function up$23(db) {
13615
+ async function up$24(db) {
13615
13616
  for (const stmt of DDL_STATEMENTS) await sql.raw(stmt).execute(db);
13616
13617
  await sql`
13617
13618
  INSERT INTO cluster_meta (key, value)
@@ -13633,7 +13634,7 @@ async function up$23(db) {
13633
13634
  * Rollback drops everything created above. Uses CASCADE on table drops to cut
13634
13635
  * through the FK graph without relying on exact topological order.
13635
13636
  */
13636
- async function down$23(db) {
13637
+ async function down$24(db) {
13637
13638
  for (const [trig, tbl] of [["source_secrets_change_trigger", "scoped_secrets"], ["sources_change_trigger", "sources"]]) await sql.raw(`DROP TRIGGER IF EXISTS ${trig} ON public.${tbl}`).execute(db);
13638
13639
  for (const table of [
13639
13640
  "workflow_registrations",
@@ -14353,8 +14354,8 @@ var init__001_initial = __esmMin((() => {
14353
14354
  //#endregion
14354
14355
  //#region src/db/migrations/002_config_versions_key_version.ts
14355
14356
  var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
14356
- down: () => down$22,
14357
- up: () => up$22
14357
+ down: () => down$23,
14358
+ up: () => up$23
14358
14359
  });
14359
14360
  /**
14360
14361
  * Add key_version column to config_versions so that sensitive-field encryption
@@ -14368,13 +14369,13 @@ var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
14368
14369
  * No index is needed: rotation does a full-table scan; reads are by
14369
14370
  * `version` primary key and never filter on `key_version`.
14370
14371
  */
14371
- async function up$22(db) {
14372
+ async function up$23(db) {
14372
14373
  await sql`
14373
14374
  ALTER TABLE public.config_versions
14374
14375
  ADD COLUMN key_version integer NOT NULL DEFAULT 1
14375
14376
  `.execute(db);
14376
14377
  }
14377
- async function down$22(db) {
14378
+ async function down$23(db) {
14378
14379
  await sql`
14379
14380
  ALTER TABLE public.config_versions
14380
14381
  DROP COLUMN key_version
@@ -14384,8 +14385,8 @@ var init__002_config_versions_key_version = __esmMin((() => {}));
14384
14385
  //#endregion
14385
14386
  //#region src/db/migrations/003_access_log.ts
14386
14387
  var _003_access_log_exports = /* @__PURE__ */ __exportAll({
14387
- down: () => down$21,
14388
- up: () => up$21
14388
+ down: () => down$22,
14389
+ up: () => up$22
14389
14390
  });
14390
14391
  /**
14391
14392
  * Access log: one row per read or orchestrator-admin mutation attributable
@@ -14409,7 +14410,7 @@ var _003_access_log_exports = /* @__PURE__ */ __exportAll({
14409
14410
  * Retention is TTL-based via expires_at; packages/orchestrator/src/queue/
14410
14411
  * cleanup.ts picks up the prune pass.
14411
14412
  */
14412
- async function up$21(db) {
14413
+ async function up$22(db) {
14413
14414
  await sql`
14414
14415
  CREATE TABLE public.access_log (
14415
14416
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -14447,21 +14448,21 @@ async function up$21(db) {
14447
14448
  ON public.access_log (actor_type, actor_id, created_at DESC)
14448
14449
  `.execute(db);
14449
14450
  }
14450
- async function down$21(db) {
14451
+ async function down$22(db) {
14451
14452
  await sql`DROP TABLE IF EXISTS public.access_log`.execute(db);
14452
14453
  }
14453
14454
  var init__003_access_log = __esmMin((() => {}));
14454
14455
  //#endregion
14455
14456
  //#region src/db/migrations/004_rename_bundle_to_source.ts
14456
14457
  var _004_rename_bundle_to_source_exports = /* @__PURE__ */ __exportAll({
14457
- down: () => down$20,
14458
- up: () => up$20
14458
+ down: () => down$21,
14459
+ up: () => up$21
14459
14460
  });
14460
- async function up$20(db) {
14461
+ async function up$21(db) {
14461
14462
  await db.schema.alterTable("dispatch_queue").renameColumn("bundle_url", "source_tar_url").execute();
14462
14463
  await db.schema.alterTable("dispatch_queue").renameColumn("bundle_hash", "source_tar_hash").execute();
14463
14464
  }
14464
- async function down$20(db) {
14465
+ async function down$21(db) {
14465
14466
  await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_url", "bundle_url").execute();
14466
14467
  await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_hash", "bundle_hash").execute();
14467
14468
  }
@@ -14469,8 +14470,8 @@ var init__004_rename_bundle_to_source = __esmMin((() => {}));
14469
14470
  //#endregion
14470
14471
  //#region src/db/migrations/005_cold_store_chunk_counter.ts
14471
14472
  var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
14472
- down: () => down$19,
14473
- up: () => up$19
14473
+ down: () => down$20,
14474
+ up: () => up$20
14474
14475
  });
14475
14476
  /**
14476
14477
  * Cold-store chunk counter table.
@@ -14487,7 +14488,7 @@ var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
14487
14488
  *
14488
14489
  * sections 5 and 8.
14489
14490
  */
14490
- async function up$19(db) {
14491
+ async function up$20(db) {
14491
14492
  await sql`
14492
14493
  CREATE TABLE public.cold_store_chunk_counts (
14493
14494
  db TEXT NOT NULL,
@@ -14505,15 +14506,15 @@ async function up$19(db) {
14505
14506
  ON public.cold_store_chunk_counts (db, table_name)
14506
14507
  `.execute(db);
14507
14508
  }
14508
- async function down$19(db) {
14509
+ async function down$20(db) {
14509
14510
  await sql`DROP TABLE IF EXISTS public.cold_store_chunk_counts`.execute(db);
14510
14511
  }
14511
14512
  var init__005_cold_store_chunk_counter = __esmMin((() => {}));
14512
14513
  //#endregion
14513
14514
  //#region src/db/migrations/006_runs_jobs_steps_archived_at.ts
14514
14515
  var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
14515
- down: () => down$18,
14516
- up: () => up$18
14516
+ down: () => down$19,
14517
+ up: () => up$19
14517
14518
  });
14518
14519
  /**
14519
14520
  * `execution_runs` / `execution_jobs` / `execution_steps` cold-store
@@ -14549,7 +14550,7 @@ var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
14549
14550
  * - `idx_execution_jobs_routing_key_created (routing_key, created_at)`
14550
14551
  * - `idx_execution_steps_routing_key_created (routing_key, created_at)`
14551
14552
  */
14552
- async function up$18(db) {
14553
+ async function up$19(db) {
14553
14554
  await sql`
14554
14555
  ALTER TABLE public.execution_runs
14555
14556
  ADD COLUMN archived_at TIMESTAMPTZ NULL,
@@ -14594,7 +14595,7 @@ async function up$18(db) {
14594
14595
  ON public.execution_steps (routing_key, created_at)
14595
14596
  `.execute(db);
14596
14597
  }
14597
- async function down$18(db) {
14598
+ async function down$19(db) {
14598
14599
  await sql`DROP INDEX IF EXISTS public.idx_execution_steps_routing_key_created`.execute(db);
14599
14600
  await sql`
14600
14601
  ALTER TABLE public.execution_steps
@@ -14620,8 +14621,8 @@ var init__006_runs_jobs_steps_archived_at = __esmMin((() => {}));
14620
14621
  //#endregion
14621
14622
  //#region src/db/migrations/007_audit_logs_archived_at.ts
14622
14623
  var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
14623
- down: () => down$17,
14624
- up: () => up$17
14624
+ down: () => down$18,
14625
+ up: () => up$18
14625
14626
  });
14626
14627
  /**
14627
14628
  * `secret_audit_log` and `access_log` cold-store schema additions, plus
@@ -14661,7 +14662,7 @@ var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
14661
14662
  * `down()` would not have meaningful retention bounds. Acceptable for
14662
14663
  * staging.
14663
14664
  */
14664
- async function up$17(db) {
14665
+ async function up$18(db) {
14665
14666
  await sql`
14666
14667
  ALTER TABLE public.secret_audit_log
14667
14668
  ADD COLUMN archived_at TIMESTAMPTZ NULL,
@@ -14682,7 +14683,7 @@ async function up$17(db) {
14682
14683
  DROP COLUMN IF EXISTS expires_at
14683
14684
  `.execute(db);
14684
14685
  }
14685
- async function down$17(db) {
14686
+ async function down$18(db) {
14686
14687
  await sql`
14687
14688
  ALTER TABLE public.access_log
14688
14689
  ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '90 days')
@@ -14711,8 +14712,8 @@ var init__007_audit_logs_archived_at = __esmMin((() => {}));
14711
14712
  //#endregion
14712
14713
  //#region src/db/migrations/008_event_log_archived_at.ts
14713
14714
  var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
14714
- down: () => down$16,
14715
- up: () => up$16
14715
+ down: () => down$17,
14716
+ up: () => up$17
14716
14717
  });
14717
14718
  /**
14718
14719
  * `event_log` cold-store schema additions plus removal of the
@@ -14750,7 +14751,7 @@ var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
14750
14751
  * — best effort; rows inserted between `up()` and a hypothetical
14751
14752
  * `down()` would not have meaningful retention bounds.
14752
14753
  */
14753
- async function up$16(db) {
14754
+ async function up$17(db) {
14754
14755
  await sql`
14755
14756
  ALTER TABLE public.event_log
14756
14757
  ADD COLUMN archived_at TIMESTAMPTZ NULL,
@@ -14766,7 +14767,7 @@ async function up$16(db) {
14766
14767
  DROP COLUMN IF EXISTS expires_at
14767
14768
  `.execute(db);
14768
14769
  }
14769
- async function down$16(db) {
14770
+ async function down$17(db) {
14770
14771
  await sql`
14771
14772
  ALTER TABLE public.event_log
14772
14773
  ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '30 days')
@@ -14786,8 +14787,8 @@ var init__008_event_log_archived_at = __esmMin((() => {}));
14786
14787
  //#endregion
14787
14788
  //#region src/db/migrations/009_access_log_trigram.ts
14788
14789
  var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
14789
- down: () => down$15,
14790
- up: () => up$15
14790
+ down: () => down$16,
14791
+ up: () => up$16
14791
14792
  });
14792
14793
  /**
14793
14794
  * Trigram (pg_trgm) index on access_log.error_message for the federated
@@ -14800,7 +14801,7 @@ var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
14800
14801
  * EXISTS` are both safe to re-run. No CONCURRENTLY because Kysely runs
14801
14802
  * migrations inside a transaction; the lock is brief on a sampled table.
14802
14803
  */
14803
- async function up$15(db) {
14804
+ async function up$16(db) {
14804
14805
  await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
14805
14806
  await sql`
14806
14807
  CREATE INDEX IF NOT EXISTS access_log_error_message_trgm_idx
@@ -14809,15 +14810,15 @@ async function up$15(db) {
14809
14810
  WHERE error_message IS NOT NULL
14810
14811
  `.execute(db);
14811
14812
  }
14812
- async function down$15(db) {
14813
+ async function down$16(db) {
14813
14814
  await sql`DROP INDEX IF EXISTS public.access_log_error_message_trgm_idx`.execute(db);
14814
14815
  }
14815
14816
  var init__009_access_log_trigram = __esmMin((() => {}));
14816
14817
  //#endregion
14817
14818
  //#region src/db/migrations/010_cold_store_chunks.ts
14818
14819
  var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
14819
- down: () => down$14,
14820
- up: () => up$14
14820
+ down: () => down$15,
14821
+ up: () => up$15
14821
14822
  });
14822
14823
  /**
14823
14824
  * Cold-store chunk index — Phase 2 (cold-store purge).
@@ -14849,7 +14850,7 @@ var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
14849
14850
  * forever. Adapters that don't opt into per-bucket archival via
14850
14851
  * `coldTtlDays` don't insert here either.
14851
14852
  */
14852
- async function up$14(db) {
14853
+ async function up$15(db) {
14853
14854
  await sql`
14854
14855
  CREATE TABLE public.cold_store_chunks (
14855
14856
  db TEXT NOT NULL,
@@ -14876,15 +14877,15 @@ async function up$14(db) {
14876
14877
  ON public.cold_store_chunks (db, table_name, tenant_id, archived_at DESC)
14877
14878
  `.execute(db);
14878
14879
  }
14879
- async function down$14(db) {
14880
+ async function down$15(db) {
14880
14881
  await sql`DROP TABLE IF EXISTS public.cold_store_chunks`.execute(db);
14881
14882
  }
14882
14883
  var init__010_cold_store_chunks = __esmMin((() => {}));
14883
14884
  //#endregion
14884
14885
  //#region src/db/migrations/011_drop_source_secrets_notify.ts
14885
14886
  var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
14886
- down: () => down$13,
14887
- up: () => up$13
14887
+ down: () => down$14,
14888
+ up: () => up$14
14888
14889
  });
14889
14890
  /**
14890
14891
  * Drop the `source_secrets_change_trigger` and the
@@ -14903,11 +14904,11 @@ var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
14903
14904
  * in `001_initial.ts`. They wake up no consumer until the `WebhookSecretManager`
14904
14905
  * is restored, so this migration is safe to roll back.
14905
14906
  */
14906
- async function up$13(db) {
14907
+ async function up$14(db) {
14907
14908
  await sql`DROP TRIGGER IF EXISTS source_secrets_change_trigger ON public.scoped_secrets`.execute(db);
14908
14909
  await sql`DROP FUNCTION IF EXISTS public.notify_source_secrets_change() CASCADE`.execute(db);
14909
14910
  }
14910
- async function down$13(db) {
14911
+ async function down$14(db) {
14911
14912
  await sql`
14912
14913
  CREATE OR REPLACE FUNCTION public.notify_source_secrets_change() RETURNS trigger
14913
14914
  LANGUAGE plpgsql
@@ -14947,8 +14948,8 @@ var init__011_drop_source_secrets_notify = __esmMin((() => {}));
14947
14948
  //#endregion
14948
14949
  //#region src/db/migrations/012_peer_credentials_active_uniq.ts
14949
14950
  var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
14950
- down: () => down$12,
14951
- up: () => up$12
14951
+ down: () => down$13,
14952
+ up: () => up$13
14952
14953
  });
14953
14954
  /**
14954
14955
  * Add a partial unique index on `peer_credentials (instance_id) WHERE
@@ -14974,7 +14975,7 @@ var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
14974
14975
  * `down()` only drops the index; it does NOT undo the dedupe (there's no
14975
14976
  * safe way to recreate revoked rows, and the dedupe is monotonic).
14976
14977
  */
14977
- async function up$12(db) {
14978
+ async function up$13(db) {
14978
14979
  await sql`
14979
14980
  UPDATE public.peer_credentials
14980
14981
  SET revoked_at = NOW()
@@ -14992,15 +14993,15 @@ async function up$12(db) {
14992
14993
  WHERE revoked_at IS NULL
14993
14994
  `.execute(db);
14994
14995
  }
14995
- async function down$12(db) {
14996
+ async function down$13(db) {
14996
14997
  await sql`DROP INDEX IF EXISTS public.peer_credentials_active_uniq`.execute(db);
14997
14998
  }
14998
14999
  var init__012_peer_credentials_active_uniq = __esmMin((() => {}));
14999
15000
  //#endregion
15000
15001
  //#region src/db/migrations/013_execution_log_bytes.ts
15001
15002
  var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
15002
- down: () => down$11,
15003
- up: () => up$11
15003
+ down: () => down$12,
15004
+ up: () => up$12
15004
15005
  });
15005
15006
  /**
15006
15007
  * Add `log_bytes BIGINT NOT NULL DEFAULT 0` columns to `execution_runs` and
@@ -15019,7 +15020,7 @@ var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
15019
15020
  *
15020
15021
  * Idempotent (`ADD COLUMN IF NOT EXISTS`).
15021
15022
  */
15022
- async function up$11(db) {
15023
+ async function up$12(db) {
15023
15024
  await sql`
15024
15025
  ALTER TABLE public.execution_runs
15025
15026
  ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
@@ -15029,7 +15030,7 @@ async function up$11(db) {
15029
15030
  ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
15030
15031
  `.execute(db);
15031
15032
  }
15032
- async function down$11(db) {
15033
+ async function down$12(db) {
15033
15034
  await sql`ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS log_bytes`.execute(db);
15034
15035
  await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS log_bytes`.execute(db);
15035
15036
  }
@@ -15037,8 +15038,8 @@ var init__013_execution_log_bytes = __esmMin((() => {}));
15037
15038
  //#endregion
15038
15039
  //#region src/db/migrations/014_kici_events_lease_retry.ts
15039
15040
  var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
15040
- down: () => down$10,
15041
- up: () => up$10
15041
+ down: () => down$11,
15042
+ up: () => up$11
15042
15043
  });
15043
15044
  /**
15044
15045
  * Add lease + retry + DLQ columns to `kici_events` so the EventRouter can
@@ -15072,7 +15073,7 @@ var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
15072
15073
  *
15073
15074
  * Idempotent (`ADD COLUMN IF NOT EXISTS` + `CREATE INDEX IF NOT EXISTS`).
15074
15075
  */
15075
- async function up$10(db) {
15076
+ async function up$11(db) {
15076
15077
  await sql`
15077
15078
  ALTER TABLE public.kici_events
15078
15079
  ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ,
@@ -15103,7 +15104,7 @@ async function up$10(db) {
15103
15104
  WHERE dlq_at IS NOT NULL
15104
15105
  `.execute(db);
15105
15106
  }
15106
- async function down$10(db) {
15107
+ async function down$11(db) {
15107
15108
  await sql`DROP INDEX IF EXISTS public.idx_kici_events_dlq`.execute(db);
15108
15109
  await sql`DROP INDEX IF EXISTS public.idx_kici_events_lease_expired`.execute(db);
15109
15110
  await sql`DROP INDEX IF EXISTS public.idx_kici_events_retry_due`.execute(db);
@@ -15122,8 +15123,8 @@ var init__014_kici_events_lease_retry = __esmMin((() => {}));
15122
15123
  //#endregion
15123
15124
  //#region src/db/migrations/015_org_settings_customer_scoped.ts
15124
15125
  var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
15125
- down: () => down$9,
15126
- up: () => up$9
15126
+ down: () => down$10,
15127
+ up: () => up$10
15127
15128
  });
15128
15129
  /**
15129
15130
  * Org-scope `org_settings` and qualify each glob entry by source.
@@ -15151,7 +15152,7 @@ var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
15151
15152
  * Idempotent: a re-run on an already-migrated DB sees `customer_id` exists
15152
15153
  * and the list columns are already jsonb, so it is a no-op.
15153
15154
  */
15154
- async function up$9(db) {
15155
+ async function up$10(db) {
15155
15156
  if ((await sql`
15156
15157
  SELECT EXISTS (
15157
15158
  SELECT 1 FROM information_schema.columns
@@ -15276,7 +15277,7 @@ async function up$9(db) {
15276
15277
  await sql`DROP TABLE _org_settings_merged`.execute(db);
15277
15278
  await sql`DROP TABLE _org_settings_stage`.execute(db);
15278
15279
  }
15279
- async function down$9(db) {
15280
+ async function down$10(db) {
15280
15281
  if (!(await sql`
15281
15282
  SELECT EXISTS (
15282
15283
  SELECT 1 FROM information_schema.columns
@@ -15302,8 +15303,8 @@ var init__015_org_settings_customer_scoped = __esmMin((() => {}));
15302
15303
  //#endregion
15303
15304
  //#region src/db/migrations/016_org_settings_allow_http_npm.ts
15304
15305
  var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
15305
- down: () => down$8,
15306
- up: () => up$8
15306
+ down: () => down$9,
15307
+ up: () => up$9
15307
15308
  });
15308
15309
  /**
15309
15310
  * Add `org_settings.allow_http_npm_registries boolean NOT NULL DEFAULT false`.
@@ -15316,7 +15317,7 @@ var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
15316
15317
  *
15317
15318
  * Idempotent: a re-run on a DB that already has the column is a no-op.
15318
15319
  */
15319
- async function up$8(db) {
15320
+ async function up$9(db) {
15320
15321
  if ((await sql`
15321
15322
  SELECT EXISTS (
15322
15323
  SELECT 1 FROM information_schema.columns
@@ -15330,7 +15331,7 @@ async function up$8(db) {
15330
15331
  ADD COLUMN allow_http_npm_registries boolean NOT NULL DEFAULT false
15331
15332
  `.execute(db);
15332
15333
  }
15333
- async function down$8(db) {
15334
+ async function down$9(db) {
15334
15335
  await sql`
15335
15336
  ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_http_npm_registries
15336
15337
  `.execute(db);
@@ -15339,13 +15340,13 @@ var init__016_org_settings_allow_http_npm = __esmMin((() => {}));
15339
15340
  //#endregion
15340
15341
  //#region src/db/migrations/017_org_id_widen.ts
15341
15342
  var _017_org_id_widen_exports = /* @__PURE__ */ __exportAll({
15342
- down: () => down$7,
15343
- up: () => up$7
15343
+ down: () => down$8,
15344
+ up: () => up$8
15344
15345
  });
15345
- async function up$7(db) {
15346
+ async function up$8(db) {
15346
15347
  for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(16)`).execute(db);
15347
15348
  }
15348
- async function down$7(db) {
15349
+ async function down$8(db) {
15349
15350
  for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(12)`).execute(db);
15350
15351
  }
15351
15352
  var ORG_ID_TABLES$1;
@@ -15364,14 +15365,14 @@ var init__017_org_id_widen = __esmMin((() => {
15364
15365
  //#endregion
15365
15366
  //#region src/db/migrations/018_org_id_prefix_backfill.ts
15366
15367
  var _018_org_id_prefix_backfill_exports = /* @__PURE__ */ __exportAll({
15367
- down: () => down$6,
15368
- up: () => up$6
15368
+ down: () => down$7,
15369
+ up: () => up$7
15369
15370
  });
15370
- async function up$6(db) {
15371
+ async function up$7(db) {
15371
15372
  for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = 'org_' || org_id WHERE org_id <> 'kici-admin' AND org_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
15372
15373
  for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = 'org_' || customer_id WHERE customer_id <> 'kici-admin' AND customer_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
15373
15374
  }
15374
- async function down$6(db) {
15375
+ async function down$7(db) {
15375
15376
  for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = substring(customer_id from 5) WHERE customer_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
15376
15377
  for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = substring(org_id from 5) WHERE org_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
15377
15378
  }
@@ -15397,8 +15398,8 @@ var init__018_org_id_prefix_backfill = __esmMin((() => {
15397
15398
  //#endregion
15398
15399
  //#region src/db/migrations/019_generic_sources_change_notify.ts
15399
15400
  var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
15400
- down: () => down$5,
15401
- up: () => up$5
15401
+ down: () => down$6,
15402
+ up: () => up$6
15402
15403
  });
15403
15404
  /**
15404
15405
  * Add a Postgres trigger on `generic_webhook_sources` that emits
@@ -15421,7 +15422,7 @@ var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
15421
15422
  * (`notify_sources_change()` + `sources_change_trigger`, defined in
15422
15423
  * `001_initial.ts`).
15423
15424
  */
15424
- async function up$5(db) {
15425
+ async function up$6(db) {
15425
15426
  await sql`
15426
15427
  CREATE FUNCTION public.notify_generic_sources_change() RETURNS trigger
15427
15428
  LANGUAGE plpgsql
@@ -15442,7 +15443,7 @@ async function up$5(db) {
15442
15443
  FOR EACH ROW EXECUTE FUNCTION public.notify_generic_sources_change()
15443
15444
  `.execute(db);
15444
15445
  }
15445
- async function down$5(db) {
15446
+ async function down$6(db) {
15446
15447
  await sql`DROP TRIGGER IF EXISTS generic_sources_change_trigger ON public.generic_webhook_sources`.execute(db);
15447
15448
  await sql`DROP FUNCTION IF EXISTS public.notify_generic_sources_change()`.execute(db);
15448
15449
  }
@@ -15450,8 +15451,8 @@ var init__019_generic_sources_change_notify = __esmMin((() => {}));
15450
15451
  //#endregion
15451
15452
  //#region src/db/migrations/020_org_settings_dashboard_write_policy.ts
15452
15453
  var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportAll({
15453
- down: () => down$4,
15454
- up: () => up$4
15454
+ down: () => down$5,
15455
+ up: () => up$5
15455
15456
  });
15456
15457
  /**
15457
15458
  * Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
@@ -15466,7 +15467,7 @@ var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportA
15466
15467
  *
15467
15468
  * Idempotent: a re-run on a DB that already has the column is a no-op.
15468
15469
  */
15469
- async function up$4(db) {
15470
+ async function up$5(db) {
15470
15471
  if ((await sql`
15471
15472
  SELECT EXISTS (
15472
15473
  SELECT 1 FROM information_schema.columns
@@ -15480,7 +15481,7 @@ async function up$4(db) {
15480
15481
  ADD COLUMN dashboard_write_policy jsonb NOT NULL DEFAULT '{}'::jsonb
15481
15482
  `.execute(db);
15482
15483
  }
15483
- async function down$4(db) {
15484
+ async function down$5(db) {
15484
15485
  await sql`
15485
15486
  ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dashboard_write_policy
15486
15487
  `.execute(db);
@@ -15489,8 +15490,8 @@ var init__020_org_settings_dashboard_write_policy = __esmMin((() => {}));
15489
15490
  //#endregion
15490
15491
  //#region src/db/migrations/021_check_run_tracking.ts
15491
15492
  var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
15492
- down: () => down$3,
15493
- up: () => up$3
15493
+ down: () => down$4,
15494
+ up: () => up$4
15494
15495
  });
15495
15496
  /**
15496
15497
  * Add `check_run_tracking` table for HA-safe check-run state persistence.
@@ -15514,7 +15515,7 @@ var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
15514
15515
  *
15515
15516
  * Idempotent: a re-run on a DB that already has the table is a no-op.
15516
15517
  */
15517
- async function up$3(db) {
15518
+ async function up$4(db) {
15518
15519
  if ((await sql`
15519
15520
  SELECT EXISTS (
15520
15521
  SELECT 1 FROM information_schema.tables
@@ -15545,15 +15546,15 @@ async function up$3(db) {
15545
15546
  WHERE run_id IS NOT NULL
15546
15547
  `.execute(db);
15547
15548
  }
15548
- async function down$3(db) {
15549
+ async function down$4(db) {
15549
15550
  await sql`DROP TABLE IF EXISTS public.check_run_tracking`.execute(db);
15550
15551
  }
15551
15552
  var init__021_check_run_tracking = __esmMin((() => {}));
15552
15553
  //#endregion
15553
15554
  //#region src/db/migrations/022_scaler_manager_state.ts
15554
15555
  var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
15555
- down: () => down$2,
15556
- up: () => up$2
15556
+ down: () => down$3,
15557
+ up: () => up$3
15557
15558
  });
15558
15559
  /**
15559
15560
  * Add three tables persisting `ScalerManager` per-coord state:
@@ -15580,7 +15581,7 @@ var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
15580
15581
  * Idempotent: a re-run on a DB that already has any of these tables
15581
15582
  * leaves the existing one alone.
15582
15583
  */
15583
- async function up$2(db) {
15584
+ async function up$3(db) {
15584
15585
  const tableExists = async (name) => {
15585
15586
  return (await sql`
15586
15587
  SELECT EXISTS (
@@ -15630,7 +15631,7 @@ async function up$2(db) {
15630
15631
  `.execute(db);
15631
15632
  }
15632
15633
  }
15633
- async function down$2(db) {
15634
+ async function down$3(db) {
15634
15635
  await sql`DROP TABLE IF EXISTS public.scaler_reservations`.execute(db);
15635
15636
  await sql`DROP TABLE IF EXISTS public.scaler_agent_jobs`.execute(db);
15636
15637
  await sql`DROP TABLE IF EXISTS public.scaler_spawning_agents`.execute(db);
@@ -15639,8 +15640,8 @@ var init__022_scaler_manager_state = __esmMin((() => {}));
15639
15640
  //#endregion
15640
15641
  //#region src/db/migrations/023_dispatch_queue_recovery_deadline.ts
15641
15642
  var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll({
15642
- down: () => down$1,
15643
- up: () => up$1
15643
+ down: () => down$2,
15644
+ up: () => up$2
15644
15645
  });
15645
15646
  /**
15646
15647
  * Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
@@ -15664,7 +15665,7 @@ var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll(
15664
15665
  * Idempotent: re-running on a DB that already has either column is a
15665
15666
  * no-op.
15666
15667
  */
15667
- async function up$1(db) {
15668
+ async function up$2(db) {
15668
15669
  const colExists = async (name) => {
15669
15670
  return (await sql`
15670
15671
  SELECT EXISTS (
@@ -15689,7 +15690,7 @@ async function up$1(db) {
15689
15690
  WHERE recovery_deadline IS NOT NULL
15690
15691
  `.execute(db);
15691
15692
  }
15692
- async function down$1(db) {
15693
+ async function down$2(db) {
15693
15694
  await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_recovery_deadline`.execute(db);
15694
15695
  await sql`
15695
15696
  ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS recovery_agent_id
@@ -15702,8 +15703,8 @@ var init__023_dispatch_queue_recovery_deadline = __esmMin((() => {}));
15702
15703
  //#endregion
15703
15704
  //#region src/db/migrations/024_dispatch_queue_provisioning_error.ts
15704
15705
  var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll({
15705
- down: () => down,
15706
- up: () => up
15706
+ down: () => down$1,
15707
+ up: () => up$1
15707
15708
  });
15708
15709
  /**
15709
15710
  * Add `dispatch_queue.last_provisioning_error TEXT` recording the most
@@ -15719,7 +15720,7 @@ var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll
15719
15720
  *
15720
15721
  * Idempotent: re-running on a DB that already has the column is a no-op.
15721
15722
  */
15722
- async function up(db) {
15723
+ async function up$1(db) {
15723
15724
  const colExists = async (name) => {
15724
15725
  return (await sql`
15725
15726
  SELECT EXISTS (
@@ -15735,13 +15736,60 @@ async function up(db) {
15735
15736
  ADD COLUMN last_provisioning_error TEXT
15736
15737
  `.execute(db);
15737
15738
  }
15738
- async function down(db) {
15739
+ async function down$1(db) {
15739
15740
  await sql`
15740
15741
  ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS last_provisioning_error
15741
15742
  `.execute(db);
15742
15743
  }
15743
15744
  var init__024_dispatch_queue_provisioning_error = __esmMin((() => {}));
15744
15745
  //#endregion
15746
+ //#region src/db/migrations/025_init_failure.ts
15747
+ var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
15748
+ down: () => down,
15749
+ up: () => up
15750
+ });
15751
+ /**
15752
+ * Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
15753
+ *
15754
+ * Presence of this column on a row means the run/job never executed a step
15755
+ * because of an init-phase failure; absence (NULL) means a normal run.
15756
+ * Shape on the wire is `InitFailure` from `@kici-dev/engine`. The dashboard
15757
+ * reads this column directly so it can render the right banner without
15758
+ * round-tripping to the orchestrator (which may be offline).
15759
+ *
15760
+ * Idempotent: re-running on a DB that already has either column is a no-op
15761
+ * for that column.
15762
+ */
15763
+ async function up(db) {
15764
+ const colExists = async (table, name) => {
15765
+ return (await sql`
15766
+ SELECT EXISTS (
15767
+ SELECT 1 FROM information_schema.columns
15768
+ WHERE table_schema = 'public'
15769
+ AND table_name = ${table}
15770
+ AND column_name = ${name}
15771
+ ) AS exists
15772
+ `.execute(db)).rows[0]?.exists ?? false;
15773
+ };
15774
+ if (!await colExists("execution_runs", "init_failure")) await sql`
15775
+ ALTER TABLE public.execution_runs
15776
+ ADD COLUMN init_failure JSONB DEFAULT NULL
15777
+ `.execute(db);
15778
+ if (!await colExists("execution_jobs", "init_failure")) await sql`
15779
+ ALTER TABLE public.execution_jobs
15780
+ ADD COLUMN init_failure JSONB DEFAULT NULL
15781
+ `.execute(db);
15782
+ }
15783
+ async function down(db) {
15784
+ await sql`
15785
+ ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS init_failure
15786
+ `.execute(db);
15787
+ await sql`
15788
+ ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS init_failure
15789
+ `.execute(db);
15790
+ }
15791
+ var init__025_init_failure = __esmMin((() => {}));
15792
+ //#endregion
15745
15793
  //#region src/db/migration-provider.ts
15746
15794
  function createMigrationProvider() {
15747
15795
  return { async getMigrations() {
@@ -15769,7 +15817,8 @@ function createMigrationProvider() {
15769
15817
  "021_check_run_tracking": _021_check_run_tracking_exports,
15770
15818
  "022_scaler_manager_state": _022_scaler_manager_state_exports,
15771
15819
  "023_dispatch_queue_recovery_deadline": _023_dispatch_queue_recovery_deadline_exports,
15772
- "024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports
15820
+ "024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports,
15821
+ "025_init_failure": _025_init_failure_exports
15773
15822
  };
15774
15823
  } };
15775
15824
  }
@@ -15798,6 +15847,7 @@ var init_migration_provider = __esmMin((() => {
15798
15847
  init__022_scaler_manager_state();
15799
15848
  init__023_dispatch_queue_recovery_deadline();
15800
15849
  init__024_dispatch_queue_provisioning_error();
15850
+ init__025_init_failure();
15801
15851
  }));
15802
15852
  //#endregion
15803
15853
  //#region src/db/migrator.ts
@@ -21704,15 +21754,15 @@ var init_admin_config = __esmMin((() => {
21704
21754
  function createHealthRoutes$1(deps = {}) {
21705
21755
  return createHealthRoutes({
21706
21756
  livenessInfo: () => ({
21707
- version: "0.1.12",
21708
- buildDate: "2026-05-27T16:46:38.133Z",
21709
- buildCommit: "d220ac45c",
21710
- sdkVersion: "0.1.12",
21757
+ version: "0.1.14",
21758
+ buildDate: "2026-05-30T03:04:39.672Z",
21759
+ buildCommit: "0fc7a43c9",
21760
+ sdkVersion: "0.1.14",
21711
21761
  sdkBundleHash: "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128",
21712
- sharedVersion: "0.1.12",
21713
- sharedBundleHash: "36a23a454fd75ec35f258d39f254543ec0451c6b543fe3618ce427e6e6369aae",
21714
- engineVersion: "0.1.12",
21715
- engineBundleHash: "4ab08e63bcb8558b1191c5d3ab8d58b71950b77d911af56c9955ae1d787c4302"
21762
+ sharedVersion: "0.1.14",
21763
+ sharedBundleHash: "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4",
21764
+ engineVersion: "0.1.14",
21765
+ engineBundleHash: "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932"
21716
21766
  }),
21717
21767
  readinessCheck: deps.db ? async () => {
21718
21768
  const checks = {};
@@ -21746,7 +21796,7 @@ function createCapabilitiesRoutes() {
21746
21796
  const app = new Hono();
21747
21797
  app.get("/api/v1/capabilities", (c) => {
21748
21798
  const manifest = {
21749
- orchestratorVersion: "0.1.12",
21799
+ orchestratorVersion: "0.1.14",
21750
21800
  protocolVersion: PROTOCOL_VERSION,
21751
21801
  minProtocolVersion: MIN_PROTOCOL_VERSION
21752
21802
  };
@@ -22048,6 +22098,61 @@ var init_certs = __esmMin((() => {
22048
22098
  WARN_DAYS = 30;
22049
22099
  }));
22050
22100
  //#endregion
22101
+ //#region src/diagnostics/checks/scaler.ts
22102
+ function truncate(s) {
22103
+ return s.length > MAX_DETAIL_LEN ? `${s.slice(0, MAX_DETAIL_LEN - 1)}…` : s;
22104
+ }
22105
+ async function checkScalerProvisioning(deps) {
22106
+ const start = Date.now();
22107
+ const mgr = deps.scalerManager;
22108
+ const backends = mgr?.getStatus().backends ?? [];
22109
+ if (!mgr || backends.length === 0) return [{
22110
+ name: "scaler",
22111
+ status: "pass",
22112
+ message: "No scaler backends configured",
22113
+ durationMs: Date.now() - start
22114
+ }];
22115
+ const recent = mgr.recentSpawnFailures(SCALER_FAILURE_WINDOW_MS, Date.now());
22116
+ return backends.map((backend) => {
22117
+ const summary = recent.get(backend.name);
22118
+ const durationMs = Date.now() - start;
22119
+ if (!summary || summary.boundCount + summary.unboundCount === 0) return {
22120
+ name: `scaler:${backend.name}`,
22121
+ status: "pass",
22122
+ message: "0 spawn failures in last 5m",
22123
+ details: {
22124
+ windowMs: SCALER_FAILURE_WINDOW_MS,
22125
+ backendType: backend.type,
22126
+ boundCount: 0,
22127
+ unboundCount: 0
22128
+ },
22129
+ durationMs
22130
+ };
22131
+ const total = summary.boundCount + summary.unboundCount;
22132
+ const status = summary.boundCount > 0 ? "fail" : "warn";
22133
+ const message = `${total} spawn failures in last 5m (${summary.boundCount} bound, ${summary.unboundCount} warm-pool; last: ${truncate(summary.lastError)})`;
22134
+ return {
22135
+ name: `scaler:${backend.name}`,
22136
+ status,
22137
+ message,
22138
+ details: {
22139
+ windowMs: SCALER_FAILURE_WINDOW_MS,
22140
+ backendType: summary.backendType,
22141
+ boundCount: summary.boundCount,
22142
+ unboundCount: summary.unboundCount,
22143
+ lastError: summary.lastError,
22144
+ lastAtMs: summary.lastAtMs
22145
+ },
22146
+ durationMs
22147
+ };
22148
+ });
22149
+ }
22150
+ var SCALER_FAILURE_WINDOW_MS, MAX_DETAIL_LEN;
22151
+ var init_scaler$1 = __esmMin((() => {
22152
+ SCALER_FAILURE_WINDOW_MS = 300 * 1e3;
22153
+ MAX_DETAIL_LEN = 120;
22154
+ }));
22155
+ //#endregion
22051
22156
  //#region src/diagnostics/checks/index.ts
22052
22157
  var defaultChecks;
22053
22158
  var init_checks = __esmMin((() => {
@@ -22057,13 +22162,15 @@ var init_checks = __esmMin((() => {
22057
22162
  init_disk();
22058
22163
  init_config$2();
22059
22164
  init_certs();
22165
+ init_scaler$1();
22060
22166
  defaultChecks = [
22061
22167
  checkDbConnectivity,
22062
22168
  checkWsToPlatform,
22063
22169
  checkAgentConnectivity,
22064
22170
  checkDiskSpace,
22065
22171
  checkConfigValidity,
22066
- checkCertificateExpiry
22172
+ checkCertificateExpiry,
22173
+ checkScalerProvisioning
22067
22174
  ];
22068
22175
  }));
22069
22176
  //#endregion
@@ -22079,14 +22186,14 @@ var init_checks = __esmMin((() => {
22079
22186
  async function runDiagnostics(deps, checks, options) {
22080
22187
  const checksToRun = checks ?? defaultChecks;
22081
22188
  const timeoutMs = options?.timeoutMs ?? 5e3;
22082
- return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).map((result, index) => {
22083
- if (result.status === "fulfilled") return result.value;
22084
- return {
22189
+ return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).flatMap((result, index) => {
22190
+ if (result.status === "fulfilled") return Array.isArray(result.value) ? result.value : [result.value];
22191
+ return [{
22085
22192
  name: `Check ${index + 1}`,
22086
22193
  status: "fail",
22087
22194
  message: `Unexpected error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
22088
22195
  durationMs: 0
22089
- };
22196
+ }];
22090
22197
  });
22091
22198
  }
22092
22199
  /**
@@ -23477,7 +23584,11 @@ async function runBuildJob(args) {
23477
23584
  jobName: buildJobName,
23478
23585
  runsOnLabels: buildJobInput.runsOnLabels
23479
23586
  }], setup.info.routingKey, void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
23480
- await deps.executionTracker.failRun(runId, reason);
23587
+ await deps.executionTracker.failRun(runId, reason, {
23588
+ scope: "run",
23589
+ category: InitFailureCategory.enum.build_coordination,
23590
+ message: reason
23591
+ });
23481
23592
  }
23482
23593
  return;
23483
23594
  }
@@ -23530,8 +23641,14 @@ async function recordBuildFailure(args) {
23530
23641
  });
23531
23642
  }
23532
23643
  if (deps.executionTracker) try {
23533
- if (buildJobTrackedEarly) await deps.executionTracker.onBuildFailed(runId);
23534
- else await deps.executionTracker.onBuildFailedBeforeTracking(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, setup.info.routingKey, buildTriggerEvent(event.type, event.action), extractCommitMessage(setup.info.event, setup.info.payload), toErrorMessage(err));
23644
+ const buildFailureReason = toErrorMessage(err);
23645
+ const buildInitFailure = {
23646
+ scope: "run",
23647
+ category: InitFailureCategory.enum.build_coordination,
23648
+ message: buildFailureReason
23649
+ };
23650
+ if (buildJobTrackedEarly) await deps.executionTracker.onBuildFailed(runId, buildInitFailure);
23651
+ else await deps.executionTracker.onBuildFailedBeforeTracking(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, setup.info.routingKey, buildTriggerEvent(event.type, event.action), extractCommitMessage(setup.info.event, setup.info.payload), buildFailureReason, buildInitFailure);
23535
23652
  } catch (cleanupErr) {
23536
23653
  logger$37.warn("Failed to mark run as failed after build error", {
23537
23654
  runId,
@@ -23741,11 +23858,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
23741
23858
  const declaredContexts = workflow.contexts ?? [];
23742
23859
  if (declaredContexts.length > 0) {
23743
23860
  if (!deps.secretResolver) {
23744
- logger$37.error("Workflow declares secret contexts but secrets subsystem is not configured (KICI_SECRET_KEY missing)", {
23861
+ const reason = "Workflow declares secret contexts but secrets subsystem is not configured (KICI_SECRET_KEY missing)";
23862
+ logger$37.error(reason, {
23745
23863
  workflow: workflow.name,
23746
23864
  contexts: declaredContexts
23747
23865
  });
23748
- return { skipDispatch: true };
23866
+ return {
23867
+ skipDispatch: true,
23868
+ reason
23869
+ };
23749
23870
  }
23750
23871
  try {
23751
23872
  const mergedSecrets = {};
@@ -23760,11 +23881,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
23760
23881
  resolvedNamespacedSecrets = mergedNamespaced;
23761
23882
  }
23762
23883
  } catch (err) {
23884
+ const errMessage = toErrorMessage(err);
23763
23885
  logger$37.error("Secret resolution failed, skipping workflow", {
23764
23886
  workflow: workflow.name,
23765
- error: toErrorMessage(err)
23887
+ error: errMessage
23766
23888
  });
23767
- return { skipDispatch: true };
23889
+ return {
23890
+ skipDispatch: true,
23891
+ reason: `Secret resolution failed: ${errMessage}`
23892
+ };
23768
23893
  }
23769
23894
  }
23770
23895
  let runPublicKeyBase64;
@@ -23842,7 +23967,10 @@ async function resolveWorkflowInstallSecrets(ctx, secrets) {
23842
23967
  workflow: workflow.name,
23843
23968
  reason: result.reason
23844
23969
  });
23845
- return { skipDispatch: true };
23970
+ return {
23971
+ skipDispatch: true,
23972
+ reason: `Workflow install-secrets resolution rejected dispatch: ${result.reason}`
23973
+ };
23846
23974
  }
23847
23975
  if (result.contributorStripped) logger$37.warn("Skipping registries:/installEnv: resolution for untrusted contributor — install will fail naturally if private deps are required", {
23848
23976
  runId,
@@ -24295,6 +24423,7 @@ async function clusterRouteRootJobs(args) {
24295
24423
  });
24296
24424
  rejectedJobs.push({
24297
24425
  jobId: syntheticId,
24426
+ jobName: jtr.jobName,
24298
24427
  reason: result.reason
24299
24428
  });
24300
24429
  } else if (result.status === "queued-no-backend") logger$37.warn("Job has no matching backend (cluster fallback), skipping", {
@@ -24400,6 +24529,7 @@ async function dispatchSingleOrchPath(args) {
24400
24529
  });
24401
24530
  rejectedJobs.push({
24402
24531
  jobId: syntheticId,
24532
+ jobName: jobOrFactory.name,
24403
24533
  reason: result.reason
24404
24534
  });
24405
24535
  } else if (result.status === "queued-no-backend") logger$37.warn("Job has no matching backend, skipping execution tracking", {
@@ -24504,6 +24634,11 @@ async function recordRunStart(args) {
24504
24634
  });
24505
24635
  });
24506
24636
  }
24637
+ function categorizeRejectReason(reason) {
24638
+ const lower = reason.toLowerCase();
24639
+ if (/no\s+agent|no\s+matching\s+backend/.test(lower)) return InitFailureCategory.enum.no_agent;
24640
+ return InitFailureCategory.enum.environment_rules;
24641
+ }
24507
24642
  async function insertEdgesAndMarkRejected(args) {
24508
24643
  const { ctx, staticJobs, dispatchedJobs, rejectedJobs } = args;
24509
24644
  const { deps, runId } = ctx;
@@ -24517,7 +24652,15 @@ async function insertEdgesAndMarkRejected(args) {
24517
24652
  }
24518
24653
  if (deps.executionTracker && rejectedJobs.length > 0) {
24519
24654
  const now = Date.now();
24520
- for (const { jobId, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, { error: reason }).catch((err) => {
24655
+ for (const { jobId, jobName, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, {
24656
+ error: reason,
24657
+ initFailure: {
24658
+ scope: "job",
24659
+ category: categorizeRejectReason(reason),
24660
+ message: reason,
24661
+ jobName
24662
+ }
24663
+ }).catch((err) => {
24521
24664
  logger$37.error("Failed to mark rejected job as failed", {
24522
24665
  runId,
24523
24666
  jobId,
@@ -24698,17 +24841,30 @@ function startDeferredInitDispatch(args) {
24698
24841
  lockJob
24699
24842
  });
24700
24843
  } catch (err) {
24844
+ const errMsg = toErrorMessage(err);
24701
24845
  logger$37.error("Deferred init job failed", {
24702
24846
  runId,
24703
24847
  workflow: workflow.name,
24704
24848
  job: lockJob.name,
24705
- error: toErrorMessage(err)
24849
+ error: errMsg
24706
24850
  });
24707
- if (deps.executionTracker) deps.executionTracker.addJobsToRun(runId, [{
24708
- jobId: `init-failed-${lockJob.name}`,
24709
- jobName: lockJob.name,
24710
- runsOnLabels: Array.isArray(lockJob.runsOn) ? [...lockJob.runsOn] : [lockJob.runsOn]
24711
- }]).catch(() => {});
24851
+ if (deps.executionTracker) {
24852
+ const jobId = `init-failed-${lockJob.name}`;
24853
+ await deps.executionTracker.addJobsToRun(runId, [{
24854
+ jobId,
24855
+ jobName: lockJob.name,
24856
+ runsOnLabels: Array.isArray(lockJob.runsOn) ? [...lockJob.runsOn] : [lockJob.runsOn]
24857
+ }]).catch(() => {});
24858
+ await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
24859
+ error: errMsg,
24860
+ initFailure: {
24861
+ scope: "job",
24862
+ category: InitFailureCategory.enum.dynamic_eval,
24863
+ message: errMsg,
24864
+ jobName: lockJob.name
24865
+ }
24866
+ }).catch(() => {});
24867
+ }
24712
24868
  }
24713
24869
  })();
24714
24870
  }
@@ -25250,6 +25406,39 @@ async function ensureExecutionRunForDeferred(args) {
25250
25406
  await deps.executionTracker.onExecutionStarted(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, summarizeDecision(decision), [], setup.info.routingKey, declaredContexts.length > 0 ? [...declaredContexts] : void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
25251
25407
  }
25252
25408
  /**
25409
+ * Persist a failed `execution_runs` row for a pre-dispatch early-exit so the
25410
+ * dashboard's Runs view surfaces secret_resolution / install_secrets /
25411
+ * environment_rules rejections instead of leaving the run with zero trace.
25412
+ *
25413
+ * Called from each of the three early-exit sites in `dispatchMatchedWorkflow`
25414
+ * (workflow secrets, install secrets, all-jobs-rejected). The helper is a
25415
+ * no-op when no `executionTracker` is wired into deps (test / minimal
25416
+ * deployments).
25417
+ */
25418
+ async function recordInitFailureFromSkip(args) {
25419
+ const { ctx, setup, category, reason } = args;
25420
+ const { deps, workflow, repoIdentifier, credentials, event, ref, runId } = ctx;
25421
+ if (!deps.executionTracker) return;
25422
+ await deps.executionTracker.recordInitFailureRun({
25423
+ runId,
25424
+ workflowName: workflow.name,
25425
+ provider: setup.info.provider,
25426
+ repoIdentifier,
25427
+ ref: event.sourceBranch ?? event.targetBranch,
25428
+ sha: ref,
25429
+ deliveryId: setup.effectiveDeliveryId,
25430
+ providerContext: credentials ?? {},
25431
+ routingKey: setup.info.routingKey,
25432
+ initFailure: {
25433
+ scope: "run",
25434
+ category,
25435
+ message: reason
25436
+ },
25437
+ triggerEvent: buildTriggerEvent(event.type, event.action),
25438
+ commitMessage: extractCommitMessage(setup.info.event, setup.info.payload)
25439
+ });
25440
+ }
25441
+ /**
25253
25442
  * Dispatch a single matched workflow.
25254
25443
  *
25255
25444
  * Narrative orchestrator: each phase is a typed helper that returns a typed
@@ -25269,9 +25458,26 @@ async function dispatchMatchedWorkflow(ctx) {
25269
25458
  const buildPrep = await prepareCacheAndBuild(ctx, setup);
25270
25459
  if (buildPrep.abort) return { dispatchedJobCount: 0 };
25271
25460
  const secretsResult = await resolveWorkflowSecretsAndKey(ctx);
25272
- if ("skipDispatch" in secretsResult) return { dispatchedJobCount: 0 };
25461
+ if ("skipDispatch" in secretsResult) {
25462
+ await recordInitFailureFromSkip({
25463
+ ctx,
25464
+ setup,
25465
+ category: InitFailureCategory.enum.secret_resolution,
25466
+ reason: secretsResult.reason
25467
+ });
25468
+ return { dispatchedJobCount: 0 };
25469
+ }
25273
25470
  const secrets = secretsResult;
25274
- if ((await resolveWorkflowInstallSecrets(ctx, secrets)).skipDispatch) return { dispatchedJobCount: 0 };
25471
+ const installResult = await resolveWorkflowInstallSecrets(ctx, secrets);
25472
+ if (installResult.skipDispatch) {
25473
+ await recordInitFailureFromSkip({
25474
+ ctx,
25475
+ setup,
25476
+ category: InitFailureCategory.enum.install_secrets,
25477
+ reason: installResult.reason
25478
+ });
25479
+ return { dispatchedJobCount: 0 };
25480
+ }
25275
25481
  const evalResult = await evaluateJobEnvironments({
25276
25482
  ctx,
25277
25483
  setup,
@@ -25318,6 +25524,12 @@ async function dispatchMatchedWorkflow(ctx) {
25318
25524
  dispatchedJobs,
25319
25525
  rejectedJobs
25320
25526
  });
25527
+ if (dispatchedJobs.length === 0 && evalResult.deferredInitJobs.length === 0 && buildPrep.dynamicEntries.length === 0 && rejectedJobs.length > 0) await recordInitFailureFromSkip({
25528
+ ctx,
25529
+ setup,
25530
+ category: InitFailureCategory.enum.environment_rules,
25531
+ reason: rejectedJobs[0].reason
25532
+ });
25321
25533
  if (evalResult.deferredInitJobs.length > 0) await ensureExecutionRunForDeferred({
25322
25534
  ctx,
25323
25535
  setup,
@@ -28039,7 +28251,8 @@ function createApp(deps) {
28039
28251
  platformUrl: deps.config.platformUrl,
28040
28252
  agentRegistry: deps.registry,
28041
28253
  config: deps.config,
28042
- tlsCertPath: deps.config.tlsCertPath
28254
+ tlsCertPath: deps.config.tlsCertPath,
28255
+ scalerManager: deps.scalerManager
28043
28256
  }));
28044
28257
  {
28045
28258
  const peerHandlerRef = deps.peerHandler;
@@ -31199,6 +31412,55 @@ var init_machine_ledger = __esmMin((() => {
31199
31412
  };
31200
31413
  }));
31201
31414
  //#endregion
31415
+ //#region src/scaler/failure-tracker.ts
31416
+ var DEFAULT_MAX_ENTRIES, ScalerFailureTracker;
31417
+ var init_failure_tracker = __esmMin((() => {
31418
+ DEFAULT_MAX_ENTRIES = 256;
31419
+ ScalerFailureTracker = class {
31420
+ records = [];
31421
+ maxEntries;
31422
+ constructor(maxEntries = DEFAULT_MAX_ENTRIES) {
31423
+ this.maxEntries = maxEntries;
31424
+ }
31425
+ /** Record a failure, evicting the oldest entry when over capacity. */
31426
+ record(rec) {
31427
+ this.records.push(rec);
31428
+ if (this.records.length > this.maxEntries) this.records.shift();
31429
+ }
31430
+ /**
31431
+ * Group failures newer than `nowMs - windowMs` by backend instance name.
31432
+ * `nowMs` is injected so callers control the clock (and tests are
31433
+ * deterministic).
31434
+ */
31435
+ recentByBackend(windowMs, nowMs) {
31436
+ const cutoff = nowMs - windowMs;
31437
+ const out = /* @__PURE__ */ new Map();
31438
+ for (const rec of this.records) {
31439
+ if (rec.timestampMs < cutoff) continue;
31440
+ const existing = out.get(rec.backendName);
31441
+ if (!existing) {
31442
+ out.set(rec.backendName, {
31443
+ backendType: rec.backendType,
31444
+ boundCount: rec.bound ? 1 : 0,
31445
+ unboundCount: rec.bound ? 0 : 1,
31446
+ lastError: rec.detail,
31447
+ lastAtMs: rec.timestampMs
31448
+ });
31449
+ continue;
31450
+ }
31451
+ if (rec.bound) existing.boundCount += 1;
31452
+ else existing.unboundCount += 1;
31453
+ if (rec.timestampMs >= existing.lastAtMs) {
31454
+ existing.lastError = rec.detail;
31455
+ existing.lastAtMs = rec.timestampMs;
31456
+ existing.backendType = rec.backendType;
31457
+ }
31458
+ }
31459
+ return out;
31460
+ }
31461
+ };
31462
+ }));
31463
+ //#endregion
31202
31464
  //#region src/scaler/manager.ts
31203
31465
  /**
31204
31466
  * ScalerManager: central coordinator for all scaler backends.
@@ -31262,10 +31524,13 @@ var init_manager = __esmMin((() => {
31262
31524
  init_machine_ledger();
31263
31525
  init_prometheus();
31264
31526
  init_types$1();
31527
+ init_failure_tracker();
31265
31528
  logger$27 = createLogger({ prefix: "scaler" });
31266
31529
  ScalerManager = class {
31267
31530
  backends = /* @__PURE__ */ new Map();
31268
31531
  backendRoles = /* @__PURE__ */ new Map();
31532
+ /** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
31533
+ failureTracker = new ScalerFailureTracker();
31269
31534
  globalMaxAgents;
31270
31535
  /** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
31271
31536
  resourceCaps = /* @__PURE__ */ new Map();
@@ -31907,6 +32172,13 @@ var init_manager = __esmMin((() => {
31907
32172
  /**
31908
32173
  * Return status summary for metrics and health endpoints.
31909
32174
  */
32175
+ /**
32176
+ * Recent scaler spawn failures grouped per backend instance, for the
32177
+ * diagnose scaler check. `nowMs` is injected by the caller.
32178
+ */
32179
+ recentSpawnFailures(windowMs, nowMs) {
32180
+ return this.failureTracker.recentByBackend(windowMs, nowMs);
32181
+ }
31910
32182
  getStatus() {
31911
32183
  const backendStatuses = [];
31912
32184
  const enrichedEntries = this.getEnrichedScalerEntries();
@@ -32193,6 +32465,13 @@ var init_manager = __esmMin((() => {
32193
32465
  jobId: jobId ?? null,
32194
32466
  detail: event.detail
32195
32467
  });
32468
+ this.failureTracker.record({
32469
+ backendName: backendName ?? "unknown",
32470
+ backendType: backend,
32471
+ bound: Boolean(jobId),
32472
+ detail: event.detail ?? "",
32473
+ timestampMs: event.timestampMs
32474
+ });
32196
32475
  }
32197
32476
  if (runId && jobId && this.onScalerEvent) this.onScalerEvent(runId, jobId, event);
32198
32477
  else if (this.onScalerEvent) {
@@ -33948,6 +34227,22 @@ var init_execution_tracker = __esmMin((() => {
33948
34227
  orgId;
33949
34228
  jobQueue;
33950
34229
  runs = /* @__PURE__ */ new Map();
34230
+ /**
34231
+ * Per-run async-mutex chain. `onJobStatus` and `addJobsToRun` mutate the same
34232
+ * `run.jobs` Map / `execution_jobs` row; without serialization a job-status
34233
+ * reply that lands mid synthetic→real swap (`addJobsToRun`) clobbers the swap
34234
+ * and the run hangs in `running` forever (see `withRunLock`). The map value is
34235
+ * the tail of the chain for that runId; entries are GC'd when the last holder
34236
+ * releases.
34237
+ */
34238
+ runLockTails = /* @__PURE__ */ new Map();
34239
+ /**
34240
+ * Tracks which runIds the current async context already holds in
34241
+ * `withRunLock`, so reentrant calls (e.g. onJobStatus → scheduler hook →
34242
+ * dispatchReadyJob → addJobsToRun, all the same runId) bypass re-acquisition
34243
+ * instead of deadlocking. Propagates across awaits via AsyncLocalStorage.
34244
+ */
34245
+ heldRunLocks = new AsyncLocalStorage();
33951
34246
  /** Tracks which runs are test runs for observer broadcasting. */
33952
34247
  testRunIds = /* @__PURE__ */ new Set();
33953
34248
  /**
@@ -34102,7 +34397,44 @@ var init_execution_tracker = __esmMin((() => {
34102
34397
  }
34103
34398
  return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
34104
34399
  }
34400
+ /**
34401
+ * Run `fn` while holding a per-run lock, serializing the run-mutating methods
34402
+ * (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
34403
+ * synthetic→real job swap and wedge the run in `running`.
34404
+ *
34405
+ * The lock is **reentrant**: three paths re-enter a locked method within the
34406
+ * same async context for the same runId — onJobStatus → scheduler hook →
34407
+ * dispatchReadyJob → addJobsToRun; onJobStatus → enforceSchedulerInvariant →
34408
+ * onJobStatus; runSchedulerHook → onJobStatus (skip). A non-reentrant mutex
34409
+ * would deadlock on these, so a context that already holds the runId's lock
34410
+ * (tracked via `heldRunLocks`) runs `fn` inline. A genuinely concurrent caller
34411
+ * for the same runId (a separate WS message) is a different async context and
34412
+ * correctly waits. All reentrant paths are same-runId, so there is no
34413
+ * cross-run lock-ordering deadlock.
34414
+ */
34415
+ async withRunLock(runId, fn) {
34416
+ if (this.heldRunLocks.getStore()?.has(runId)) return fn();
34417
+ const prev = this.runLockTails.get(runId) ?? Promise.resolve();
34418
+ let release;
34419
+ const mine = new Promise((resolve) => {
34420
+ release = resolve;
34421
+ });
34422
+ const chained = prev.then(() => mine);
34423
+ this.runLockTails.set(runId, chained);
34424
+ await prev.catch(() => {});
34425
+ const nextHeld = new Set(this.heldRunLocks.getStore() ?? []);
34426
+ nextHeld.add(runId);
34427
+ try {
34428
+ return await this.heldRunLocks.run(nextHeld, fn);
34429
+ } finally {
34430
+ release();
34431
+ if (this.runLockTails.get(runId) === chained) this.runLockTails.delete(runId);
34432
+ }
34433
+ }
34105
34434
  async addJobsToRun(runId, jobs, dispatchedContexts, replaceSyntheticId) {
34435
+ return this.withRunLock(runId, () => this.addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId));
34436
+ }
34437
+ async addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId) {
34106
34438
  const run = this.runs.get(runId);
34107
34439
  if (!run) {
34108
34440
  logger$23.warn("addJobsToRun: run not found in memory", { runId });
@@ -34197,6 +34529,9 @@ var init_execution_tracker = __esmMin((() => {
34197
34529
  * fires the onExecutionComplete callback.
34198
34530
  */
34199
34531
  async onJobStatus(runId, jobId, state, timestamp, agentId, data) {
34532
+ return this.withRunLock(runId, () => this.onJobStatusImpl(runId, jobId, state, timestamp, agentId, data));
34533
+ }
34534
+ async onJobStatusImpl(runId, jobId, state, timestamp, agentId, data) {
34200
34535
  let run = this.runs.get(runId);
34201
34536
  let job = run?.jobs.get(jobId);
34202
34537
  if (!run) {
@@ -34317,6 +34652,8 @@ var init_execution_tracker = __esmMin((() => {
34317
34652
  * run's in-memory map. Returns the freshly-inserted job entry.
34318
34653
  */
34319
34654
  async recoverJobFromDispatchQueue(run, runId, jobId) {
34655
+ const existing = run.jobs.get(jobId);
34656
+ if (existing) return existing;
34320
34657
  const jobName = (await this.db.selectFrom("dispatch_queue").select(["job_name"]).where("id", "=", jobId).executeTakeFirst())?.job_name ?? jobId;
34321
34658
  run.jobs.set(jobId, {
34322
34659
  name: jobName,
@@ -34351,6 +34688,7 @@ var init_execution_tracker = __esmMin((() => {
34351
34688
  updateValues.duration_ms = timestamp - job.startedAt;
34352
34689
  }
34353
34690
  if (data?.error) updateValues.error_message = String(data.error);
34691
+ if (data?.initFailure) updateValues.init_failure = JSON.stringify(data.initFailure);
34354
34692
  if (data?.outputs) updateValues.outputs = JSON.stringify(data.outputs);
34355
34693
  jobLogBytesTotal = this.jobLogBytes.get(runId)?.get(jobId) ?? 0;
34356
34694
  updateValues.log_bytes = jobLogBytesTotal;
@@ -34469,7 +34807,8 @@ var init_execution_tracker = __esmMin((() => {
34469
34807
  const completedAtMs = isTerminal ? timestamp : void 0;
34470
34808
  const duration = isTerminal && job.startedAt ? timestamp - job.startedAt : void 0;
34471
34809
  const errorMsg = isTerminal && data?.error ? String(data.error) : void 0;
34472
- this.onJobStatusChange?.(runId, jobId, job.name, state, timestamp, startedAtMs, completedAtMs, duration, errorMsg, job.agentId, job.runsOnLabels, jobLogBytesTotal);
34810
+ const initFailure = isTerminal && data?.initFailure ? data.initFailure : void 0;
34811
+ this.onJobStatusChange?.(runId, jobId, job.name, state, timestamp, startedAtMs, completedAtMs, duration, errorMsg, job.agentId, job.runsOnLabels, jobLogBytesTotal, initFailure);
34473
34812
  }
34474
34813
  /**
34475
34814
  * Phase 6: reconcile orphaned steps. If a job reaches terminal state but
@@ -34666,13 +35005,14 @@ var init_execution_tracker = __esmMin((() => {
34666
35005
  * onExecutionStarted but the build subsequently fails. Without this,
34667
35006
  * the execution_runs row would stay in a non-terminal state.
34668
35007
  */
34669
- async onBuildFailed(runId) {
35008
+ async onBuildFailed(runId, initFailure) {
34670
35009
  const now = /* @__PURE__ */ new Date();
34671
- const failureReason = "Build job failed";
35010
+ const failureReason = initFailure?.message ?? "Build job failed";
34672
35011
  await this.db.updateTable("execution_runs").set({
34673
35012
  status: ExecutionRunStatus.enum.failed,
34674
35013
  completed_at: now,
34675
- failure_reason: failureReason
35014
+ failure_reason: failureReason,
35015
+ ...initFailure && { init_failure: JSON.stringify(initFailure) }
34676
35016
  }).where("run_id", "=", runId).execute();
34677
35017
  await this.db.updateTable("execution_jobs").set({
34678
35018
  status: ExecutionJobStatus.enum.failed,
@@ -34695,7 +35035,7 @@ var init_execution_tracker = __esmMin((() => {
34695
35035
  parentRunId: run.parentRunId,
34696
35036
  originalRunId: run.originalRunId,
34697
35037
  triggeredBy: run.triggeredBy
34698
- }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason);
35038
+ }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason, void 0, initFailure);
34699
35039
  this.runs.delete(runId);
34700
35040
  this.jobLogBytes.delete(runId);
34701
35041
  }
@@ -34708,7 +35048,7 @@ var init_execution_tracker = __esmMin((() => {
34708
35048
  * Inserts a minimal execution_runs row with status='failed' directly so the E2E
34709
35049
  * test (and dashboard) can observe the failure instead of a missing run.
34710
35050
  */
34711
- async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason) {
35051
+ async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason, initFailure) {
34712
35052
  const now = /* @__PURE__ */ new Date();
34713
35053
  const reason = failureReason ?? "Build job timed out before execution tracking started";
34714
35054
  await this.db.insertInto("execution_runs").values({
@@ -34725,7 +35065,8 @@ var init_execution_tracker = __esmMin((() => {
34725
35065
  started_at: now,
34726
35066
  completed_at: now,
34727
35067
  status: ExecutionRunStatus.enum.failed,
34728
- failure_reason: reason
35068
+ failure_reason: reason,
35069
+ ...initFailure && { init_failure: JSON.stringify(initFailure) }
34729
35070
  }).execute();
34730
35071
  logger$23.info("Created failed execution run (build failed before tracking)", {
34731
35072
  runId,
@@ -34734,6 +35075,55 @@ var init_execution_tracker = __esmMin((() => {
34734
35075
  });
34735
35076
  }
34736
35077
  /**
35078
+ * Insert a `failed` execution_runs row directly for an init failure that
35079
+ * occurred BEFORE onExecutionStarted ran (so no in-memory state exists
35080
+ * and no jobs were dispatched). Also writes the structured init_failure
35081
+ * signal and fires onExecutionStatusChange so Platform's projection picks
35082
+ * it up via the normal forward path. Idempotent: if a row already exists
35083
+ * for this runId, the insert is a no-op (ON CONFLICT DO NOTHING).
35084
+ *
35085
+ * Closes the silent pre-run-failure gap — without this helper, secret /
35086
+ * install-secret / all-jobs-rejected early-exits in dispatch-matched-workflow
35087
+ * leave no trace on the dashboard.
35088
+ */
35089
+ async recordInitFailureRun(args) {
35090
+ const now = /* @__PURE__ */ new Date();
35091
+ await this.db.insertInto("execution_runs").values({
35092
+ run_id: args.runId,
35093
+ routing_key: args.routingKey,
35094
+ workflow_name: args.workflowName,
35095
+ provider: args.provider,
35096
+ repo_identifier: args.repoIdentifier,
35097
+ ref: args.ref,
35098
+ sha: args.sha,
35099
+ delivery_id: args.deliveryId,
35100
+ trigger_decision: null,
35101
+ provider_context: JSON.stringify(args.providerContext),
35102
+ started_at: now,
35103
+ completed_at: now,
35104
+ status: ExecutionRunStatus.enum.failed,
35105
+ failure_reason: args.initFailure.message,
35106
+ init_failure: JSON.stringify(args.initFailure)
35107
+ }).onConflict((oc) => oc.column("run_id").doNothing()).execute();
35108
+ executionsTotal.add(1, { status: ExecutionRunStatus.enum.failed });
35109
+ this.onExecutionStatusChange?.(args.runId, ExecutionRunStatus.enum.failed, {
35110
+ workflowName: args.workflowName,
35111
+ provider: args.provider,
35112
+ repoIdentifier: args.repoIdentifier,
35113
+ sha: args.sha,
35114
+ routingKey: args.routingKey,
35115
+ ref: args.ref,
35116
+ triggerEvent: args.triggerEvent,
35117
+ commitMessage: args.commitMessage
35118
+ }, 0, now.getTime(), now.getTime(), 0, args.initFailure.message, 0, args.initFailure);
35119
+ logger$23.info("Recorded init-failure execution run", {
35120
+ runId: args.runId,
35121
+ workflowName: args.workflowName,
35122
+ category: args.initFailure.category,
35123
+ scope: args.initFailure.scope
35124
+ });
35125
+ }
35126
+ /**
34737
35127
  * Mark a run as failed immediately with a reason message.
34738
35128
  *
34739
35129
  * Used when no agents are available to dispatch any jobs (e.g. cron-triggered
@@ -34741,7 +35131,7 @@ var init_execution_tracker = __esmMin((() => {
34741
35131
  * Instead of leaving the run in 'running' for OrphanRecovery to catch after 5 min,
34742
35132
  * this fails it right away.
34743
35133
  */
34744
- async failRun(runId, reason) {
35134
+ async failRun(runId, reason, initFailure) {
34745
35135
  const now = /* @__PURE__ */ new Date();
34746
35136
  const run = this.runs.get(runId);
34747
35137
  const durationMs = run ? now.getTime() - run.startedAt : 0;
@@ -34749,7 +35139,8 @@ var init_execution_tracker = __esmMin((() => {
34749
35139
  status: ExecutionRunStatus.enum.failed,
34750
35140
  completed_at: now,
34751
35141
  duration_ms: durationMs,
34752
- failure_reason: reason
35142
+ failure_reason: reason,
35143
+ ...initFailure && { init_failure: JSON.stringify(initFailure) }
34753
35144
  }).where("run_id", "=", runId).execute();
34754
35145
  await this.db.updateTable("execution_jobs").set({
34755
35146
  status: ExecutionJobStatus.enum.failed,
@@ -34773,7 +35164,7 @@ var init_execution_tracker = __esmMin((() => {
34773
35164
  parentRunId: run.parentRunId,
34774
35165
  originalRunId: run.originalRunId,
34775
35166
  triggeredBy: run.triggeredBy
34776
- }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason);
35167
+ }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason, void 0, initFailure);
34777
35168
  this.runs.delete(runId);
34778
35169
  this.jobLogBytes.delete(runId);
34779
35170
  }
@@ -44028,14 +44419,14 @@ var init_worker_core = __esmMin((() => {
44028
44419
  init_agent_handler();
44029
44420
  init_worker_status();
44030
44421
  init_agent_heartbeat();
44031
- ORCHESTRATOR_VERSION$1 = "0.1.12";
44032
- WORKER_BUILD_COMMIT = "d220ac45c";
44033
- WORKER_SDK_VERSION = "0.1.12";
44422
+ ORCHESTRATOR_VERSION$1 = "0.1.14";
44423
+ WORKER_BUILD_COMMIT = "0fc7a43c9";
44424
+ WORKER_SDK_VERSION = "0.1.14";
44034
44425
  WORKER_SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
44035
- WORKER_SHARED_VERSION = "0.1.12";
44036
- WORKER_SHARED_BUNDLE_HASH = "36a23a454fd75ec35f258d39f254543ec0451c6b543fe3618ce427e6e6369aae";
44037
- WORKER_ENGINE_VERSION = "0.1.12";
44038
- WORKER_ENGINE_BUNDLE_HASH = "4ab08e63bcb8558b1191c5d3ab8d58b71950b77d911af56c9955ae1d787c4302";
44426
+ WORKER_SHARED_VERSION = "0.1.14";
44427
+ WORKER_SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
44428
+ WORKER_ENGINE_VERSION = "0.1.14";
44429
+ WORKER_ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
44039
44430
  logger$2 = createLogger({ prefix: "worker" });
44040
44431
  DRAIN_TIMEOUT_MS = 3e5;
44041
44432
  }));
@@ -44059,14 +44450,14 @@ var init_worker_core = __esmMin((() => {
44059
44450
  * Graceful shutdown:
44060
44451
  * agent WS -> heartbeat -> HTTP -> DB
44061
44452
  */
44062
- const ORCHESTRATOR_VERSION = "0.1.12";
44063
- const BUILD_COMMIT = "d220ac45c";
44064
- const SDK_VERSION = "0.1.12";
44453
+ const ORCHESTRATOR_VERSION = "0.1.14";
44454
+ const BUILD_COMMIT = "0fc7a43c9";
44455
+ const SDK_VERSION = "0.1.14";
44065
44456
  const SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
44066
- const SHARED_VERSION = "0.1.12";
44067
- const SHARED_BUNDLE_HASH = "36a23a454fd75ec35f258d39f254543ec0451c6b543fe3618ce427e6e6369aae";
44068
- const ENGINE_VERSION = "0.1.12";
44069
- const ENGINE_BUNDLE_HASH = "4ab08e63bcb8558b1191c5d3ab8d58b71950b77d911af56c9955ae1d787c4302";
44457
+ const SHARED_VERSION = "0.1.14";
44458
+ const SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
44459
+ const ENGINE_VERSION = "0.1.14";
44460
+ const ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
44070
44461
  const otelSdk = initTelemetry({
44071
44462
  serviceName: "kici-orchestrator",
44072
44463
  otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT