@kici-dev/orchestrator 0.1.13 → 0.1.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/server.js CHANGED
@@ -3,7 +3,7 @@ import { dirname as __cjs_dirname } from "node:path";
3
3
  __cjs_dirname(__cjs_fileURLToPath(import.meta.url));
4
4
  import "node:module";
5
5
  import { BaseColdStore, RingBuffer, createDb, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, createPool, createS3Client, encodeKeySegment, enrichRequestContext, formatDuration, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, logger, requestContext, serializeError, setServiceName, setupGracefulShutdown, sha256, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
6
- import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EVENT_LOG_PAYLOAD_CHUNK_BYTES, EventLogPayloadStreamError, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, ORCH_CAPABILITIES, OrchRole, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WEBHOOK_RELAY_MAX_BODY_BYTES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, dashboardRunDetailApiResponseSchema, dashboardStepLogsApiResponseSchema, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, joinRequestSchema, logPullPlatformToOrchSchema, matchAllWorkflows, matchScopePattern, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, platformToOrchestratorMessageSchema, resolveRoleLabels, resolveSecretsForEnvironment, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, stringifyActor, stripScopePrefix, testEventSchema } from "@kici-dev/engine";
6
+ import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EVENT_LOG_PAYLOAD_CHUNK_BYTES, EventLogPayloadStreamError, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, ORCH_CAPABILITIES, OrchRole, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WEBHOOK_RELAY_MAX_BODY_BYTES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, dashboardRunDetailApiResponseSchema, dashboardStepLogsApiResponseSchema, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, joinRequestSchema, logPullPlatformToOrchSchema, matchAllWorkflows, matchScopePattern, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, platformToOrchestratorMessageSchema, resolveRoleLabels, resolveSecretsForEnvironment, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, stringifyActor, stripScopePrefix, testEventSchema } from "@kici-dev/engine";
7
7
  import { verifySignature } from "@kici-dev/engine/webhook/signature";
8
8
  import { X509Certificate, createCipheriv, createDecipheriv, createHmac, createPrivateKey, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
9
9
  import os, { cpus, freemem, homedir, hostname, release, tmpdir, totalmem, uptime, userInfo, version } from "node:os";
@@ -49,6 +49,7 @@ import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCom
49
49
  import { Upload } from "@aws-sdk/lib-storage";
50
50
  import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
51
51
  import stripAnsi from "strip-ansi";
52
+ import { AsyncLocalStorage } from "node:async_hooks";
52
53
  import { ORCH_PUSHED_METRIC_NAMES } from "@kici-dev/engine/metrics/catalog-policy";
53
54
  var __defProp = Object.defineProperty;
54
55
  var __esmMin = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
@@ -3850,7 +3851,11 @@ async function runBuildJob(args) {
3850
3851
  jobName: buildJobName,
3851
3852
  runsOnLabels: buildJobInput.runsOnLabels
3852
3853
  }], setup.info.routingKey, void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage$1(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
3853
- await deps.executionTracker.failRun(runId, reason);
3854
+ await deps.executionTracker.failRun(runId, reason, {
3855
+ scope: "run",
3856
+ category: InitFailureCategory.enum.build_coordination,
3857
+ message: reason
3858
+ });
3854
3859
  }
3855
3860
  return;
3856
3861
  }
@@ -3903,8 +3908,14 @@ async function recordBuildFailure(args) {
3903
3908
  });
3904
3909
  }
3905
3910
  if (deps.executionTracker) try {
3906
- if (buildJobTrackedEarly) await deps.executionTracker.onBuildFailed(runId);
3907
- else await deps.executionTracker.onBuildFailedBeforeTracking(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, setup.info.routingKey, buildTriggerEvent(event.type, event.action), extractCommitMessage$1(setup.info.event, setup.info.payload), toErrorMessage(err));
3911
+ const buildFailureReason = toErrorMessage(err);
3912
+ const buildInitFailure = {
3913
+ scope: "run",
3914
+ category: InitFailureCategory.enum.build_coordination,
3915
+ message: buildFailureReason
3916
+ };
3917
+ if (buildJobTrackedEarly) await deps.executionTracker.onBuildFailed(runId, buildInitFailure);
3918
+ else await deps.executionTracker.onBuildFailedBeforeTracking(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, setup.info.routingKey, buildTriggerEvent(event.type, event.action), extractCommitMessage$1(setup.info.event, setup.info.payload), buildFailureReason, buildInitFailure);
3908
3919
  } catch (cleanupErr) {
3909
3920
  logger$92.warn("Failed to mark run as failed after build error", {
3910
3921
  runId,
@@ -4114,11 +4125,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
4114
4125
  const declaredContexts = workflow.contexts ?? [];
4115
4126
  if (declaredContexts.length > 0) {
4116
4127
  if (!deps.secretResolver) {
4117
- logger$92.error("Workflow declares secret contexts but secrets subsystem is not configured (KICI_SECRET_KEY missing)", {
4128
+ const reason = "Workflow declares secret contexts but secrets subsystem is not configured (KICI_SECRET_KEY missing)";
4129
+ logger$92.error(reason, {
4118
4130
  workflow: workflow.name,
4119
4131
  contexts: declaredContexts
4120
4132
  });
4121
- return { skipDispatch: true };
4133
+ return {
4134
+ skipDispatch: true,
4135
+ reason
4136
+ };
4122
4137
  }
4123
4138
  try {
4124
4139
  const mergedSecrets = {};
@@ -4133,11 +4148,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
4133
4148
  resolvedNamespacedSecrets = mergedNamespaced;
4134
4149
  }
4135
4150
  } catch (err) {
4151
+ const errMessage = toErrorMessage(err);
4136
4152
  logger$92.error("Secret resolution failed, skipping workflow", {
4137
4153
  workflow: workflow.name,
4138
- error: toErrorMessage(err)
4154
+ error: errMessage
4139
4155
  });
4140
- return { skipDispatch: true };
4156
+ return {
4157
+ skipDispatch: true,
4158
+ reason: `Secret resolution failed: ${errMessage}`
4159
+ };
4141
4160
  }
4142
4161
  }
4143
4162
  let runPublicKeyBase64;
@@ -4215,7 +4234,10 @@ async function resolveWorkflowInstallSecrets(ctx, secrets) {
4215
4234
  workflow: workflow.name,
4216
4235
  reason: result.reason
4217
4236
  });
4218
- return { skipDispatch: true };
4237
+ return {
4238
+ skipDispatch: true,
4239
+ reason: `Workflow install-secrets resolution rejected dispatch: ${result.reason}`
4240
+ };
4219
4241
  }
4220
4242
  if (result.contributorStripped) logger$92.warn("Skipping registries:/installEnv: resolution for untrusted contributor — install will fail naturally if private deps are required", {
4221
4243
  runId,
@@ -4668,6 +4690,7 @@ async function clusterRouteRootJobs(args) {
4668
4690
  });
4669
4691
  rejectedJobs.push({
4670
4692
  jobId: syntheticId,
4693
+ jobName: jtr.jobName,
4671
4694
  reason: result.reason
4672
4695
  });
4673
4696
  } else if (result.status === "queued-no-backend") logger$92.warn("Job has no matching backend (cluster fallback), skipping", {
@@ -4773,6 +4796,7 @@ async function dispatchSingleOrchPath(args) {
4773
4796
  });
4774
4797
  rejectedJobs.push({
4775
4798
  jobId: syntheticId,
4799
+ jobName: jobOrFactory.name,
4776
4800
  reason: result.reason
4777
4801
  });
4778
4802
  } else if (result.status === "queued-no-backend") logger$92.warn("Job has no matching backend, skipping execution tracking", {
@@ -4877,6 +4901,11 @@ async function recordRunStart(args) {
4877
4901
  });
4878
4902
  });
4879
4903
  }
4904
+ function categorizeRejectReason(reason) {
4905
+ const lower = reason.toLowerCase();
4906
+ if (/no\s+agent|no\s+matching\s+backend/.test(lower)) return InitFailureCategory.enum.no_agent;
4907
+ return InitFailureCategory.enum.environment_rules;
4908
+ }
4880
4909
  async function insertEdgesAndMarkRejected(args) {
4881
4910
  const { ctx, staticJobs, dispatchedJobs, rejectedJobs } = args;
4882
4911
  const { deps, runId } = ctx;
@@ -4890,7 +4919,15 @@ async function insertEdgesAndMarkRejected(args) {
4890
4919
  }
4891
4920
  if (deps.executionTracker && rejectedJobs.length > 0) {
4892
4921
  const now = Date.now();
4893
- for (const { jobId, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, { error: reason }).catch((err) => {
4922
+ for (const { jobId, jobName, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, {
4923
+ error: reason,
4924
+ initFailure: {
4925
+ scope: "job",
4926
+ category: categorizeRejectReason(reason),
4927
+ message: reason,
4928
+ jobName
4929
+ }
4930
+ }).catch((err) => {
4894
4931
  logger$92.error("Failed to mark rejected job as failed", {
4895
4932
  runId,
4896
4933
  jobId,
@@ -5071,17 +5108,30 @@ function startDeferredInitDispatch(args) {
5071
5108
  lockJob
5072
5109
  });
5073
5110
  } catch (err) {
5111
+ const errMsg = toErrorMessage(err);
5074
5112
  logger$92.error("Deferred init job failed", {
5075
5113
  runId,
5076
5114
  workflow: workflow.name,
5077
5115
  job: lockJob.name,
5078
- error: toErrorMessage(err)
5116
+ error: errMsg
5079
5117
  });
5080
- if (deps.executionTracker) deps.executionTracker.addJobsToRun(runId, [{
5081
- jobId: `init-failed-${lockJob.name}`,
5082
- jobName: lockJob.name,
5083
- runsOnLabels: Array.isArray(lockJob.runsOn) ? [...lockJob.runsOn] : [lockJob.runsOn]
5084
- }]).catch(() => {});
5118
+ if (deps.executionTracker) {
5119
+ const jobId = `init-failed-${lockJob.name}`;
5120
+ await deps.executionTracker.addJobsToRun(runId, [{
5121
+ jobId,
5122
+ jobName: lockJob.name,
5123
+ runsOnLabels: Array.isArray(lockJob.runsOn) ? [...lockJob.runsOn] : [lockJob.runsOn]
5124
+ }]).catch(() => {});
5125
+ await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
5126
+ error: errMsg,
5127
+ initFailure: {
5128
+ scope: "job",
5129
+ category: InitFailureCategory.enum.dynamic_eval,
5130
+ message: errMsg,
5131
+ jobName: lockJob.name
5132
+ }
5133
+ }).catch(() => {});
5134
+ }
5085
5135
  }
5086
5136
  })();
5087
5137
  }
@@ -5623,6 +5673,39 @@ async function ensureExecutionRunForDeferred(args) {
5623
5673
  await deps.executionTracker.onExecutionStarted(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, summarizeDecision(decision), [], setup.info.routingKey, declaredContexts.length > 0 ? [...declaredContexts] : void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage$1(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
5624
5674
  }
5625
5675
  /**
5676
+ * Persist a failed `execution_runs` row for a pre-dispatch early-exit so the
5677
+ * dashboard's Runs view surfaces secret_resolution / install_secrets /
5678
+ * environment_rules rejections instead of leaving the run with zero trace.
5679
+ *
5680
+ * Called from each of the three early-exit sites in `dispatchMatchedWorkflow`
5681
+ * (workflow secrets, install secrets, all-jobs-rejected). The helper is a
5682
+ * no-op when no `executionTracker` is wired into deps (test / minimal
5683
+ * deployments).
5684
+ */
5685
+ async function recordInitFailureFromSkip(args) {
5686
+ const { ctx, setup, category, reason } = args;
5687
+ const { deps, workflow, repoIdentifier, credentials, event, ref, runId } = ctx;
5688
+ if (!deps.executionTracker) return;
5689
+ await deps.executionTracker.recordInitFailureRun({
5690
+ runId,
5691
+ workflowName: workflow.name,
5692
+ provider: setup.info.provider,
5693
+ repoIdentifier,
5694
+ ref: event.sourceBranch ?? event.targetBranch,
5695
+ sha: ref,
5696
+ deliveryId: setup.effectiveDeliveryId,
5697
+ providerContext: credentials ?? {},
5698
+ routingKey: setup.info.routingKey,
5699
+ initFailure: {
5700
+ scope: "run",
5701
+ category,
5702
+ message: reason
5703
+ },
5704
+ triggerEvent: buildTriggerEvent(event.type, event.action),
5705
+ commitMessage: extractCommitMessage$1(setup.info.event, setup.info.payload)
5706
+ });
5707
+ }
5708
+ /**
5626
5709
  * Dispatch a single matched workflow.
5627
5710
  *
5628
5711
  * Narrative orchestrator: each phase is a typed helper that returns a typed
@@ -5642,9 +5725,26 @@ async function dispatchMatchedWorkflow(ctx) {
5642
5725
  const buildPrep = await prepareCacheAndBuild(ctx, setup);
5643
5726
  if (buildPrep.abort) return { dispatchedJobCount: 0 };
5644
5727
  const secretsResult = await resolveWorkflowSecretsAndKey(ctx);
5645
- if ("skipDispatch" in secretsResult) return { dispatchedJobCount: 0 };
5728
+ if ("skipDispatch" in secretsResult) {
5729
+ await recordInitFailureFromSkip({
5730
+ ctx,
5731
+ setup,
5732
+ category: InitFailureCategory.enum.secret_resolution,
5733
+ reason: secretsResult.reason
5734
+ });
5735
+ return { dispatchedJobCount: 0 };
5736
+ }
5646
5737
  const secrets = secretsResult;
5647
- if ((await resolveWorkflowInstallSecrets(ctx, secrets)).skipDispatch) return { dispatchedJobCount: 0 };
5738
+ const installResult = await resolveWorkflowInstallSecrets(ctx, secrets);
5739
+ if (installResult.skipDispatch) {
5740
+ await recordInitFailureFromSkip({
5741
+ ctx,
5742
+ setup,
5743
+ category: InitFailureCategory.enum.install_secrets,
5744
+ reason: installResult.reason
5745
+ });
5746
+ return { dispatchedJobCount: 0 };
5747
+ }
5648
5748
  const evalResult = await evaluateJobEnvironments({
5649
5749
  ctx,
5650
5750
  setup,
@@ -5691,6 +5791,12 @@ async function dispatchMatchedWorkflow(ctx) {
5691
5791
  dispatchedJobs,
5692
5792
  rejectedJobs
5693
5793
  });
5794
+ if (dispatchedJobs.length === 0 && evalResult.deferredInitJobs.length === 0 && buildPrep.dynamicEntries.length === 0 && rejectedJobs.length > 0) await recordInitFailureFromSkip({
5795
+ ctx,
5796
+ setup,
5797
+ category: InitFailureCategory.enum.environment_rules,
5798
+ reason: rejectedJobs[0].reason
5799
+ });
5694
5800
  if (evalResult.deferredInitJobs.length > 0) await ensureExecutionRunForDeferred({
5695
5801
  ctx,
5696
5802
  setup,
@@ -8078,7 +8184,8 @@ var init_handler = __esmMin((() => {
8078
8184
  let runRow = await this.db.selectFrom("execution_runs").select([
8079
8185
  "trust_tier",
8080
8186
  "lock_file_source",
8081
- "contributor_username"
8187
+ "contributor_username",
8188
+ "init_failure"
8082
8189
  ]).where("run_id", "=", msg.runId).executeTakeFirst();
8083
8190
  let jobs = await this.db.selectFrom("execution_jobs").select([
8084
8191
  "job_id",
@@ -8091,7 +8198,8 @@ var init_handler = __esmMin((() => {
8091
8198
  "duration_ms",
8092
8199
  "error_message",
8093
8200
  "runs_on_labels",
8094
- "outputs"
8201
+ "outputs",
8202
+ "init_failure"
8095
8203
  ]).where("run_id", "=", msg.runId).orderBy("created_at", "asc").execute();
8096
8204
  let steps = await this.db.selectFrom("execution_steps").select([
8097
8205
  "job_id",
@@ -8133,6 +8241,7 @@ var init_handler = __esmMin((() => {
8133
8241
  }
8134
8242
  const responseJobs = jobs.map((job) => {
8135
8243
  const jobSteps = stepsByJob.get(job.job_id) ?? [];
8244
+ const jobInitFailure = job.init_failure ?? void 0;
8136
8245
  return {
8137
8246
  jobId: job.job_id,
8138
8247
  jobName: job.job_name,
@@ -8147,6 +8256,7 @@ var init_handler = __esmMin((() => {
8147
8256
  runsOnLabels: job.runs_on_labels ?? null,
8148
8257
  outputs: job.outputs ? typeof job.outputs === "string" ? JSON.parse(job.outputs) : job.outputs : null,
8149
8258
  secretOutputKeys: secretKeysByJob.get(job.job_id) ?? null,
8259
+ ...jobInitFailure && { initFailure: jobInitFailure },
8150
8260
  steps: jobSteps.map((step) => ({
8151
8261
  stepIndex: step.step_index,
8152
8262
  stepName: step.step_name,
@@ -8166,9 +8276,11 @@ var init_handler = __esmMin((() => {
8166
8276
  lockFileSource: runRow.lock_file_source ?? null,
8167
8277
  contributorUsername: runRow.contributor_username ?? null
8168
8278
  } : void 0;
8279
+ const runInitFailure = runRow?.init_failure ?? void 0;
8169
8280
  const validated = dashboardRunDetailApiResponseSchema.safeParse({
8170
8281
  jobs: responseJobs,
8171
- ...trustContext && { trustContext }
8282
+ ...trustContext && { trustContext },
8283
+ ...runInitFailure && { initFailure: runInitFailure }
8172
8284
  });
8173
8285
  if (!validated.success) {
8174
8286
  logger$86.error("Outgoing dashboard.run.detail response validation failed", {
@@ -8195,7 +8307,8 @@ var init_handler = __esmMin((() => {
8195
8307
  type: "dashboard.run.detail.response",
8196
8308
  requestId: msg.requestId,
8197
8309
  jobs: validated.data.jobs,
8198
- ...validated.data.trustContext && { trustContext: validated.data.trustContext }
8310
+ ...validated.data.trustContext && { trustContext: validated.data.trustContext },
8311
+ ...validated.data.initFailure && { initFailure: validated.data.initFailure }
8199
8312
  });
8200
8313
  } catch (err) {
8201
8314
  logger$86.error("Error handling dashboard.run.detail", {
@@ -9195,7 +9308,8 @@ var init_handler = __esmMin((() => {
9195
9308
  out.run = {
9196
9309
  trust_tier: typeof r.trust_tier === "string" ? r.trust_tier : null,
9197
9310
  lock_file_source: typeof r.lock_file_source === "string" ? r.lock_file_source : null,
9198
- contributor_username: typeof r.contributor_username === "string" ? r.contributor_username : null
9311
+ contributor_username: typeof r.contributor_username === "string" ? r.contributor_username : null,
9312
+ init_failure: r.init_failure ?? null
9199
9313
  };
9200
9314
  }
9201
9315
  out.jobs = jobRows.map((r) => ({
@@ -9209,7 +9323,8 @@ var init_handler = __esmMin((() => {
9209
9323
  duration_ms: typeof r.duration_ms === "number" ? r.duration_ms : null,
9210
9324
  error_message: typeof r.error_message === "string" ? r.error_message : null,
9211
9325
  runs_on_labels: typeof r.runs_on_labels === "string" ? r.runs_on_labels : null,
9212
- outputs: typeof r.outputs === "string" ? r.outputs : null
9326
+ outputs: typeof r.outputs === "string" ? r.outputs : null,
9327
+ init_failure: r.init_failure ?? null
9213
9328
  }));
9214
9329
  stepRows.sort((a, b) => Number(a.step_index ?? 0) - Number(b.step_index ?? 0));
9215
9330
  out.steps = stepRows.map((r) => ({
@@ -15175,7 +15290,7 @@ var init_peer_client = __esmMin((() => {
15175
15290
  init_peer_crypto();
15176
15291
  init_peer_credentials();
15177
15292
  logger$74 = createLogger({ prefix: "peer-client" });
15178
- SOFTWARE_VERSION$1 = "0.1.13";
15293
+ SOFTWARE_VERSION$1 = "0.1.14";
15179
15294
  PeerClient$1 = class {
15180
15295
  ws = null;
15181
15296
  _state = "disconnected";
@@ -16641,7 +16756,7 @@ var init_peer_handler = __esmMin((() => {
16641
16756
  init_peer_crypto();
16642
16757
  init_join_token();
16643
16758
  logger$73 = createLogger({ prefix: "peer-handler" });
16644
- SOFTWARE_VERSION = "0.1.13";
16759
+ SOFTWARE_VERSION = "0.1.14";
16645
16760
  RATE_LIMIT_MAX = 5;
16646
16761
  RATE_LIMIT_WINDOW_MS = 6e4;
16647
16762
  }));
@@ -17904,15 +18019,15 @@ var init_cluster = __esmMin((() => {
17904
18019
  * Uses a 15-minute TTL and 10,000 max entries.
17905
18020
  */
17906
18021
  var contributor_cache_exports = /* @__PURE__ */ __exportAll({ ContributorCache: () => ContributorCache$1 });
17907
- var DEFAULT_TTL_MS$1, DEFAULT_MAX_ENTRIES, ContributorCache$1;
18022
+ var DEFAULT_TTL_MS$1, DEFAULT_MAX_ENTRIES$1, ContributorCache$1;
17908
18023
  var init_contributor_cache = __esmMin((() => {
17909
18024
  DEFAULT_TTL_MS$1 = 900 * 1e3;
17910
- DEFAULT_MAX_ENTRIES = 1e4;
18025
+ DEFAULT_MAX_ENTRIES$1 = 1e4;
17911
18026
  ContributorCache$1 = class {
17912
18027
  cache;
17913
18028
  constructor(options) {
17914
18029
  this.cache = new LRUCache({
17915
- max: options?.maxEntries ?? DEFAULT_MAX_ENTRIES,
18030
+ max: options?.maxEntries ?? DEFAULT_MAX_ENTRIES$1,
17916
18031
  ttl: options?.ttlMs ?? DEFAULT_TTL_MS$1
17917
18032
  });
17918
18033
  }
@@ -27326,10 +27441,10 @@ var init_admin_sources = __esmMin((() => {
27326
27441
  //#endregion
27327
27442
  //#region src/db/migrations/001_initial.ts
27328
27443
  var _001_initial_exports = /* @__PURE__ */ __exportAll({
27329
- down: () => down$23,
27330
- up: () => up$23
27444
+ down: () => down$24,
27445
+ up: () => up$24
27331
27446
  });
27332
- async function up$23(db) {
27447
+ async function up$24(db) {
27333
27448
  for (const stmt of DDL_STATEMENTS) await sql.raw(stmt).execute(db);
27334
27449
  await sql`
27335
27450
  INSERT INTO cluster_meta (key, value)
@@ -27351,7 +27466,7 @@ async function up$23(db) {
27351
27466
  * Rollback drops everything created above. Uses CASCADE on table drops to cut
27352
27467
  * through the FK graph without relying on exact topological order.
27353
27468
  */
27354
- async function down$23(db) {
27469
+ async function down$24(db) {
27355
27470
  for (const [trig, tbl] of [["source_secrets_change_trigger", "scoped_secrets"], ["sources_change_trigger", "sources"]]) await sql.raw(`DROP TRIGGER IF EXISTS ${trig} ON public.${tbl}`).execute(db);
27356
27471
  for (const table of [
27357
27472
  "workflow_registrations",
@@ -28071,8 +28186,8 @@ var init__001_initial = __esmMin((() => {
28071
28186
  //#endregion
28072
28187
  //#region src/db/migrations/002_config_versions_key_version.ts
28073
28188
  var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
28074
- down: () => down$22,
28075
- up: () => up$22
28189
+ down: () => down$23,
28190
+ up: () => up$23
28076
28191
  });
28077
28192
  /**
28078
28193
  * Add key_version column to config_versions so that sensitive-field encryption
@@ -28086,13 +28201,13 @@ var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
28086
28201
  * No index is needed: rotation does a full-table scan; reads are by
28087
28202
  * `version` primary key and never filter on `key_version`.
28088
28203
  */
28089
- async function up$22(db) {
28204
+ async function up$23(db) {
28090
28205
  await sql`
28091
28206
  ALTER TABLE public.config_versions
28092
28207
  ADD COLUMN key_version integer NOT NULL DEFAULT 1
28093
28208
  `.execute(db);
28094
28209
  }
28095
- async function down$22(db) {
28210
+ async function down$23(db) {
28096
28211
  await sql`
28097
28212
  ALTER TABLE public.config_versions
28098
28213
  DROP COLUMN key_version
@@ -28102,8 +28217,8 @@ var init__002_config_versions_key_version = __esmMin((() => {}));
28102
28217
  //#endregion
28103
28218
  //#region src/db/migrations/003_access_log.ts
28104
28219
  var _003_access_log_exports = /* @__PURE__ */ __exportAll({
28105
- down: () => down$21,
28106
- up: () => up$21
28220
+ down: () => down$22,
28221
+ up: () => up$22
28107
28222
  });
28108
28223
  /**
28109
28224
  * Access log: one row per read or orchestrator-admin mutation attributable
@@ -28127,7 +28242,7 @@ var _003_access_log_exports = /* @__PURE__ */ __exportAll({
28127
28242
  * Retention is TTL-based via expires_at; packages/orchestrator/src/queue/
28128
28243
  * cleanup.ts picks up the prune pass.
28129
28244
  */
28130
- async function up$21(db) {
28245
+ async function up$22(db) {
28131
28246
  await sql`
28132
28247
  CREATE TABLE public.access_log (
28133
28248
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -28165,21 +28280,21 @@ async function up$21(db) {
28165
28280
  ON public.access_log (actor_type, actor_id, created_at DESC)
28166
28281
  `.execute(db);
28167
28282
  }
28168
- async function down$21(db) {
28283
+ async function down$22(db) {
28169
28284
  await sql`DROP TABLE IF EXISTS public.access_log`.execute(db);
28170
28285
  }
28171
28286
  var init__003_access_log = __esmMin((() => {}));
28172
28287
  //#endregion
28173
28288
  //#region src/db/migrations/004_rename_bundle_to_source.ts
28174
28289
  var _004_rename_bundle_to_source_exports = /* @__PURE__ */ __exportAll({
28175
- down: () => down$20,
28176
- up: () => up$20
28290
+ down: () => down$21,
28291
+ up: () => up$21
28177
28292
  });
28178
- async function up$20(db) {
28293
+ async function up$21(db) {
28179
28294
  await db.schema.alterTable("dispatch_queue").renameColumn("bundle_url", "source_tar_url").execute();
28180
28295
  await db.schema.alterTable("dispatch_queue").renameColumn("bundle_hash", "source_tar_hash").execute();
28181
28296
  }
28182
- async function down$20(db) {
28297
+ async function down$21(db) {
28183
28298
  await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_url", "bundle_url").execute();
28184
28299
  await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_hash", "bundle_hash").execute();
28185
28300
  }
@@ -28187,8 +28302,8 @@ var init__004_rename_bundle_to_source = __esmMin((() => {}));
28187
28302
  //#endregion
28188
28303
  //#region src/db/migrations/005_cold_store_chunk_counter.ts
28189
28304
  var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
28190
- down: () => down$19,
28191
- up: () => up$19
28305
+ down: () => down$20,
28306
+ up: () => up$20
28192
28307
  });
28193
28308
  /**
28194
28309
  * Cold-store chunk counter table.
@@ -28205,7 +28320,7 @@ var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
28205
28320
  *
28206
28321
  * sections 5 and 8.
28207
28322
  */
28208
- async function up$19(db) {
28323
+ async function up$20(db) {
28209
28324
  await sql`
28210
28325
  CREATE TABLE public.cold_store_chunk_counts (
28211
28326
  db TEXT NOT NULL,
@@ -28223,15 +28338,15 @@ async function up$19(db) {
28223
28338
  ON public.cold_store_chunk_counts (db, table_name)
28224
28339
  `.execute(db);
28225
28340
  }
28226
- async function down$19(db) {
28341
+ async function down$20(db) {
28227
28342
  await sql`DROP TABLE IF EXISTS public.cold_store_chunk_counts`.execute(db);
28228
28343
  }
28229
28344
  var init__005_cold_store_chunk_counter = __esmMin((() => {}));
28230
28345
  //#endregion
28231
28346
  //#region src/db/migrations/006_runs_jobs_steps_archived_at.ts
28232
28347
  var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
28233
- down: () => down$18,
28234
- up: () => up$18
28348
+ down: () => down$19,
28349
+ up: () => up$19
28235
28350
  });
28236
28351
  /**
28237
28352
  * `execution_runs` / `execution_jobs` / `execution_steps` cold-store
@@ -28267,7 +28382,7 @@ var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
28267
28382
  * - `idx_execution_jobs_routing_key_created (routing_key, created_at)`
28268
28383
  * - `idx_execution_steps_routing_key_created (routing_key, created_at)`
28269
28384
  */
28270
- async function up$18(db) {
28385
+ async function up$19(db) {
28271
28386
  await sql`
28272
28387
  ALTER TABLE public.execution_runs
28273
28388
  ADD COLUMN archived_at TIMESTAMPTZ NULL,
@@ -28312,7 +28427,7 @@ async function up$18(db) {
28312
28427
  ON public.execution_steps (routing_key, created_at)
28313
28428
  `.execute(db);
28314
28429
  }
28315
- async function down$18(db) {
28430
+ async function down$19(db) {
28316
28431
  await sql`DROP INDEX IF EXISTS public.idx_execution_steps_routing_key_created`.execute(db);
28317
28432
  await sql`
28318
28433
  ALTER TABLE public.execution_steps
@@ -28338,8 +28453,8 @@ var init__006_runs_jobs_steps_archived_at = __esmMin((() => {}));
28338
28453
  //#endregion
28339
28454
  //#region src/db/migrations/007_audit_logs_archived_at.ts
28340
28455
  var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
28341
- down: () => down$17,
28342
- up: () => up$17
28456
+ down: () => down$18,
28457
+ up: () => up$18
28343
28458
  });
28344
28459
  /**
28345
28460
  * `secret_audit_log` and `access_log` cold-store schema additions, plus
@@ -28379,7 +28494,7 @@ var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
28379
28494
  * `down()` would not have meaningful retention bounds. Acceptable for
28380
28495
  * staging.
28381
28496
  */
28382
- async function up$17(db) {
28497
+ async function up$18(db) {
28383
28498
  await sql`
28384
28499
  ALTER TABLE public.secret_audit_log
28385
28500
  ADD COLUMN archived_at TIMESTAMPTZ NULL,
@@ -28400,7 +28515,7 @@ async function up$17(db) {
28400
28515
  DROP COLUMN IF EXISTS expires_at
28401
28516
  `.execute(db);
28402
28517
  }
28403
- async function down$17(db) {
28518
+ async function down$18(db) {
28404
28519
  await sql`
28405
28520
  ALTER TABLE public.access_log
28406
28521
  ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '90 days')
@@ -28429,8 +28544,8 @@ var init__007_audit_logs_archived_at = __esmMin((() => {}));
28429
28544
  //#endregion
28430
28545
  //#region src/db/migrations/008_event_log_archived_at.ts
28431
28546
  var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
28432
- down: () => down$16,
28433
- up: () => up$16
28547
+ down: () => down$17,
28548
+ up: () => up$17
28434
28549
  });
28435
28550
  /**
28436
28551
  * `event_log` cold-store schema additions plus removal of the
@@ -28468,7 +28583,7 @@ var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
28468
28583
  * — best effort; rows inserted between `up()` and a hypothetical
28469
28584
  * `down()` would not have meaningful retention bounds.
28470
28585
  */
28471
- async function up$16(db) {
28586
+ async function up$17(db) {
28472
28587
  await sql`
28473
28588
  ALTER TABLE public.event_log
28474
28589
  ADD COLUMN archived_at TIMESTAMPTZ NULL,
@@ -28484,7 +28599,7 @@ async function up$16(db) {
28484
28599
  DROP COLUMN IF EXISTS expires_at
28485
28600
  `.execute(db);
28486
28601
  }
28487
- async function down$16(db) {
28602
+ async function down$17(db) {
28488
28603
  await sql`
28489
28604
  ALTER TABLE public.event_log
28490
28605
  ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '30 days')
@@ -28504,8 +28619,8 @@ var init__008_event_log_archived_at = __esmMin((() => {}));
28504
28619
  //#endregion
28505
28620
  //#region src/db/migrations/009_access_log_trigram.ts
28506
28621
  var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
28507
- down: () => down$15,
28508
- up: () => up$15
28622
+ down: () => down$16,
28623
+ up: () => up$16
28509
28624
  });
28510
28625
  /**
28511
28626
  * Trigram (pg_trgm) index on access_log.error_message for the federated
@@ -28518,7 +28633,7 @@ var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
28518
28633
  * EXISTS` are both safe to re-run. No CONCURRENTLY because Kysely runs
28519
28634
  * migrations inside a transaction; the lock is brief on a sampled table.
28520
28635
  */
28521
- async function up$15(db) {
28636
+ async function up$16(db) {
28522
28637
  await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
28523
28638
  await sql`
28524
28639
  CREATE INDEX IF NOT EXISTS access_log_error_message_trgm_idx
@@ -28527,15 +28642,15 @@ async function up$15(db) {
28527
28642
  WHERE error_message IS NOT NULL
28528
28643
  `.execute(db);
28529
28644
  }
28530
- async function down$15(db) {
28645
+ async function down$16(db) {
28531
28646
  await sql`DROP INDEX IF EXISTS public.access_log_error_message_trgm_idx`.execute(db);
28532
28647
  }
28533
28648
  var init__009_access_log_trigram = __esmMin((() => {}));
28534
28649
  //#endregion
28535
28650
  //#region src/db/migrations/010_cold_store_chunks.ts
28536
28651
  var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
28537
- down: () => down$14,
28538
- up: () => up$14
28652
+ down: () => down$15,
28653
+ up: () => up$15
28539
28654
  });
28540
28655
  /**
28541
28656
  * Cold-store chunk index — Phase 2 (cold-store purge).
@@ -28567,7 +28682,7 @@ var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
28567
28682
  * forever. Adapters that don't opt into per-bucket archival via
28568
28683
  * `coldTtlDays` don't insert here either.
28569
28684
  */
28570
- async function up$14(db) {
28685
+ async function up$15(db) {
28571
28686
  await sql`
28572
28687
  CREATE TABLE public.cold_store_chunks (
28573
28688
  db TEXT NOT NULL,
@@ -28594,15 +28709,15 @@ async function up$14(db) {
28594
28709
  ON public.cold_store_chunks (db, table_name, tenant_id, archived_at DESC)
28595
28710
  `.execute(db);
28596
28711
  }
28597
- async function down$14(db) {
28712
+ async function down$15(db) {
28598
28713
  await sql`DROP TABLE IF EXISTS public.cold_store_chunks`.execute(db);
28599
28714
  }
28600
28715
  var init__010_cold_store_chunks = __esmMin((() => {}));
28601
28716
  //#endregion
28602
28717
  //#region src/db/migrations/011_drop_source_secrets_notify.ts
28603
28718
  var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
28604
- down: () => down$13,
28605
- up: () => up$13
28719
+ down: () => down$14,
28720
+ up: () => up$14
28606
28721
  });
28607
28722
  /**
28608
28723
  * Drop the `source_secrets_change_trigger` and the
@@ -28621,11 +28736,11 @@ var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
28621
28736
  * in `001_initial.ts`. They wake up no consumer until the `WebhookSecretManager`
28622
28737
  * is restored, so this migration is safe to roll back.
28623
28738
  */
28624
- async function up$13(db) {
28739
+ async function up$14(db) {
28625
28740
  await sql`DROP TRIGGER IF EXISTS source_secrets_change_trigger ON public.scoped_secrets`.execute(db);
28626
28741
  await sql`DROP FUNCTION IF EXISTS public.notify_source_secrets_change() CASCADE`.execute(db);
28627
28742
  }
28628
- async function down$13(db) {
28743
+ async function down$14(db) {
28629
28744
  await sql`
28630
28745
  CREATE OR REPLACE FUNCTION public.notify_source_secrets_change() RETURNS trigger
28631
28746
  LANGUAGE plpgsql
@@ -28665,8 +28780,8 @@ var init__011_drop_source_secrets_notify = __esmMin((() => {}));
28665
28780
  //#endregion
28666
28781
  //#region src/db/migrations/012_peer_credentials_active_uniq.ts
28667
28782
  var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
28668
- down: () => down$12,
28669
- up: () => up$12
28783
+ down: () => down$13,
28784
+ up: () => up$13
28670
28785
  });
28671
28786
  /**
28672
28787
  * Add a partial unique index on `peer_credentials (instance_id) WHERE
@@ -28692,7 +28807,7 @@ var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
28692
28807
  * `down()` only drops the index; it does NOT undo the dedupe (there's no
28693
28808
  * safe way to recreate revoked rows, and the dedupe is monotonic).
28694
28809
  */
28695
- async function up$12(db) {
28810
+ async function up$13(db) {
28696
28811
  await sql`
28697
28812
  UPDATE public.peer_credentials
28698
28813
  SET revoked_at = NOW()
@@ -28710,15 +28825,15 @@ async function up$12(db) {
28710
28825
  WHERE revoked_at IS NULL
28711
28826
  `.execute(db);
28712
28827
  }
28713
- async function down$12(db) {
28828
+ async function down$13(db) {
28714
28829
  await sql`DROP INDEX IF EXISTS public.peer_credentials_active_uniq`.execute(db);
28715
28830
  }
28716
28831
  var init__012_peer_credentials_active_uniq = __esmMin((() => {}));
28717
28832
  //#endregion
28718
28833
  //#region src/db/migrations/013_execution_log_bytes.ts
28719
28834
  var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
28720
- down: () => down$11,
28721
- up: () => up$11
28835
+ down: () => down$12,
28836
+ up: () => up$12
28722
28837
  });
28723
28838
  /**
28724
28839
  * Add `log_bytes BIGINT NOT NULL DEFAULT 0` columns to `execution_runs` and
@@ -28737,7 +28852,7 @@ var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
28737
28852
  *
28738
28853
  * Idempotent (`ADD COLUMN IF NOT EXISTS`).
28739
28854
  */
28740
- async function up$11(db) {
28855
+ async function up$12(db) {
28741
28856
  await sql`
28742
28857
  ALTER TABLE public.execution_runs
28743
28858
  ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
@@ -28747,7 +28862,7 @@ async function up$11(db) {
28747
28862
  ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
28748
28863
  `.execute(db);
28749
28864
  }
28750
- async function down$11(db) {
28865
+ async function down$12(db) {
28751
28866
  await sql`ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS log_bytes`.execute(db);
28752
28867
  await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS log_bytes`.execute(db);
28753
28868
  }
@@ -28755,8 +28870,8 @@ var init__013_execution_log_bytes = __esmMin((() => {}));
28755
28870
  //#endregion
28756
28871
  //#region src/db/migrations/014_kici_events_lease_retry.ts
28757
28872
  var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
28758
- down: () => down$10,
28759
- up: () => up$10
28873
+ down: () => down$11,
28874
+ up: () => up$11
28760
28875
  });
28761
28876
  /**
28762
28877
  * Add lease + retry + DLQ columns to `kici_events` so the EventRouter can
@@ -28790,7 +28905,7 @@ var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
28790
28905
  *
28791
28906
  * Idempotent (`ADD COLUMN IF NOT EXISTS` + `CREATE INDEX IF NOT EXISTS`).
28792
28907
  */
28793
- async function up$10(db) {
28908
+ async function up$11(db) {
28794
28909
  await sql`
28795
28910
  ALTER TABLE public.kici_events
28796
28911
  ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ,
@@ -28821,7 +28936,7 @@ async function up$10(db) {
28821
28936
  WHERE dlq_at IS NOT NULL
28822
28937
  `.execute(db);
28823
28938
  }
28824
- async function down$10(db) {
28939
+ async function down$11(db) {
28825
28940
  await sql`DROP INDEX IF EXISTS public.idx_kici_events_dlq`.execute(db);
28826
28941
  await sql`DROP INDEX IF EXISTS public.idx_kici_events_lease_expired`.execute(db);
28827
28942
  await sql`DROP INDEX IF EXISTS public.idx_kici_events_retry_due`.execute(db);
@@ -28840,8 +28955,8 @@ var init__014_kici_events_lease_retry = __esmMin((() => {}));
28840
28955
  //#endregion
28841
28956
  //#region src/db/migrations/015_org_settings_customer_scoped.ts
28842
28957
  var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
28843
- down: () => down$9,
28844
- up: () => up$9
28958
+ down: () => down$10,
28959
+ up: () => up$10
28845
28960
  });
28846
28961
  /**
28847
28962
  * Org-scope `org_settings` and qualify each glob entry by source.
@@ -28869,7 +28984,7 @@ var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
28869
28984
  * Idempotent: a re-run on an already-migrated DB sees `customer_id` exists
28870
28985
  * and the list columns are already jsonb, so it is a no-op.
28871
28986
  */
28872
- async function up$9(db) {
28987
+ async function up$10(db) {
28873
28988
  if ((await sql`
28874
28989
  SELECT EXISTS (
28875
28990
  SELECT 1 FROM information_schema.columns
@@ -28994,7 +29109,7 @@ async function up$9(db) {
28994
29109
  await sql`DROP TABLE _org_settings_merged`.execute(db);
28995
29110
  await sql`DROP TABLE _org_settings_stage`.execute(db);
28996
29111
  }
28997
- async function down$9(db) {
29112
+ async function down$10(db) {
28998
29113
  if (!(await sql`
28999
29114
  SELECT EXISTS (
29000
29115
  SELECT 1 FROM information_schema.columns
@@ -29020,8 +29135,8 @@ var init__015_org_settings_customer_scoped = __esmMin((() => {}));
29020
29135
  //#endregion
29021
29136
  //#region src/db/migrations/016_org_settings_allow_http_npm.ts
29022
29137
  var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
29023
- down: () => down$8,
29024
- up: () => up$8
29138
+ down: () => down$9,
29139
+ up: () => up$9
29025
29140
  });
29026
29141
  /**
29027
29142
  * Add `org_settings.allow_http_npm_registries boolean NOT NULL DEFAULT false`.
@@ -29034,7 +29149,7 @@ var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
29034
29149
  *
29035
29150
  * Idempotent: a re-run on a DB that already has the column is a no-op.
29036
29151
  */
29037
- async function up$8(db) {
29152
+ async function up$9(db) {
29038
29153
  if ((await sql`
29039
29154
  SELECT EXISTS (
29040
29155
  SELECT 1 FROM information_schema.columns
@@ -29048,7 +29163,7 @@ async function up$8(db) {
29048
29163
  ADD COLUMN allow_http_npm_registries boolean NOT NULL DEFAULT false
29049
29164
  `.execute(db);
29050
29165
  }
29051
- async function down$8(db) {
29166
+ async function down$9(db) {
29052
29167
  await sql`
29053
29168
  ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_http_npm_registries
29054
29169
  `.execute(db);
@@ -29057,13 +29172,13 @@ var init__016_org_settings_allow_http_npm = __esmMin((() => {}));
29057
29172
  //#endregion
29058
29173
  //#region src/db/migrations/017_org_id_widen.ts
29059
29174
  var _017_org_id_widen_exports = /* @__PURE__ */ __exportAll({
29060
- down: () => down$7,
29061
- up: () => up$7
29175
+ down: () => down$8,
29176
+ up: () => up$8
29062
29177
  });
29063
- async function up$7(db) {
29178
+ async function up$8(db) {
29064
29179
  for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(16)`).execute(db);
29065
29180
  }
29066
- async function down$7(db) {
29181
+ async function down$8(db) {
29067
29182
  for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(12)`).execute(db);
29068
29183
  }
29069
29184
  var ORG_ID_TABLES$1;
@@ -29082,14 +29197,14 @@ var init__017_org_id_widen = __esmMin((() => {
29082
29197
  //#endregion
29083
29198
  //#region src/db/migrations/018_org_id_prefix_backfill.ts
29084
29199
  var _018_org_id_prefix_backfill_exports = /* @__PURE__ */ __exportAll({
29085
- down: () => down$6,
29086
- up: () => up$6
29200
+ down: () => down$7,
29201
+ up: () => up$7
29087
29202
  });
29088
- async function up$6(db) {
29203
+ async function up$7(db) {
29089
29204
  for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = 'org_' || org_id WHERE org_id <> 'kici-admin' AND org_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
29090
29205
  for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = 'org_' || customer_id WHERE customer_id <> 'kici-admin' AND customer_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
29091
29206
  }
29092
- async function down$6(db) {
29207
+ async function down$7(db) {
29093
29208
  for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = substring(customer_id from 5) WHERE customer_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
29094
29209
  for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = substring(org_id from 5) WHERE org_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
29095
29210
  }
@@ -29115,8 +29230,8 @@ var init__018_org_id_prefix_backfill = __esmMin((() => {
29115
29230
  //#endregion
29116
29231
  //#region src/db/migrations/019_generic_sources_change_notify.ts
29117
29232
  var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
29118
- down: () => down$5,
29119
- up: () => up$5
29233
+ down: () => down$6,
29234
+ up: () => up$6
29120
29235
  });
29121
29236
  /**
29122
29237
  * Add a Postgres trigger on `generic_webhook_sources` that emits
@@ -29139,7 +29254,7 @@ var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
29139
29254
  * (`notify_sources_change()` + `sources_change_trigger`, defined in
29140
29255
  * `001_initial.ts`).
29141
29256
  */
29142
- async function up$5(db) {
29257
+ async function up$6(db) {
29143
29258
  await sql`
29144
29259
  CREATE FUNCTION public.notify_generic_sources_change() RETURNS trigger
29145
29260
  LANGUAGE plpgsql
@@ -29160,7 +29275,7 @@ async function up$5(db) {
29160
29275
  FOR EACH ROW EXECUTE FUNCTION public.notify_generic_sources_change()
29161
29276
  `.execute(db);
29162
29277
  }
29163
- async function down$5(db) {
29278
+ async function down$6(db) {
29164
29279
  await sql`DROP TRIGGER IF EXISTS generic_sources_change_trigger ON public.generic_webhook_sources`.execute(db);
29165
29280
  await sql`DROP FUNCTION IF EXISTS public.notify_generic_sources_change()`.execute(db);
29166
29281
  }
@@ -29168,8 +29283,8 @@ var init__019_generic_sources_change_notify = __esmMin((() => {}));
29168
29283
  //#endregion
29169
29284
  //#region src/db/migrations/020_org_settings_dashboard_write_policy.ts
29170
29285
  var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportAll({
29171
- down: () => down$4,
29172
- up: () => up$4
29286
+ down: () => down$5,
29287
+ up: () => up$5
29173
29288
  });
29174
29289
  /**
29175
29290
  * Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
@@ -29184,7 +29299,7 @@ var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportA
29184
29299
  *
29185
29300
  * Idempotent: a re-run on a DB that already has the column is a no-op.
29186
29301
  */
29187
- async function up$4(db) {
29302
+ async function up$5(db) {
29188
29303
  if ((await sql`
29189
29304
  SELECT EXISTS (
29190
29305
  SELECT 1 FROM information_schema.columns
@@ -29198,7 +29313,7 @@ async function up$4(db) {
29198
29313
  ADD COLUMN dashboard_write_policy jsonb NOT NULL DEFAULT '{}'::jsonb
29199
29314
  `.execute(db);
29200
29315
  }
29201
- async function down$4(db) {
29316
+ async function down$5(db) {
29202
29317
  await sql`
29203
29318
  ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dashboard_write_policy
29204
29319
  `.execute(db);
@@ -29207,8 +29322,8 @@ var init__020_org_settings_dashboard_write_policy = __esmMin((() => {}));
29207
29322
  //#endregion
29208
29323
  //#region src/db/migrations/021_check_run_tracking.ts
29209
29324
  var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
29210
- down: () => down$3,
29211
- up: () => up$3
29325
+ down: () => down$4,
29326
+ up: () => up$4
29212
29327
  });
29213
29328
  /**
29214
29329
  * Add `check_run_tracking` table for HA-safe check-run state persistence.
@@ -29232,7 +29347,7 @@ var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
29232
29347
  *
29233
29348
  * Idempotent: a re-run on a DB that already has the table is a no-op.
29234
29349
  */
29235
- async function up$3(db) {
29350
+ async function up$4(db) {
29236
29351
  if ((await sql`
29237
29352
  SELECT EXISTS (
29238
29353
  SELECT 1 FROM information_schema.tables
@@ -29263,15 +29378,15 @@ async function up$3(db) {
29263
29378
  WHERE run_id IS NOT NULL
29264
29379
  `.execute(db);
29265
29380
  }
29266
- async function down$3(db) {
29381
+ async function down$4(db) {
29267
29382
  await sql`DROP TABLE IF EXISTS public.check_run_tracking`.execute(db);
29268
29383
  }
29269
29384
  var init__021_check_run_tracking = __esmMin((() => {}));
29270
29385
  //#endregion
29271
29386
  //#region src/db/migrations/022_scaler_manager_state.ts
29272
29387
  var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
29273
- down: () => down$2,
29274
- up: () => up$2
29388
+ down: () => down$3,
29389
+ up: () => up$3
29275
29390
  });
29276
29391
  /**
29277
29392
  * Add three tables persisting `ScalerManager` per-coord state:
@@ -29298,7 +29413,7 @@ var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
29298
29413
  * Idempotent: a re-run on a DB that already has any of these tables
29299
29414
  * leaves the existing one alone.
29300
29415
  */
29301
- async function up$2(db) {
29416
+ async function up$3(db) {
29302
29417
  const tableExists = async (name) => {
29303
29418
  return (await sql`
29304
29419
  SELECT EXISTS (
@@ -29348,7 +29463,7 @@ async function up$2(db) {
29348
29463
  `.execute(db);
29349
29464
  }
29350
29465
  }
29351
- async function down$2(db) {
29466
+ async function down$3(db) {
29352
29467
  await sql`DROP TABLE IF EXISTS public.scaler_reservations`.execute(db);
29353
29468
  await sql`DROP TABLE IF EXISTS public.scaler_agent_jobs`.execute(db);
29354
29469
  await sql`DROP TABLE IF EXISTS public.scaler_spawning_agents`.execute(db);
@@ -29357,8 +29472,8 @@ var init__022_scaler_manager_state = __esmMin((() => {}));
29357
29472
  //#endregion
29358
29473
  //#region src/db/migrations/023_dispatch_queue_recovery_deadline.ts
29359
29474
  var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll({
29360
- down: () => down$1,
29361
- up: () => up$1
29475
+ down: () => down$2,
29476
+ up: () => up$2
29362
29477
  });
29363
29478
  /**
29364
29479
  * Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
@@ -29382,7 +29497,7 @@ var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll(
29382
29497
  * Idempotent: re-running on a DB that already has either column is a
29383
29498
  * no-op.
29384
29499
  */
29385
- async function up$1(db) {
29500
+ async function up$2(db) {
29386
29501
  const colExists = async (name) => {
29387
29502
  return (await sql`
29388
29503
  SELECT EXISTS (
@@ -29407,7 +29522,7 @@ async function up$1(db) {
29407
29522
  WHERE recovery_deadline IS NOT NULL
29408
29523
  `.execute(db);
29409
29524
  }
29410
- async function down$1(db) {
29525
+ async function down$2(db) {
29411
29526
  await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_recovery_deadline`.execute(db);
29412
29527
  await sql`
29413
29528
  ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS recovery_agent_id
@@ -29420,8 +29535,8 @@ var init__023_dispatch_queue_recovery_deadline = __esmMin((() => {}));
29420
29535
  //#endregion
29421
29536
  //#region src/db/migrations/024_dispatch_queue_provisioning_error.ts
29422
29537
  var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll({
29423
- down: () => down,
29424
- up: () => up
29538
+ down: () => down$1,
29539
+ up: () => up$1
29425
29540
  });
29426
29541
  /**
29427
29542
  * Add `dispatch_queue.last_provisioning_error TEXT` recording the most
@@ -29437,7 +29552,7 @@ var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll
29437
29552
  *
29438
29553
  * Idempotent: re-running on a DB that already has the column is a no-op.
29439
29554
  */
29440
- async function up(db) {
29555
+ async function up$1(db) {
29441
29556
  const colExists = async (name) => {
29442
29557
  return (await sql`
29443
29558
  SELECT EXISTS (
@@ -29453,13 +29568,60 @@ async function up(db) {
29453
29568
  ADD COLUMN last_provisioning_error TEXT
29454
29569
  `.execute(db);
29455
29570
  }
29456
- async function down(db) {
29571
+ async function down$1(db) {
29457
29572
  await sql`
29458
29573
  ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS last_provisioning_error
29459
29574
  `.execute(db);
29460
29575
  }
29461
29576
  var init__024_dispatch_queue_provisioning_error = __esmMin((() => {}));
29462
29577
  //#endregion
29578
+ //#region src/db/migrations/025_init_failure.ts
29579
+ var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
29580
+ down: () => down,
29581
+ up: () => up
29582
+ });
29583
+ /**
29584
+ * Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
29585
+ *
29586
+ * Presence of this column on a row means the run/job never executed a step
29587
+ * because of an init-phase failure; absence (NULL) means a normal run.
29588
+ * Shape on the wire is `InitFailure` from `@kici-dev/engine`. The dashboard
29589
+ * reads this column directly so it can render the right banner without
29590
+ * round-tripping to the orchestrator (which may be offline).
29591
+ *
29592
+ * Idempotent: re-running on a DB that already has either column is a no-op
29593
+ * for that column.
29594
+ */
29595
+ async function up(db) {
29596
+ const colExists = async (table, name) => {
29597
+ return (await sql`
29598
+ SELECT EXISTS (
29599
+ SELECT 1 FROM information_schema.columns
29600
+ WHERE table_schema = 'public'
29601
+ AND table_name = ${table}
29602
+ AND column_name = ${name}
29603
+ ) AS exists
29604
+ `.execute(db)).rows[0]?.exists ?? false;
29605
+ };
29606
+ if (!await colExists("execution_runs", "init_failure")) await sql`
29607
+ ALTER TABLE public.execution_runs
29608
+ ADD COLUMN init_failure JSONB DEFAULT NULL
29609
+ `.execute(db);
29610
+ if (!await colExists("execution_jobs", "init_failure")) await sql`
29611
+ ALTER TABLE public.execution_jobs
29612
+ ADD COLUMN init_failure JSONB DEFAULT NULL
29613
+ `.execute(db);
29614
+ }
29615
+ async function down(db) {
29616
+ await sql`
29617
+ ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS init_failure
29618
+ `.execute(db);
29619
+ await sql`
29620
+ ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS init_failure
29621
+ `.execute(db);
29622
+ }
29623
+ var init__025_init_failure = __esmMin((() => {}));
29624
+ //#endregion
29463
29625
  //#region src/db/migration-provider.ts
29464
29626
  function createMigrationProvider() {
29465
29627
  return { async getMigrations() {
@@ -29487,7 +29649,8 @@ function createMigrationProvider() {
29487
29649
  "021_check_run_tracking": _021_check_run_tracking_exports,
29488
29650
  "022_scaler_manager_state": _022_scaler_manager_state_exports,
29489
29651
  "023_dispatch_queue_recovery_deadline": _023_dispatch_queue_recovery_deadline_exports,
29490
- "024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports
29652
+ "024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports,
29653
+ "025_init_failure": _025_init_failure_exports
29491
29654
  };
29492
29655
  } };
29493
29656
  }
@@ -29516,6 +29679,7 @@ var init_migration_provider = __esmMin((() => {
29516
29679
  init__022_scaler_manager_state();
29517
29680
  init__023_dispatch_queue_recovery_deadline();
29518
29681
  init__024_dispatch_queue_provisioning_error();
29682
+ init__025_init_failure();
29519
29683
  }));
29520
29684
  //#endregion
29521
29685
  //#region src/db/migrator.ts
@@ -34217,15 +34381,15 @@ var init_admin_config = __esmMin((() => {
34217
34381
  function createHealthRoutes$1(deps = {}) {
34218
34382
  return createHealthRoutes({
34219
34383
  livenessInfo: () => ({
34220
- version: "0.1.13",
34221
- buildDate: "2026-05-28T10:29:07.719Z",
34222
- buildCommit: "b3ee727cb",
34223
- sdkVersion: "0.1.13",
34384
+ version: "0.1.14",
34385
+ buildDate: "2026-05-30T03:04:39.672Z",
34386
+ buildCommit: "0fc7a43c9",
34387
+ sdkVersion: "0.1.14",
34224
34388
  sdkBundleHash: "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128",
34225
- sharedVersion: "0.1.13",
34226
- sharedBundleHash: "36a23a454fd75ec35f258d39f254543ec0451c6b543fe3618ce427e6e6369aae",
34227
- engineVersion: "0.1.13",
34228
- engineBundleHash: "4ab08e63bcb8558b1191c5d3ab8d58b71950b77d911af56c9955ae1d787c4302"
34389
+ sharedVersion: "0.1.14",
34390
+ sharedBundleHash: "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4",
34391
+ engineVersion: "0.1.14",
34392
+ engineBundleHash: "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932"
34229
34393
  }),
34230
34394
  readinessCheck: deps.db ? async () => {
34231
34395
  const checks = {};
@@ -34259,7 +34423,7 @@ function createCapabilitiesRoutes() {
34259
34423
  const app = new Hono();
34260
34424
  app.get("/api/v1/capabilities", (c) => {
34261
34425
  const manifest = {
34262
- orchestratorVersion: "0.1.13",
34426
+ orchestratorVersion: "0.1.14",
34263
34427
  protocolVersion: PROTOCOL_VERSION,
34264
34428
  minProtocolVersion: MIN_PROTOCOL_VERSION
34265
34429
  };
@@ -34561,6 +34725,61 @@ var init_certs = __esmMin((() => {
34561
34725
  WARN_DAYS = 30;
34562
34726
  }));
34563
34727
  //#endregion
34728
+ //#region src/diagnostics/checks/scaler.ts
34729
+ function truncate(s) {
34730
+ return s.length > MAX_DETAIL_LEN ? `${s.slice(0, MAX_DETAIL_LEN - 1)}…` : s;
34731
+ }
34732
+ async function checkScalerProvisioning(deps) {
34733
+ const start = Date.now();
34734
+ const mgr = deps.scalerManager;
34735
+ const backends = mgr?.getStatus().backends ?? [];
34736
+ if (!mgr || backends.length === 0) return [{
34737
+ name: "scaler",
34738
+ status: "pass",
34739
+ message: "No scaler backends configured",
34740
+ durationMs: Date.now() - start
34741
+ }];
34742
+ const recent = mgr.recentSpawnFailures(SCALER_FAILURE_WINDOW_MS, Date.now());
34743
+ return backends.map((backend) => {
34744
+ const summary = recent.get(backend.name);
34745
+ const durationMs = Date.now() - start;
34746
+ if (!summary || summary.boundCount + summary.unboundCount === 0) return {
34747
+ name: `scaler:${backend.name}`,
34748
+ status: "pass",
34749
+ message: "0 spawn failures in last 5m",
34750
+ details: {
34751
+ windowMs: SCALER_FAILURE_WINDOW_MS,
34752
+ backendType: backend.type,
34753
+ boundCount: 0,
34754
+ unboundCount: 0
34755
+ },
34756
+ durationMs
34757
+ };
34758
+ const total = summary.boundCount + summary.unboundCount;
34759
+ const status = summary.boundCount > 0 ? "fail" : "warn";
34760
+ const message = `${total} spawn failures in last 5m (${summary.boundCount} bound, ${summary.unboundCount} warm-pool; last: ${truncate(summary.lastError)})`;
34761
+ return {
34762
+ name: `scaler:${backend.name}`,
34763
+ status,
34764
+ message,
34765
+ details: {
34766
+ windowMs: SCALER_FAILURE_WINDOW_MS,
34767
+ backendType: summary.backendType,
34768
+ boundCount: summary.boundCount,
34769
+ unboundCount: summary.unboundCount,
34770
+ lastError: summary.lastError,
34771
+ lastAtMs: summary.lastAtMs
34772
+ },
34773
+ durationMs
34774
+ };
34775
+ });
34776
+ }
34777
+ var SCALER_FAILURE_WINDOW_MS, MAX_DETAIL_LEN;
34778
+ var init_scaler$1 = __esmMin((() => {
34779
+ SCALER_FAILURE_WINDOW_MS = 300 * 1e3;
34780
+ MAX_DETAIL_LEN = 120;
34781
+ }));
34782
+ //#endregion
34564
34783
  //#region src/diagnostics/checks/index.ts
34565
34784
  var defaultChecks;
34566
34785
  var init_checks = __esmMin((() => {
@@ -34570,13 +34789,15 @@ var init_checks = __esmMin((() => {
34570
34789
  init_disk();
34571
34790
  init_config$1();
34572
34791
  init_certs();
34792
+ init_scaler$1();
34573
34793
  defaultChecks = [
34574
34794
  checkDbConnectivity,
34575
34795
  checkWsToPlatform,
34576
34796
  checkAgentConnectivity,
34577
34797
  checkDiskSpace,
34578
34798
  checkConfigValidity,
34579
- checkCertificateExpiry
34799
+ checkCertificateExpiry,
34800
+ checkScalerProvisioning
34580
34801
  ];
34581
34802
  }));
34582
34803
  //#endregion
@@ -34592,14 +34813,14 @@ var init_checks = __esmMin((() => {
34592
34813
  async function runDiagnostics(deps, checks, options) {
34593
34814
  const checksToRun = checks ?? defaultChecks;
34594
34815
  const timeoutMs = options?.timeoutMs ?? 5e3;
34595
- return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).map((result, index) => {
34596
- if (result.status === "fulfilled") return result.value;
34597
- return {
34816
+ return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).flatMap((result, index) => {
34817
+ if (result.status === "fulfilled") return Array.isArray(result.value) ? result.value : [result.value];
34818
+ return [{
34598
34819
  name: `Check ${index + 1}`,
34599
34820
  status: "fail",
34600
34821
  message: `Unexpected error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
34601
34822
  durationMs: 0
34602
- };
34823
+ }];
34603
34824
  });
34604
34825
  }
34605
34826
  /**
@@ -35696,7 +35917,8 @@ function createApp(deps) {
35696
35917
  platformUrl: deps.config.platformUrl,
35697
35918
  agentRegistry: deps.registry,
35698
35919
  config: deps.config,
35699
- tlsCertPath: deps.config.tlsCertPath
35920
+ tlsCertPath: deps.config.tlsCertPath,
35921
+ scalerManager: deps.scalerManager
35700
35922
  }));
35701
35923
  {
35702
35924
  const peerHandlerRef = deps.peerHandler;
@@ -38856,6 +39078,55 @@ var init_machine_ledger = __esmMin((() => {
38856
39078
  };
38857
39079
  }));
38858
39080
  //#endregion
39081
+ //#region src/scaler/failure-tracker.ts
39082
+ var DEFAULT_MAX_ENTRIES, ScalerFailureTracker;
39083
+ var init_failure_tracker = __esmMin((() => {
39084
+ DEFAULT_MAX_ENTRIES = 256;
39085
+ ScalerFailureTracker = class {
39086
+ records = [];
39087
+ maxEntries;
39088
+ constructor(maxEntries = DEFAULT_MAX_ENTRIES) {
39089
+ this.maxEntries = maxEntries;
39090
+ }
39091
+ /** Record a failure, evicting the oldest entry when over capacity. */
39092
+ record(rec) {
39093
+ this.records.push(rec);
39094
+ if (this.records.length > this.maxEntries) this.records.shift();
39095
+ }
39096
+ /**
39097
+ * Group failures newer than `nowMs - windowMs` by backend instance name.
39098
+ * `nowMs` is injected so callers control the clock (and tests are
39099
+ * deterministic).
39100
+ */
39101
+ recentByBackend(windowMs, nowMs) {
39102
+ const cutoff = nowMs - windowMs;
39103
+ const out = /* @__PURE__ */ new Map();
39104
+ for (const rec of this.records) {
39105
+ if (rec.timestampMs < cutoff) continue;
39106
+ const existing = out.get(rec.backendName);
39107
+ if (!existing) {
39108
+ out.set(rec.backendName, {
39109
+ backendType: rec.backendType,
39110
+ boundCount: rec.bound ? 1 : 0,
39111
+ unboundCount: rec.bound ? 0 : 1,
39112
+ lastError: rec.detail,
39113
+ lastAtMs: rec.timestampMs
39114
+ });
39115
+ continue;
39116
+ }
39117
+ if (rec.bound) existing.boundCount += 1;
39118
+ else existing.unboundCount += 1;
39119
+ if (rec.timestampMs >= existing.lastAtMs) {
39120
+ existing.lastError = rec.detail;
39121
+ existing.lastAtMs = rec.timestampMs;
39122
+ existing.backendType = rec.backendType;
39123
+ }
39124
+ }
39125
+ return out;
39126
+ }
39127
+ };
39128
+ }));
39129
+ //#endregion
38859
39130
  //#region src/scaler/manager.ts
38860
39131
  /**
38861
39132
  * ScalerManager: central coordinator for all scaler backends.
@@ -38919,10 +39190,13 @@ var init_manager = __esmMin((() => {
38919
39190
  init_machine_ledger();
38920
39191
  init_prometheus();
38921
39192
  init_types$1();
39193
+ init_failure_tracker();
38922
39194
  logger$25 = createLogger({ prefix: "scaler" });
38923
39195
  ScalerManager = class {
38924
39196
  backends = /* @__PURE__ */ new Map();
38925
39197
  backendRoles = /* @__PURE__ */ new Map();
39198
+ /** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
39199
+ failureTracker = new ScalerFailureTracker();
38926
39200
  globalMaxAgents;
38927
39201
  /** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
38928
39202
  resourceCaps = /* @__PURE__ */ new Map();
@@ -39564,6 +39838,13 @@ var init_manager = __esmMin((() => {
39564
39838
  /**
39565
39839
  * Return status summary for metrics and health endpoints.
39566
39840
  */
39841
+ /**
39842
+ * Recent scaler spawn failures grouped per backend instance, for the
39843
+ * diagnose scaler check. `nowMs` is injected by the caller.
39844
+ */
39845
+ recentSpawnFailures(windowMs, nowMs) {
39846
+ return this.failureTracker.recentByBackend(windowMs, nowMs);
39847
+ }
39567
39848
  getStatus() {
39568
39849
  const backendStatuses = [];
39569
39850
  const enrichedEntries = this.getEnrichedScalerEntries();
@@ -39850,6 +40131,13 @@ var init_manager = __esmMin((() => {
39850
40131
  jobId: jobId ?? null,
39851
40132
  detail: event.detail
39852
40133
  });
40134
+ this.failureTracker.record({
40135
+ backendName: backendName ?? "unknown",
40136
+ backendType: backend,
40137
+ bound: Boolean(jobId),
40138
+ detail: event.detail ?? "",
40139
+ timestampMs: event.timestampMs
40140
+ });
39853
40141
  }
39854
40142
  if (runId && jobId && this.onScalerEvent) this.onScalerEvent(runId, jobId, event);
39855
40143
  else if (this.onScalerEvent) {
@@ -41605,6 +41893,22 @@ var init_execution_tracker = __esmMin((() => {
41605
41893
  orgId;
41606
41894
  jobQueue;
41607
41895
  runs = /* @__PURE__ */ new Map();
41896
+ /**
41897
+ * Per-run async-mutex chain. `onJobStatus` and `addJobsToRun` mutate the same
41898
+ * `run.jobs` Map / `execution_jobs` row; without serialization a job-status
41899
+ * reply that lands mid synthetic→real swap (`addJobsToRun`) clobbers the swap
41900
+ * and the run hangs in `running` forever (see `withRunLock`). The map value is
41901
+ * the tail of the chain for that runId; entries are GC'd when the last holder
41902
+ * releases.
41903
+ */
41904
+ runLockTails = /* @__PURE__ */ new Map();
41905
+ /**
41906
+ * Tracks which runIds the current async context already holds in
41907
+ * `withRunLock`, so reentrant calls (e.g. onJobStatus → scheduler hook →
41908
+ * dispatchReadyJob → addJobsToRun, all the same runId) bypass re-acquisition
41909
+ * instead of deadlocking. Propagates across awaits via AsyncLocalStorage.
41910
+ */
41911
+ heldRunLocks = new AsyncLocalStorage();
41608
41912
  /** Tracks which runs are test runs for observer broadcasting. */
41609
41913
  testRunIds = /* @__PURE__ */ new Set();
41610
41914
  /**
@@ -41759,7 +42063,44 @@ var init_execution_tracker = __esmMin((() => {
41759
42063
  }
41760
42064
  return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
41761
42065
  }
42066
+ /**
42067
+ * Run `fn` while holding a per-run lock, serializing the run-mutating methods
42068
+ * (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
42069
+ * synthetic→real job swap and wedge the run in `running`.
42070
+ *
42071
+ * The lock is **reentrant**: three paths re-enter a locked method within the
42072
+ * same async context for the same runId — onJobStatus → scheduler hook →
42073
+ * dispatchReadyJob → addJobsToRun; onJobStatus → enforceSchedulerInvariant →
42074
+ * onJobStatus; runSchedulerHook → onJobStatus (skip). A non-reentrant mutex
42075
+ * would deadlock on these, so a context that already holds the runId's lock
42076
+ * (tracked via `heldRunLocks`) runs `fn` inline. A genuinely concurrent caller
42077
+ * for the same runId (a separate WS message) is a different async context and
42078
+ * correctly waits. All reentrant paths are same-runId, so there is no
42079
+ * cross-run lock-ordering deadlock.
42080
+ */
42081
+ async withRunLock(runId, fn) {
42082
+ if (this.heldRunLocks.getStore()?.has(runId)) return fn();
42083
+ const prev = this.runLockTails.get(runId) ?? Promise.resolve();
42084
+ let release;
42085
+ const mine = new Promise((resolve) => {
42086
+ release = resolve;
42087
+ });
42088
+ const chained = prev.then(() => mine);
42089
+ this.runLockTails.set(runId, chained);
42090
+ await prev.catch(() => {});
42091
+ const nextHeld = new Set(this.heldRunLocks.getStore() ?? []);
42092
+ nextHeld.add(runId);
42093
+ try {
42094
+ return await this.heldRunLocks.run(nextHeld, fn);
42095
+ } finally {
42096
+ release();
42097
+ if (this.runLockTails.get(runId) === chained) this.runLockTails.delete(runId);
42098
+ }
42099
+ }
41762
42100
  async addJobsToRun(runId, jobs, dispatchedContexts, replaceSyntheticId) {
42101
+ return this.withRunLock(runId, () => this.addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId));
42102
+ }
42103
+ async addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId) {
41763
42104
  const run = this.runs.get(runId);
41764
42105
  if (!run) {
41765
42106
  logger$21.warn("addJobsToRun: run not found in memory", { runId });
@@ -41854,6 +42195,9 @@ var init_execution_tracker = __esmMin((() => {
41854
42195
  * fires the onExecutionComplete callback.
41855
42196
  */
41856
42197
  async onJobStatus(runId, jobId, state, timestamp, agentId, data) {
42198
+ return this.withRunLock(runId, () => this.onJobStatusImpl(runId, jobId, state, timestamp, agentId, data));
42199
+ }
42200
+ async onJobStatusImpl(runId, jobId, state, timestamp, agentId, data) {
41857
42201
  let run = this.runs.get(runId);
41858
42202
  let job = run?.jobs.get(jobId);
41859
42203
  if (!run) {
@@ -41974,6 +42318,8 @@ var init_execution_tracker = __esmMin((() => {
41974
42318
  * run's in-memory map. Returns the freshly-inserted job entry.
41975
42319
  */
41976
42320
  async recoverJobFromDispatchQueue(run, runId, jobId) {
42321
+ const existing = run.jobs.get(jobId);
42322
+ if (existing) return existing;
41977
42323
  const jobName = (await this.db.selectFrom("dispatch_queue").select(["job_name"]).where("id", "=", jobId).executeTakeFirst())?.job_name ?? jobId;
41978
42324
  run.jobs.set(jobId, {
41979
42325
  name: jobName,
@@ -42008,6 +42354,7 @@ var init_execution_tracker = __esmMin((() => {
42008
42354
  updateValues.duration_ms = timestamp - job.startedAt;
42009
42355
  }
42010
42356
  if (data?.error) updateValues.error_message = String(data.error);
42357
+ if (data?.initFailure) updateValues.init_failure = JSON.stringify(data.initFailure);
42011
42358
  if (data?.outputs) updateValues.outputs = JSON.stringify(data.outputs);
42012
42359
  jobLogBytesTotal = this.jobLogBytes.get(runId)?.get(jobId) ?? 0;
42013
42360
  updateValues.log_bytes = jobLogBytesTotal;
@@ -42126,7 +42473,8 @@ var init_execution_tracker = __esmMin((() => {
42126
42473
  const completedAtMs = isTerminal ? timestamp : void 0;
42127
42474
  const duration = isTerminal && job.startedAt ? timestamp - job.startedAt : void 0;
42128
42475
  const errorMsg = isTerminal && data?.error ? String(data.error) : void 0;
42129
- this.onJobStatusChange?.(runId, jobId, job.name, state, timestamp, startedAtMs, completedAtMs, duration, errorMsg, job.agentId, job.runsOnLabels, jobLogBytesTotal);
42476
+ const initFailure = isTerminal && data?.initFailure ? data.initFailure : void 0;
42477
+ this.onJobStatusChange?.(runId, jobId, job.name, state, timestamp, startedAtMs, completedAtMs, duration, errorMsg, job.agentId, job.runsOnLabels, jobLogBytesTotal, initFailure);
42130
42478
  }
42131
42479
  /**
42132
42480
  * Phase 6: reconcile orphaned steps. If a job reaches terminal state but
@@ -42323,13 +42671,14 @@ var init_execution_tracker = __esmMin((() => {
42323
42671
  * onExecutionStarted but the build subsequently fails. Without this,
42324
42672
  * the execution_runs row would stay in a non-terminal state.
42325
42673
  */
42326
- async onBuildFailed(runId) {
42674
+ async onBuildFailed(runId, initFailure) {
42327
42675
  const now = /* @__PURE__ */ new Date();
42328
- const failureReason = "Build job failed";
42676
+ const failureReason = initFailure?.message ?? "Build job failed";
42329
42677
  await this.db.updateTable("execution_runs").set({
42330
42678
  status: ExecutionRunStatus.enum.failed,
42331
42679
  completed_at: now,
42332
- failure_reason: failureReason
42680
+ failure_reason: failureReason,
42681
+ ...initFailure && { init_failure: JSON.stringify(initFailure) }
42333
42682
  }).where("run_id", "=", runId).execute();
42334
42683
  await this.db.updateTable("execution_jobs").set({
42335
42684
  status: ExecutionJobStatus.enum.failed,
@@ -42352,7 +42701,7 @@ var init_execution_tracker = __esmMin((() => {
42352
42701
  parentRunId: run.parentRunId,
42353
42702
  originalRunId: run.originalRunId,
42354
42703
  triggeredBy: run.triggeredBy
42355
- }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason);
42704
+ }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason, void 0, initFailure);
42356
42705
  this.runs.delete(runId);
42357
42706
  this.jobLogBytes.delete(runId);
42358
42707
  }
@@ -42365,7 +42714,7 @@ var init_execution_tracker = __esmMin((() => {
42365
42714
  * Inserts a minimal execution_runs row with status='failed' directly so the E2E
42366
42715
  * test (and dashboard) can observe the failure instead of a missing run.
42367
42716
  */
42368
- async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason) {
42717
+ async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason, initFailure) {
42369
42718
  const now = /* @__PURE__ */ new Date();
42370
42719
  const reason = failureReason ?? "Build job timed out before execution tracking started";
42371
42720
  await this.db.insertInto("execution_runs").values({
@@ -42382,7 +42731,8 @@ var init_execution_tracker = __esmMin((() => {
42382
42731
  started_at: now,
42383
42732
  completed_at: now,
42384
42733
  status: ExecutionRunStatus.enum.failed,
42385
- failure_reason: reason
42734
+ failure_reason: reason,
42735
+ ...initFailure && { init_failure: JSON.stringify(initFailure) }
42386
42736
  }).execute();
42387
42737
  logger$21.info("Created failed execution run (build failed before tracking)", {
42388
42738
  runId,
@@ -42391,6 +42741,55 @@ var init_execution_tracker = __esmMin((() => {
42391
42741
  });
42392
42742
  }
42393
42743
  /**
42744
+ * Insert a `failed` execution_runs row directly for an init failure that
42745
+ * occurred BEFORE onExecutionStarted ran (so no in-memory state exists
42746
+ * and no jobs were dispatched). Also writes the structured init_failure
42747
+ * signal and fires onExecutionStatusChange so Platform's projection picks
42748
+ * it up via the normal forward path. Idempotent: if a row already exists
42749
+ * for this runId, the insert is a no-op (ON CONFLICT DO NOTHING).
42750
+ *
42751
+ * Closes the silent pre-run-failure gap — without this helper, secret /
42752
+ * install-secret / all-jobs-rejected early-exits in dispatch-matched-workflow
42753
+ * leave no trace on the dashboard.
42754
+ */
42755
+ async recordInitFailureRun(args) {
42756
+ const now = /* @__PURE__ */ new Date();
42757
+ await this.db.insertInto("execution_runs").values({
42758
+ run_id: args.runId,
42759
+ routing_key: args.routingKey,
42760
+ workflow_name: args.workflowName,
42761
+ provider: args.provider,
42762
+ repo_identifier: args.repoIdentifier,
42763
+ ref: args.ref,
42764
+ sha: args.sha,
42765
+ delivery_id: args.deliveryId,
42766
+ trigger_decision: null,
42767
+ provider_context: JSON.stringify(args.providerContext),
42768
+ started_at: now,
42769
+ completed_at: now,
42770
+ status: ExecutionRunStatus.enum.failed,
42771
+ failure_reason: args.initFailure.message,
42772
+ init_failure: JSON.stringify(args.initFailure)
42773
+ }).onConflict((oc) => oc.column("run_id").doNothing()).execute();
42774
+ executionsTotal.add(1, { status: ExecutionRunStatus.enum.failed });
42775
+ this.onExecutionStatusChange?.(args.runId, ExecutionRunStatus.enum.failed, {
42776
+ workflowName: args.workflowName,
42777
+ provider: args.provider,
42778
+ repoIdentifier: args.repoIdentifier,
42779
+ sha: args.sha,
42780
+ routingKey: args.routingKey,
42781
+ ref: args.ref,
42782
+ triggerEvent: args.triggerEvent,
42783
+ commitMessage: args.commitMessage
42784
+ }, 0, now.getTime(), now.getTime(), 0, args.initFailure.message, 0, args.initFailure);
42785
+ logger$21.info("Recorded init-failure execution run", {
42786
+ runId: args.runId,
42787
+ workflowName: args.workflowName,
42788
+ category: args.initFailure.category,
42789
+ scope: args.initFailure.scope
42790
+ });
42791
+ }
42792
+ /**
42394
42793
  * Mark a run as failed immediately with a reason message.
42395
42794
  *
42396
42795
  * Used when no agents are available to dispatch any jobs (e.g. cron-triggered
@@ -42398,7 +42797,7 @@ var init_execution_tracker = __esmMin((() => {
42398
42797
  * Instead of leaving the run in 'running' for OrphanRecovery to catch after 5 min,
42399
42798
  * this fails it right away.
42400
42799
  */
42401
- async failRun(runId, reason) {
42800
+ async failRun(runId, reason, initFailure) {
42402
42801
  const now = /* @__PURE__ */ new Date();
42403
42802
  const run = this.runs.get(runId);
42404
42803
  const durationMs = run ? now.getTime() - run.startedAt : 0;
@@ -42406,7 +42805,8 @@ var init_execution_tracker = __esmMin((() => {
42406
42805
  status: ExecutionRunStatus.enum.failed,
42407
42806
  completed_at: now,
42408
42807
  duration_ms: durationMs,
42409
- failure_reason: reason
42808
+ failure_reason: reason,
42809
+ ...initFailure && { init_failure: JSON.stringify(initFailure) }
42410
42810
  }).where("run_id", "=", runId).execute();
42411
42811
  await this.db.updateTable("execution_jobs").set({
42412
42812
  status: ExecutionJobStatus.enum.failed,
@@ -42430,7 +42830,7 @@ var init_execution_tracker = __esmMin((() => {
42430
42830
  parentRunId: run.parentRunId,
42431
42831
  originalRunId: run.originalRunId,
42432
42832
  triggeredBy: run.triggeredBy
42433
- }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason);
42833
+ }, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason, void 0, initFailure);
42434
42834
  this.runs.delete(runId);
42435
42835
  this.jobLogBytes.delete(runId);
42436
42836
  }
@@ -50549,14 +50949,14 @@ var init_worker_core = __esmMin((() => {
50549
50949
  init_agent_handler();
50550
50950
  init_worker_status();
50551
50951
  init_agent_heartbeat();
50552
- ORCHESTRATOR_VERSION$1 = "0.1.13";
50553
- WORKER_BUILD_COMMIT = "b3ee727cb";
50554
- WORKER_SDK_VERSION = "0.1.13";
50952
+ ORCHESTRATOR_VERSION$1 = "0.1.14";
50953
+ WORKER_BUILD_COMMIT = "0fc7a43c9";
50954
+ WORKER_SDK_VERSION = "0.1.14";
50555
50955
  WORKER_SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
50556
- WORKER_SHARED_VERSION = "0.1.13";
50557
- WORKER_SHARED_BUNDLE_HASH = "36a23a454fd75ec35f258d39f254543ec0451c6b543fe3618ce427e6e6369aae";
50558
- WORKER_ENGINE_VERSION = "0.1.13";
50559
- WORKER_ENGINE_BUNDLE_HASH = "4ab08e63bcb8558b1191c5d3ab8d58b71950b77d911af56c9955ae1d787c4302";
50956
+ WORKER_SHARED_VERSION = "0.1.14";
50957
+ WORKER_SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
50958
+ WORKER_ENGINE_VERSION = "0.1.14";
50959
+ WORKER_ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
50560
50960
  logger$2 = createLogger({ prefix: "worker" });
50561
50961
  DRAIN_TIMEOUT_MS = 3e5;
50562
50962
  }));
@@ -50577,14 +50977,14 @@ var init_worker_core = __esmMin((() => {
50577
50977
  * Graceful shutdown in reverse order:
50578
50978
  * Platform client -> agent WS -> heartbeat -> HTTP -> DB
50579
50979
  */
50580
- const ORCHESTRATOR_VERSION = "0.1.13";
50581
- const BUILD_COMMIT = "b3ee727cb";
50582
- const SDK_VERSION = "0.1.13";
50980
+ const ORCHESTRATOR_VERSION = "0.1.14";
50981
+ const BUILD_COMMIT = "0fc7a43c9";
50982
+ const SDK_VERSION = "0.1.14";
50583
50983
  const SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
50584
- const SHARED_VERSION = "0.1.13";
50585
- const SHARED_BUNDLE_HASH = "36a23a454fd75ec35f258d39f254543ec0451c6b543fe3618ce427e6e6369aae";
50586
- const ENGINE_VERSION = "0.1.13";
50587
- const ENGINE_BUNDLE_HASH = "4ab08e63bcb8558b1191c5d3ab8d58b71950b77d911af56c9955ae1d787c4302";
50984
+ const SHARED_VERSION = "0.1.14";
50985
+ const SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
50986
+ const ENGINE_VERSION = "0.1.14";
50987
+ const ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
50588
50988
  const otelSdk = initTelemetry({
50589
50989
  serviceName: "kici-orchestrator",
50590
50990
  otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
@@ -50662,7 +51062,7 @@ await guardStartup(logger$1, async () => {
50662
51062
  await bootstrapOrchestrator(config, {
50663
51063
  logPrefix: "server",
50664
51064
  executionTrackerExtras: () => ({
50665
- onExecutionStatusChange: (runId, status, context, jobCount, startedAt, completedAt, durationMs, failureReason, logBytes) => {
51065
+ onExecutionStatusChange: (runId, status, context, jobCount, startedAt, completedAt, durationMs, failureReason, logBytes, initFailure) => {
50666
51066
  try {
50667
51067
  platformClient.send({
50668
51068
  type: "execution.status",
@@ -50684,7 +51084,8 @@ await guardStartup(logger$1, async () => {
50684
51084
  ...completedAt !== void 0 && { completedAt },
50685
51085
  ...durationMs !== void 0 && { durationMs },
50686
51086
  ...failureReason !== void 0 && { failureReason },
50687
- ...logBytes !== void 0 && { logBytes }
51087
+ ...logBytes !== void 0 && { logBytes },
51088
+ ...initFailure && { initFailure }
50688
51089
  });
50689
51090
  } catch (err) {
50690
51091
  logger$1.warn("Failed to forward execution status to Platform", {
@@ -50694,7 +51095,7 @@ await guardStartup(logger$1, async () => {
50694
51095
  });
50695
51096
  }
50696
51097
  },
50697
- onJobStatusChange: (runId, jobId, jobName, status, timestamp, startedAt, completedAt, durationMs, errorMessage, agentId, runsOnLabels, logBytes) => {
51098
+ onJobStatusChange: (runId, jobId, jobName, status, timestamp, startedAt, completedAt, durationMs, errorMessage, agentId, runsOnLabels, logBytes, initFailure) => {
50698
51099
  try {
50699
51100
  platformClient.send({
50700
51101
  type: "job.status.forward",
@@ -50711,6 +51112,7 @@ await guardStartup(logger$1, async () => {
50711
51112
  ...logBytes !== void 0 && { logBytes },
50712
51113
  ...agentId && { agentId },
50713
51114
  ...runsOnLabels?.length && { runsOnLabels },
51115
+ ...initFailure && { initFailure },
50714
51116
  orchestratorId: config.instanceId
50715
51117
  });
50716
51118
  } catch (err) {