@kici-dev/orchestrator 0.1.12 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/agent-service/install.d.ts +7 -0
- package/dist/cli/commands/agent-service/logs.d.ts +8 -3
- package/dist/cli/commands/agent-service/restart.d.ts +5 -1
- package/dist/cli/commands/agent-service/start.d.ts +5 -1
- package/dist/cli/commands/agent-service/status.d.ts +5 -1
- package/dist/cli/commands/agent-service/stop.d.ts +5 -1
- package/dist/cli/commands/agent-service/uninstall.d.ts +6 -2
- package/dist/cli/commands/agent-service/upgrade.d.ts +13 -3
- package/dist/cli/commands/orchestrator-service/logs.d.ts +4 -0
- package/dist/cli/commands/orchestrator-service/restart.d.ts +5 -1
- package/dist/cli/commands/orchestrator-service/start.d.ts +5 -1
- package/dist/cli/commands/orchestrator-service/status.d.ts +4 -0
- package/dist/cli/commands/orchestrator-service/stop.d.ts +5 -1
- package/dist/cli/commands/orchestrator-service/uninstall.d.ts +6 -2
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +9 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +54 -2
- package/dist/cli/service/compose.d.ts +3 -2
- package/dist/cli/service/index.d.ts +7 -1
- package/dist/cli/service/instance/index-file.d.ts +30 -0
- package/dist/cli/service/instance/manifest.d.ts +30 -0
- package/dist/cli/service/instance/resolve.d.ts +59 -0
- package/dist/cli/service/instance/types.d.ts +55 -0
- package/dist/cli/service/launchd.d.ts +19 -1
- package/dist/cli/service/platform-detect.d.ts +29 -9
- package/dist/cli/service/systemd.d.ts +2 -1
- package/dist/cli/service/types.d.ts +28 -0
- package/dist/cli/service/windows.d.ts +2 -1
- package/dist/cli.js +1719 -1002
- package/dist/db/migrations/025_init_failure.d.ts +16 -0
- package/dist/db/types.d.ts +15 -0
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/checks/scaler.d.ts +13 -0
- package/dist/diagnostics/types.d.ts +5 -2
- package/dist/metrics/prometheus.d.ts +32 -4
- package/dist/reporting/execution-tracker.d.ts +77 -6
- package/dist/scaler/failure-tracker.d.ts +46 -0
- package/dist/scaler/manager.d.ts +8 -0
- package/dist/server.js +570 -168
- package/dist/standalone.js +547 -156
- package/package.json +4 -4
- package/sbom.spdx.json +91 -36
package/dist/server.js
CHANGED
|
@@ -3,7 +3,7 @@ import { dirname as __cjs_dirname } from "node:path";
|
|
|
3
3
|
__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
|
|
4
4
|
import "node:module";
|
|
5
5
|
import { BaseColdStore, RingBuffer, createDb, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, createPool, createS3Client, encodeKeySegment, enrichRequestContext, formatDuration, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, logger, requestContext, serializeError, setServiceName, setupGracefulShutdown, sha256, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
6
|
-
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EVENT_LOG_PAYLOAD_CHUNK_BYTES, EventLogPayloadStreamError, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, ORCH_CAPABILITIES, OrchRole, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WEBHOOK_RELAY_MAX_BODY_BYTES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, dashboardRunDetailApiResponseSchema, dashboardStepLogsApiResponseSchema, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, joinRequestSchema, logPullPlatformToOrchSchema, matchAllWorkflows, matchScopePattern, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, platformToOrchestratorMessageSchema, resolveRoleLabels, resolveSecretsForEnvironment, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, stringifyActor, stripScopePrefix, testEventSchema } from "@kici-dev/engine";
|
|
6
|
+
import { ALLOWED_SYSTEM_VARS, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, CheckRunConclusion, EVENT_LOG_PAYLOAD_CHUNK_BYTES, EventLogPayloadStreamError, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, InitFailureCategory, KICI_AGENT_ENV_PREFIX, KNOWN_ROLES, MIN_PROTOCOL_VERSION, ORCH_CAPABILITIES, OrchRole, PROTOCOL_VERSION, PayloadOmittedReason, RegisterableTriggerType, SCHEMA_VERSION, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, WEBHOOK_RELAY_MAX_BODY_BYTES, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_RUN_NOT_FOUND, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentToOrchestratorMessageSchema, agentTypeLabel, createWorkflowDecision, dashboardRunDetailApiResponseSchema, dashboardStepLogsApiResponseSchema, deriveOsArchLabels, flattenActor, getAccessLogColdDays, getSecretAuditLogColdDays, isLockDynamicJobFn, isLockInlineValue, isLockStaticJob, isSelfReportedLabel, joinRequestSchema, logPullPlatformToOrchSchema, matchAllWorkflows, matchScopePattern, minAccessLogWarmDays, minSecretAuditLogWarmDays, observeSubscribeSchema, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, platformToOrchestratorMessageSchema, resolveRoleLabels, resolveSecretsForEnvironment, scalerAgentLabels, scalerLabel, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, stringifyActor, stripScopePrefix, testEventSchema } from "@kici-dev/engine";
|
|
7
7
|
import { verifySignature } from "@kici-dev/engine/webhook/signature";
|
|
8
8
|
import { X509Certificate, createCipheriv, createDecipheriv, createHmac, createPrivateKey, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
9
9
|
import os, { cpus, freemem, homedir, hostname, release, tmpdir, totalmem, uptime, userInfo, version } from "node:os";
|
|
@@ -49,6 +49,7 @@ import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCom
|
|
|
49
49
|
import { Upload } from "@aws-sdk/lib-storage";
|
|
50
50
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
51
51
|
import stripAnsi from "strip-ansi";
|
|
52
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
52
53
|
import { ORCH_PUSHED_METRIC_NAMES } from "@kici-dev/engine/metrics/catalog-policy";
|
|
53
54
|
var __defProp = Object.defineProperty;
|
|
54
55
|
var __esmMin = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
@@ -3850,7 +3851,11 @@ async function runBuildJob(args) {
|
|
|
3850
3851
|
jobName: buildJobName,
|
|
3851
3852
|
runsOnLabels: buildJobInput.runsOnLabels
|
|
3852
3853
|
}], setup.info.routingKey, void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage$1(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
|
|
3853
|
-
await deps.executionTracker.failRun(runId, reason
|
|
3854
|
+
await deps.executionTracker.failRun(runId, reason, {
|
|
3855
|
+
scope: "run",
|
|
3856
|
+
category: InitFailureCategory.enum.build_coordination,
|
|
3857
|
+
message: reason
|
|
3858
|
+
});
|
|
3854
3859
|
}
|
|
3855
3860
|
return;
|
|
3856
3861
|
}
|
|
@@ -3903,8 +3908,14 @@ async function recordBuildFailure(args) {
|
|
|
3903
3908
|
});
|
|
3904
3909
|
}
|
|
3905
3910
|
if (deps.executionTracker) try {
|
|
3906
|
-
|
|
3907
|
-
|
|
3911
|
+
const buildFailureReason = toErrorMessage(err);
|
|
3912
|
+
const buildInitFailure = {
|
|
3913
|
+
scope: "run",
|
|
3914
|
+
category: InitFailureCategory.enum.build_coordination,
|
|
3915
|
+
message: buildFailureReason
|
|
3916
|
+
};
|
|
3917
|
+
if (buildJobTrackedEarly) await deps.executionTracker.onBuildFailed(runId, buildInitFailure);
|
|
3918
|
+
else await deps.executionTracker.onBuildFailedBeforeTracking(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, setup.info.routingKey, buildTriggerEvent(event.type, event.action), extractCommitMessage$1(setup.info.event, setup.info.payload), buildFailureReason, buildInitFailure);
|
|
3908
3919
|
} catch (cleanupErr) {
|
|
3909
3920
|
logger$92.warn("Failed to mark run as failed after build error", {
|
|
3910
3921
|
runId,
|
|
@@ -4114,11 +4125,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
|
|
|
4114
4125
|
const declaredContexts = workflow.contexts ?? [];
|
|
4115
4126
|
if (declaredContexts.length > 0) {
|
|
4116
4127
|
if (!deps.secretResolver) {
|
|
4117
|
-
|
|
4128
|
+
const reason = "Workflow declares secret contexts but secrets subsystem is not configured (KICI_SECRET_KEY missing)";
|
|
4129
|
+
logger$92.error(reason, {
|
|
4118
4130
|
workflow: workflow.name,
|
|
4119
4131
|
contexts: declaredContexts
|
|
4120
4132
|
});
|
|
4121
|
-
return {
|
|
4133
|
+
return {
|
|
4134
|
+
skipDispatch: true,
|
|
4135
|
+
reason
|
|
4136
|
+
};
|
|
4122
4137
|
}
|
|
4123
4138
|
try {
|
|
4124
4139
|
const mergedSecrets = {};
|
|
@@ -4133,11 +4148,15 @@ async function resolveWorkflowSecretsAndKey(ctx) {
|
|
|
4133
4148
|
resolvedNamespacedSecrets = mergedNamespaced;
|
|
4134
4149
|
}
|
|
4135
4150
|
} catch (err) {
|
|
4151
|
+
const errMessage = toErrorMessage(err);
|
|
4136
4152
|
logger$92.error("Secret resolution failed, skipping workflow", {
|
|
4137
4153
|
workflow: workflow.name,
|
|
4138
|
-
error:
|
|
4154
|
+
error: errMessage
|
|
4139
4155
|
});
|
|
4140
|
-
return {
|
|
4156
|
+
return {
|
|
4157
|
+
skipDispatch: true,
|
|
4158
|
+
reason: `Secret resolution failed: ${errMessage}`
|
|
4159
|
+
};
|
|
4141
4160
|
}
|
|
4142
4161
|
}
|
|
4143
4162
|
let runPublicKeyBase64;
|
|
@@ -4215,7 +4234,10 @@ async function resolveWorkflowInstallSecrets(ctx, secrets) {
|
|
|
4215
4234
|
workflow: workflow.name,
|
|
4216
4235
|
reason: result.reason
|
|
4217
4236
|
});
|
|
4218
|
-
return {
|
|
4237
|
+
return {
|
|
4238
|
+
skipDispatch: true,
|
|
4239
|
+
reason: `Workflow install-secrets resolution rejected dispatch: ${result.reason}`
|
|
4240
|
+
};
|
|
4219
4241
|
}
|
|
4220
4242
|
if (result.contributorStripped) logger$92.warn("Skipping registries:/installEnv: resolution for untrusted contributor — install will fail naturally if private deps are required", {
|
|
4221
4243
|
runId,
|
|
@@ -4668,6 +4690,7 @@ async function clusterRouteRootJobs(args) {
|
|
|
4668
4690
|
});
|
|
4669
4691
|
rejectedJobs.push({
|
|
4670
4692
|
jobId: syntheticId,
|
|
4693
|
+
jobName: jtr.jobName,
|
|
4671
4694
|
reason: result.reason
|
|
4672
4695
|
});
|
|
4673
4696
|
} else if (result.status === "queued-no-backend") logger$92.warn("Job has no matching backend (cluster fallback), skipping", {
|
|
@@ -4773,6 +4796,7 @@ async function dispatchSingleOrchPath(args) {
|
|
|
4773
4796
|
});
|
|
4774
4797
|
rejectedJobs.push({
|
|
4775
4798
|
jobId: syntheticId,
|
|
4799
|
+
jobName: jobOrFactory.name,
|
|
4776
4800
|
reason: result.reason
|
|
4777
4801
|
});
|
|
4778
4802
|
} else if (result.status === "queued-no-backend") logger$92.warn("Job has no matching backend, skipping execution tracking", {
|
|
@@ -4877,6 +4901,11 @@ async function recordRunStart(args) {
|
|
|
4877
4901
|
});
|
|
4878
4902
|
});
|
|
4879
4903
|
}
|
|
4904
|
+
function categorizeRejectReason(reason) {
|
|
4905
|
+
const lower = reason.toLowerCase();
|
|
4906
|
+
if (/no\s+agent|no\s+matching\s+backend/.test(lower)) return InitFailureCategory.enum.no_agent;
|
|
4907
|
+
return InitFailureCategory.enum.environment_rules;
|
|
4908
|
+
}
|
|
4880
4909
|
async function insertEdgesAndMarkRejected(args) {
|
|
4881
4910
|
const { ctx, staticJobs, dispatchedJobs, rejectedJobs } = args;
|
|
4882
4911
|
const { deps, runId } = ctx;
|
|
@@ -4890,7 +4919,15 @@ async function insertEdgesAndMarkRejected(args) {
|
|
|
4890
4919
|
}
|
|
4891
4920
|
if (deps.executionTracker && rejectedJobs.length > 0) {
|
|
4892
4921
|
const now = Date.now();
|
|
4893
|
-
for (const { jobId, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, {
|
|
4922
|
+
for (const { jobId, jobName, reason } of rejectedJobs) deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, now, void 0, {
|
|
4923
|
+
error: reason,
|
|
4924
|
+
initFailure: {
|
|
4925
|
+
scope: "job",
|
|
4926
|
+
category: categorizeRejectReason(reason),
|
|
4927
|
+
message: reason,
|
|
4928
|
+
jobName
|
|
4929
|
+
}
|
|
4930
|
+
}).catch((err) => {
|
|
4894
4931
|
logger$92.error("Failed to mark rejected job as failed", {
|
|
4895
4932
|
runId,
|
|
4896
4933
|
jobId,
|
|
@@ -5071,17 +5108,30 @@ function startDeferredInitDispatch(args) {
|
|
|
5071
5108
|
lockJob
|
|
5072
5109
|
});
|
|
5073
5110
|
} catch (err) {
|
|
5111
|
+
const errMsg = toErrorMessage(err);
|
|
5074
5112
|
logger$92.error("Deferred init job failed", {
|
|
5075
5113
|
runId,
|
|
5076
5114
|
workflow: workflow.name,
|
|
5077
5115
|
job: lockJob.name,
|
|
5078
|
-
error:
|
|
5116
|
+
error: errMsg
|
|
5079
5117
|
});
|
|
5080
|
-
if (deps.executionTracker)
|
|
5081
|
-
jobId
|
|
5082
|
-
|
|
5083
|
-
|
|
5084
|
-
|
|
5118
|
+
if (deps.executionTracker) {
|
|
5119
|
+
const jobId = `init-failed-${lockJob.name}`;
|
|
5120
|
+
await deps.executionTracker.addJobsToRun(runId, [{
|
|
5121
|
+
jobId,
|
|
5122
|
+
jobName: lockJob.name,
|
|
5123
|
+
runsOnLabels: Array.isArray(lockJob.runsOn) ? [...lockJob.runsOn] : [lockJob.runsOn]
|
|
5124
|
+
}]).catch(() => {});
|
|
5125
|
+
await deps.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, {
|
|
5126
|
+
error: errMsg,
|
|
5127
|
+
initFailure: {
|
|
5128
|
+
scope: "job",
|
|
5129
|
+
category: InitFailureCategory.enum.dynamic_eval,
|
|
5130
|
+
message: errMsg,
|
|
5131
|
+
jobName: lockJob.name
|
|
5132
|
+
}
|
|
5133
|
+
}).catch(() => {});
|
|
5134
|
+
}
|
|
5085
5135
|
}
|
|
5086
5136
|
})();
|
|
5087
5137
|
}
|
|
@@ -5623,6 +5673,39 @@ async function ensureExecutionRunForDeferred(args) {
|
|
|
5623
5673
|
await deps.executionTracker.onExecutionStarted(runId, workflow.name, setup.info.provider, repoIdentifier, event.targetBranch, ref, setup.effectiveDeliveryId, credentials, summarizeDecision(decision), [], setup.info.routingKey, declaredContexts.length > 0 ? [...declaredContexts] : void 0, buildTriggerEvent(event.type, event.action), extractCommitMessage$1(setup.info.event, setup.info.payload), void 0, void 0, void 0, setup.workflowConcurrency);
|
|
5624
5674
|
}
|
|
5625
5675
|
/**
|
|
5676
|
+
* Persist a failed `execution_runs` row for a pre-dispatch early-exit so the
|
|
5677
|
+
* dashboard's Runs view surfaces secret_resolution / install_secrets /
|
|
5678
|
+
* environment_rules rejections instead of leaving the run with zero trace.
|
|
5679
|
+
*
|
|
5680
|
+
* Called from each of the three early-exit sites in `dispatchMatchedWorkflow`
|
|
5681
|
+
* (workflow secrets, install secrets, all-jobs-rejected). The helper is a
|
|
5682
|
+
* no-op when no `executionTracker` is wired into deps (test / minimal
|
|
5683
|
+
* deployments).
|
|
5684
|
+
*/
|
|
5685
|
+
async function recordInitFailureFromSkip(args) {
|
|
5686
|
+
const { ctx, setup, category, reason } = args;
|
|
5687
|
+
const { deps, workflow, repoIdentifier, credentials, event, ref, runId } = ctx;
|
|
5688
|
+
if (!deps.executionTracker) return;
|
|
5689
|
+
await deps.executionTracker.recordInitFailureRun({
|
|
5690
|
+
runId,
|
|
5691
|
+
workflowName: workflow.name,
|
|
5692
|
+
provider: setup.info.provider,
|
|
5693
|
+
repoIdentifier,
|
|
5694
|
+
ref: event.sourceBranch ?? event.targetBranch,
|
|
5695
|
+
sha: ref,
|
|
5696
|
+
deliveryId: setup.effectiveDeliveryId,
|
|
5697
|
+
providerContext: credentials ?? {},
|
|
5698
|
+
routingKey: setup.info.routingKey,
|
|
5699
|
+
initFailure: {
|
|
5700
|
+
scope: "run",
|
|
5701
|
+
category,
|
|
5702
|
+
message: reason
|
|
5703
|
+
},
|
|
5704
|
+
triggerEvent: buildTriggerEvent(event.type, event.action),
|
|
5705
|
+
commitMessage: extractCommitMessage$1(setup.info.event, setup.info.payload)
|
|
5706
|
+
});
|
|
5707
|
+
}
|
|
5708
|
+
/**
|
|
5626
5709
|
* Dispatch a single matched workflow.
|
|
5627
5710
|
*
|
|
5628
5711
|
* Narrative orchestrator: each phase is a typed helper that returns a typed
|
|
@@ -5642,9 +5725,26 @@ async function dispatchMatchedWorkflow(ctx) {
|
|
|
5642
5725
|
const buildPrep = await prepareCacheAndBuild(ctx, setup);
|
|
5643
5726
|
if (buildPrep.abort) return { dispatchedJobCount: 0 };
|
|
5644
5727
|
const secretsResult = await resolveWorkflowSecretsAndKey(ctx);
|
|
5645
|
-
if ("skipDispatch" in secretsResult)
|
|
5728
|
+
if ("skipDispatch" in secretsResult) {
|
|
5729
|
+
await recordInitFailureFromSkip({
|
|
5730
|
+
ctx,
|
|
5731
|
+
setup,
|
|
5732
|
+
category: InitFailureCategory.enum.secret_resolution,
|
|
5733
|
+
reason: secretsResult.reason
|
|
5734
|
+
});
|
|
5735
|
+
return { dispatchedJobCount: 0 };
|
|
5736
|
+
}
|
|
5646
5737
|
const secrets = secretsResult;
|
|
5647
|
-
|
|
5738
|
+
const installResult = await resolveWorkflowInstallSecrets(ctx, secrets);
|
|
5739
|
+
if (installResult.skipDispatch) {
|
|
5740
|
+
await recordInitFailureFromSkip({
|
|
5741
|
+
ctx,
|
|
5742
|
+
setup,
|
|
5743
|
+
category: InitFailureCategory.enum.install_secrets,
|
|
5744
|
+
reason: installResult.reason
|
|
5745
|
+
});
|
|
5746
|
+
return { dispatchedJobCount: 0 };
|
|
5747
|
+
}
|
|
5648
5748
|
const evalResult = await evaluateJobEnvironments({
|
|
5649
5749
|
ctx,
|
|
5650
5750
|
setup,
|
|
@@ -5691,6 +5791,12 @@ async function dispatchMatchedWorkflow(ctx) {
|
|
|
5691
5791
|
dispatchedJobs,
|
|
5692
5792
|
rejectedJobs
|
|
5693
5793
|
});
|
|
5794
|
+
if (dispatchedJobs.length === 0 && evalResult.deferredInitJobs.length === 0 && buildPrep.dynamicEntries.length === 0 && rejectedJobs.length > 0) await recordInitFailureFromSkip({
|
|
5795
|
+
ctx,
|
|
5796
|
+
setup,
|
|
5797
|
+
category: InitFailureCategory.enum.environment_rules,
|
|
5798
|
+
reason: rejectedJobs[0].reason
|
|
5799
|
+
});
|
|
5694
5800
|
if (evalResult.deferredInitJobs.length > 0) await ensureExecutionRunForDeferred({
|
|
5695
5801
|
ctx,
|
|
5696
5802
|
setup,
|
|
@@ -8078,7 +8184,8 @@ var init_handler = __esmMin((() => {
|
|
|
8078
8184
|
let runRow = await this.db.selectFrom("execution_runs").select([
|
|
8079
8185
|
"trust_tier",
|
|
8080
8186
|
"lock_file_source",
|
|
8081
|
-
"contributor_username"
|
|
8187
|
+
"contributor_username",
|
|
8188
|
+
"init_failure"
|
|
8082
8189
|
]).where("run_id", "=", msg.runId).executeTakeFirst();
|
|
8083
8190
|
let jobs = await this.db.selectFrom("execution_jobs").select([
|
|
8084
8191
|
"job_id",
|
|
@@ -8091,7 +8198,8 @@ var init_handler = __esmMin((() => {
|
|
|
8091
8198
|
"duration_ms",
|
|
8092
8199
|
"error_message",
|
|
8093
8200
|
"runs_on_labels",
|
|
8094
|
-
"outputs"
|
|
8201
|
+
"outputs",
|
|
8202
|
+
"init_failure"
|
|
8095
8203
|
]).where("run_id", "=", msg.runId).orderBy("created_at", "asc").execute();
|
|
8096
8204
|
let steps = await this.db.selectFrom("execution_steps").select([
|
|
8097
8205
|
"job_id",
|
|
@@ -8133,6 +8241,7 @@ var init_handler = __esmMin((() => {
|
|
|
8133
8241
|
}
|
|
8134
8242
|
const responseJobs = jobs.map((job) => {
|
|
8135
8243
|
const jobSteps = stepsByJob.get(job.job_id) ?? [];
|
|
8244
|
+
const jobInitFailure = job.init_failure ?? void 0;
|
|
8136
8245
|
return {
|
|
8137
8246
|
jobId: job.job_id,
|
|
8138
8247
|
jobName: job.job_name,
|
|
@@ -8147,6 +8256,7 @@ var init_handler = __esmMin((() => {
|
|
|
8147
8256
|
runsOnLabels: job.runs_on_labels ?? null,
|
|
8148
8257
|
outputs: job.outputs ? typeof job.outputs === "string" ? JSON.parse(job.outputs) : job.outputs : null,
|
|
8149
8258
|
secretOutputKeys: secretKeysByJob.get(job.job_id) ?? null,
|
|
8259
|
+
...jobInitFailure && { initFailure: jobInitFailure },
|
|
8150
8260
|
steps: jobSteps.map((step) => ({
|
|
8151
8261
|
stepIndex: step.step_index,
|
|
8152
8262
|
stepName: step.step_name,
|
|
@@ -8166,9 +8276,11 @@ var init_handler = __esmMin((() => {
|
|
|
8166
8276
|
lockFileSource: runRow.lock_file_source ?? null,
|
|
8167
8277
|
contributorUsername: runRow.contributor_username ?? null
|
|
8168
8278
|
} : void 0;
|
|
8279
|
+
const runInitFailure = runRow?.init_failure ?? void 0;
|
|
8169
8280
|
const validated = dashboardRunDetailApiResponseSchema.safeParse({
|
|
8170
8281
|
jobs: responseJobs,
|
|
8171
|
-
...trustContext && { trustContext }
|
|
8282
|
+
...trustContext && { trustContext },
|
|
8283
|
+
...runInitFailure && { initFailure: runInitFailure }
|
|
8172
8284
|
});
|
|
8173
8285
|
if (!validated.success) {
|
|
8174
8286
|
logger$86.error("Outgoing dashboard.run.detail response validation failed", {
|
|
@@ -8195,7 +8307,8 @@ var init_handler = __esmMin((() => {
|
|
|
8195
8307
|
type: "dashboard.run.detail.response",
|
|
8196
8308
|
requestId: msg.requestId,
|
|
8197
8309
|
jobs: validated.data.jobs,
|
|
8198
|
-
...validated.data.trustContext && { trustContext: validated.data.trustContext }
|
|
8310
|
+
...validated.data.trustContext && { trustContext: validated.data.trustContext },
|
|
8311
|
+
...validated.data.initFailure && { initFailure: validated.data.initFailure }
|
|
8199
8312
|
});
|
|
8200
8313
|
} catch (err) {
|
|
8201
8314
|
logger$86.error("Error handling dashboard.run.detail", {
|
|
@@ -9195,7 +9308,8 @@ var init_handler = __esmMin((() => {
|
|
|
9195
9308
|
out.run = {
|
|
9196
9309
|
trust_tier: typeof r.trust_tier === "string" ? r.trust_tier : null,
|
|
9197
9310
|
lock_file_source: typeof r.lock_file_source === "string" ? r.lock_file_source : null,
|
|
9198
|
-
contributor_username: typeof r.contributor_username === "string" ? r.contributor_username : null
|
|
9311
|
+
contributor_username: typeof r.contributor_username === "string" ? r.contributor_username : null,
|
|
9312
|
+
init_failure: r.init_failure ?? null
|
|
9199
9313
|
};
|
|
9200
9314
|
}
|
|
9201
9315
|
out.jobs = jobRows.map((r) => ({
|
|
@@ -9209,7 +9323,8 @@ var init_handler = __esmMin((() => {
|
|
|
9209
9323
|
duration_ms: typeof r.duration_ms === "number" ? r.duration_ms : null,
|
|
9210
9324
|
error_message: typeof r.error_message === "string" ? r.error_message : null,
|
|
9211
9325
|
runs_on_labels: typeof r.runs_on_labels === "string" ? r.runs_on_labels : null,
|
|
9212
|
-
outputs: typeof r.outputs === "string" ? r.outputs : null
|
|
9326
|
+
outputs: typeof r.outputs === "string" ? r.outputs : null,
|
|
9327
|
+
init_failure: r.init_failure ?? null
|
|
9213
9328
|
}));
|
|
9214
9329
|
stepRows.sort((a, b) => Number(a.step_index ?? 0) - Number(b.step_index ?? 0));
|
|
9215
9330
|
out.steps = stepRows.map((r) => ({
|
|
@@ -15175,7 +15290,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
15175
15290
|
init_peer_crypto();
|
|
15176
15291
|
init_peer_credentials();
|
|
15177
15292
|
logger$74 = createLogger({ prefix: "peer-client" });
|
|
15178
|
-
SOFTWARE_VERSION$1 = "0.1.
|
|
15293
|
+
SOFTWARE_VERSION$1 = "0.1.14";
|
|
15179
15294
|
PeerClient$1 = class {
|
|
15180
15295
|
ws = null;
|
|
15181
15296
|
_state = "disconnected";
|
|
@@ -16641,7 +16756,7 @@ var init_peer_handler = __esmMin((() => {
|
|
|
16641
16756
|
init_peer_crypto();
|
|
16642
16757
|
init_join_token();
|
|
16643
16758
|
logger$73 = createLogger({ prefix: "peer-handler" });
|
|
16644
|
-
SOFTWARE_VERSION = "0.1.
|
|
16759
|
+
SOFTWARE_VERSION = "0.1.14";
|
|
16645
16760
|
RATE_LIMIT_MAX = 5;
|
|
16646
16761
|
RATE_LIMIT_WINDOW_MS = 6e4;
|
|
16647
16762
|
}));
|
|
@@ -17904,15 +18019,15 @@ var init_cluster = __esmMin((() => {
|
|
|
17904
18019
|
* Uses a 15-minute TTL and 10,000 max entries.
|
|
17905
18020
|
*/
|
|
17906
18021
|
var contributor_cache_exports = /* @__PURE__ */ __exportAll({ ContributorCache: () => ContributorCache$1 });
|
|
17907
|
-
var DEFAULT_TTL_MS$1, DEFAULT_MAX_ENTRIES, ContributorCache$1;
|
|
18022
|
+
var DEFAULT_TTL_MS$1, DEFAULT_MAX_ENTRIES$1, ContributorCache$1;
|
|
17908
18023
|
var init_contributor_cache = __esmMin((() => {
|
|
17909
18024
|
DEFAULT_TTL_MS$1 = 900 * 1e3;
|
|
17910
|
-
DEFAULT_MAX_ENTRIES = 1e4;
|
|
18025
|
+
DEFAULT_MAX_ENTRIES$1 = 1e4;
|
|
17911
18026
|
ContributorCache$1 = class {
|
|
17912
18027
|
cache;
|
|
17913
18028
|
constructor(options) {
|
|
17914
18029
|
this.cache = new LRUCache({
|
|
17915
|
-
max: options?.maxEntries ?? DEFAULT_MAX_ENTRIES,
|
|
18030
|
+
max: options?.maxEntries ?? DEFAULT_MAX_ENTRIES$1,
|
|
17916
18031
|
ttl: options?.ttlMs ?? DEFAULT_TTL_MS$1
|
|
17917
18032
|
});
|
|
17918
18033
|
}
|
|
@@ -27326,10 +27441,10 @@ var init_admin_sources = __esmMin((() => {
|
|
|
27326
27441
|
//#endregion
|
|
27327
27442
|
//#region src/db/migrations/001_initial.ts
|
|
27328
27443
|
var _001_initial_exports = /* @__PURE__ */ __exportAll({
|
|
27329
|
-
down: () => down$
|
|
27330
|
-
up: () => up$
|
|
27444
|
+
down: () => down$24,
|
|
27445
|
+
up: () => up$24
|
|
27331
27446
|
});
|
|
27332
|
-
async function up$
|
|
27447
|
+
async function up$24(db) {
|
|
27333
27448
|
for (const stmt of DDL_STATEMENTS) await sql.raw(stmt).execute(db);
|
|
27334
27449
|
await sql`
|
|
27335
27450
|
INSERT INTO cluster_meta (key, value)
|
|
@@ -27351,7 +27466,7 @@ async function up$23(db) {
|
|
|
27351
27466
|
* Rollback drops everything created above. Uses CASCADE on table drops to cut
|
|
27352
27467
|
* through the FK graph without relying on exact topological order.
|
|
27353
27468
|
*/
|
|
27354
|
-
async function down$
|
|
27469
|
+
async function down$24(db) {
|
|
27355
27470
|
for (const [trig, tbl] of [["source_secrets_change_trigger", "scoped_secrets"], ["sources_change_trigger", "sources"]]) await sql.raw(`DROP TRIGGER IF EXISTS ${trig} ON public.${tbl}`).execute(db);
|
|
27356
27471
|
for (const table of [
|
|
27357
27472
|
"workflow_registrations",
|
|
@@ -28071,8 +28186,8 @@ var init__001_initial = __esmMin((() => {
|
|
|
28071
28186
|
//#endregion
|
|
28072
28187
|
//#region src/db/migrations/002_config_versions_key_version.ts
|
|
28073
28188
|
var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
28074
|
-
down: () => down$
|
|
28075
|
-
up: () => up$
|
|
28189
|
+
down: () => down$23,
|
|
28190
|
+
up: () => up$23
|
|
28076
28191
|
});
|
|
28077
28192
|
/**
|
|
28078
28193
|
* Add key_version column to config_versions so that sensitive-field encryption
|
|
@@ -28086,13 +28201,13 @@ var _002_config_versions_key_version_exports = /* @__PURE__ */ __exportAll({
|
|
|
28086
28201
|
* No index is needed: rotation does a full-table scan; reads are by
|
|
28087
28202
|
* `version` primary key and never filter on `key_version`.
|
|
28088
28203
|
*/
|
|
28089
|
-
async function up$
|
|
28204
|
+
async function up$23(db) {
|
|
28090
28205
|
await sql`
|
|
28091
28206
|
ALTER TABLE public.config_versions
|
|
28092
28207
|
ADD COLUMN key_version integer NOT NULL DEFAULT 1
|
|
28093
28208
|
`.execute(db);
|
|
28094
28209
|
}
|
|
28095
|
-
async function down$
|
|
28210
|
+
async function down$23(db) {
|
|
28096
28211
|
await sql`
|
|
28097
28212
|
ALTER TABLE public.config_versions
|
|
28098
28213
|
DROP COLUMN key_version
|
|
@@ -28102,8 +28217,8 @@ var init__002_config_versions_key_version = __esmMin((() => {}));
|
|
|
28102
28217
|
//#endregion
|
|
28103
28218
|
//#region src/db/migrations/003_access_log.ts
|
|
28104
28219
|
var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
28105
|
-
down: () => down$
|
|
28106
|
-
up: () => up$
|
|
28220
|
+
down: () => down$22,
|
|
28221
|
+
up: () => up$22
|
|
28107
28222
|
});
|
|
28108
28223
|
/**
|
|
28109
28224
|
* Access log: one row per read or orchestrator-admin mutation attributable
|
|
@@ -28127,7 +28242,7 @@ var _003_access_log_exports = /* @__PURE__ */ __exportAll({
|
|
|
28127
28242
|
* Retention is TTL-based via expires_at; packages/orchestrator/src/queue/
|
|
28128
28243
|
* cleanup.ts picks up the prune pass.
|
|
28129
28244
|
*/
|
|
28130
|
-
async function up$
|
|
28245
|
+
async function up$22(db) {
|
|
28131
28246
|
await sql`
|
|
28132
28247
|
CREATE TABLE public.access_log (
|
|
28133
28248
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
@@ -28165,21 +28280,21 @@ async function up$21(db) {
|
|
|
28165
28280
|
ON public.access_log (actor_type, actor_id, created_at DESC)
|
|
28166
28281
|
`.execute(db);
|
|
28167
28282
|
}
|
|
28168
|
-
async function down$
|
|
28283
|
+
async function down$22(db) {
|
|
28169
28284
|
await sql`DROP TABLE IF EXISTS public.access_log`.execute(db);
|
|
28170
28285
|
}
|
|
28171
28286
|
var init__003_access_log = __esmMin((() => {}));
|
|
28172
28287
|
//#endregion
|
|
28173
28288
|
//#region src/db/migrations/004_rename_bundle_to_source.ts
|
|
28174
28289
|
var _004_rename_bundle_to_source_exports = /* @__PURE__ */ __exportAll({
|
|
28175
|
-
down: () => down$
|
|
28176
|
-
up: () => up$
|
|
28290
|
+
down: () => down$21,
|
|
28291
|
+
up: () => up$21
|
|
28177
28292
|
});
|
|
28178
|
-
async function up$
|
|
28293
|
+
async function up$21(db) {
|
|
28179
28294
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_url", "source_tar_url").execute();
|
|
28180
28295
|
await db.schema.alterTable("dispatch_queue").renameColumn("bundle_hash", "source_tar_hash").execute();
|
|
28181
28296
|
}
|
|
28182
|
-
async function down$
|
|
28297
|
+
async function down$21(db) {
|
|
28183
28298
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_url", "bundle_url").execute();
|
|
28184
28299
|
await db.schema.alterTable("dispatch_queue").renameColumn("source_tar_hash", "bundle_hash").execute();
|
|
28185
28300
|
}
|
|
@@ -28187,8 +28302,8 @@ var init__004_rename_bundle_to_source = __esmMin((() => {}));
|
|
|
28187
28302
|
//#endregion
|
|
28188
28303
|
//#region src/db/migrations/005_cold_store_chunk_counter.ts
|
|
28189
28304
|
var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
28190
|
-
down: () => down$
|
|
28191
|
-
up: () => up$
|
|
28305
|
+
down: () => down$20,
|
|
28306
|
+
up: () => up$20
|
|
28192
28307
|
});
|
|
28193
28308
|
/**
|
|
28194
28309
|
* Cold-store chunk counter table.
|
|
@@ -28205,7 +28320,7 @@ var _005_cold_store_chunk_counter_exports = /* @__PURE__ */ __exportAll({
|
|
|
28205
28320
|
*
|
|
28206
28321
|
* sections 5 and 8.
|
|
28207
28322
|
*/
|
|
28208
|
-
async function up$
|
|
28323
|
+
async function up$20(db) {
|
|
28209
28324
|
await sql`
|
|
28210
28325
|
CREATE TABLE public.cold_store_chunk_counts (
|
|
28211
28326
|
db TEXT NOT NULL,
|
|
@@ -28223,15 +28338,15 @@ async function up$19(db) {
|
|
|
28223
28338
|
ON public.cold_store_chunk_counts (db, table_name)
|
|
28224
28339
|
`.execute(db);
|
|
28225
28340
|
}
|
|
28226
|
-
async function down$
|
|
28341
|
+
async function down$20(db) {
|
|
28227
28342
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunk_counts`.execute(db);
|
|
28228
28343
|
}
|
|
28229
28344
|
var init__005_cold_store_chunk_counter = __esmMin((() => {}));
|
|
28230
28345
|
//#endregion
|
|
28231
28346
|
//#region src/db/migrations/006_runs_jobs_steps_archived_at.ts
|
|
28232
28347
|
var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
28233
|
-
down: () => down$
|
|
28234
|
-
up: () => up$
|
|
28348
|
+
down: () => down$19,
|
|
28349
|
+
up: () => up$19
|
|
28235
28350
|
});
|
|
28236
28351
|
/**
|
|
28237
28352
|
* `execution_runs` / `execution_jobs` / `execution_steps` cold-store
|
|
@@ -28267,7 +28382,7 @@ var _006_runs_jobs_steps_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
28267
28382
|
* - `idx_execution_jobs_routing_key_created (routing_key, created_at)`
|
|
28268
28383
|
* - `idx_execution_steps_routing_key_created (routing_key, created_at)`
|
|
28269
28384
|
*/
|
|
28270
|
-
async function up$
|
|
28385
|
+
async function up$19(db) {
|
|
28271
28386
|
await sql`
|
|
28272
28387
|
ALTER TABLE public.execution_runs
|
|
28273
28388
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -28312,7 +28427,7 @@ async function up$18(db) {
|
|
|
28312
28427
|
ON public.execution_steps (routing_key, created_at)
|
|
28313
28428
|
`.execute(db);
|
|
28314
28429
|
}
|
|
28315
|
-
async function down$
|
|
28430
|
+
async function down$19(db) {
|
|
28316
28431
|
await sql`DROP INDEX IF EXISTS public.idx_execution_steps_routing_key_created`.execute(db);
|
|
28317
28432
|
await sql`
|
|
28318
28433
|
ALTER TABLE public.execution_steps
|
|
@@ -28338,8 +28453,8 @@ var init__006_runs_jobs_steps_archived_at = __esmMin((() => {}));
|
|
|
28338
28453
|
//#endregion
|
|
28339
28454
|
//#region src/db/migrations/007_audit_logs_archived_at.ts
|
|
28340
28455
|
var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
28341
|
-
down: () => down$
|
|
28342
|
-
up: () => up$
|
|
28456
|
+
down: () => down$18,
|
|
28457
|
+
up: () => up$18
|
|
28343
28458
|
});
|
|
28344
28459
|
/**
|
|
28345
28460
|
* `secret_audit_log` and `access_log` cold-store schema additions, plus
|
|
@@ -28379,7 +28494,7 @@ var _007_audit_logs_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
28379
28494
|
* `down()` would not have meaningful retention bounds. Acceptable for
|
|
28380
28495
|
* staging.
|
|
28381
28496
|
*/
|
|
28382
|
-
async function up$
|
|
28497
|
+
async function up$18(db) {
|
|
28383
28498
|
await sql`
|
|
28384
28499
|
ALTER TABLE public.secret_audit_log
|
|
28385
28500
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -28400,7 +28515,7 @@ async function up$17(db) {
|
|
|
28400
28515
|
DROP COLUMN IF EXISTS expires_at
|
|
28401
28516
|
`.execute(db);
|
|
28402
28517
|
}
|
|
28403
|
-
async function down$
|
|
28518
|
+
async function down$18(db) {
|
|
28404
28519
|
await sql`
|
|
28405
28520
|
ALTER TABLE public.access_log
|
|
28406
28521
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '90 days')
|
|
@@ -28429,8 +28544,8 @@ var init__007_audit_logs_archived_at = __esmMin((() => {}));
|
|
|
28429
28544
|
//#endregion
|
|
28430
28545
|
//#region src/db/migrations/008_event_log_archived_at.ts
|
|
28431
28546
|
var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
28432
|
-
down: () => down$
|
|
28433
|
-
up: () => up$
|
|
28547
|
+
down: () => down$17,
|
|
28548
|
+
up: () => up$17
|
|
28434
28549
|
});
|
|
28435
28550
|
/**
|
|
28436
28551
|
* `event_log` cold-store schema additions plus removal of the
|
|
@@ -28468,7 +28583,7 @@ var _008_event_log_archived_at_exports = /* @__PURE__ */ __exportAll({
|
|
|
28468
28583
|
* — best effort; rows inserted between `up()` and a hypothetical
|
|
28469
28584
|
* `down()` would not have meaningful retention bounds.
|
|
28470
28585
|
*/
|
|
28471
|
-
async function up$
|
|
28586
|
+
async function up$17(db) {
|
|
28472
28587
|
await sql`
|
|
28473
28588
|
ALTER TABLE public.event_log
|
|
28474
28589
|
ADD COLUMN archived_at TIMESTAMPTZ NULL,
|
|
@@ -28484,7 +28599,7 @@ async function up$16(db) {
|
|
|
28484
28599
|
DROP COLUMN IF EXISTS expires_at
|
|
28485
28600
|
`.execute(db);
|
|
28486
28601
|
}
|
|
28487
|
-
async function down$
|
|
28602
|
+
async function down$17(db) {
|
|
28488
28603
|
await sql`
|
|
28489
28604
|
ALTER TABLE public.event_log
|
|
28490
28605
|
ADD COLUMN expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + INTERVAL '30 days')
|
|
@@ -28504,8 +28619,8 @@ var init__008_event_log_archived_at = __esmMin((() => {}));
|
|
|
28504
28619
|
//#endregion
|
|
28505
28620
|
//#region src/db/migrations/009_access_log_trigram.ts
|
|
28506
28621
|
var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
28507
|
-
down: () => down$
|
|
28508
|
-
up: () => up$
|
|
28622
|
+
down: () => down$16,
|
|
28623
|
+
up: () => up$16
|
|
28509
28624
|
});
|
|
28510
28625
|
/**
|
|
28511
28626
|
* Trigram (pg_trgm) index on access_log.error_message for the federated
|
|
@@ -28518,7 +28633,7 @@ var _009_access_log_trigram_exports = /* @__PURE__ */ __exportAll({
|
|
|
28518
28633
|
* EXISTS` are both safe to re-run. No CONCURRENTLY because Kysely runs
|
|
28519
28634
|
* migrations inside a transaction; the lock is brief on a sampled table.
|
|
28520
28635
|
*/
|
|
28521
|
-
async function up$
|
|
28636
|
+
async function up$16(db) {
|
|
28522
28637
|
await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
|
|
28523
28638
|
await sql`
|
|
28524
28639
|
CREATE INDEX IF NOT EXISTS access_log_error_message_trgm_idx
|
|
@@ -28527,15 +28642,15 @@ async function up$15(db) {
|
|
|
28527
28642
|
WHERE error_message IS NOT NULL
|
|
28528
28643
|
`.execute(db);
|
|
28529
28644
|
}
|
|
28530
|
-
async function down$
|
|
28645
|
+
async function down$16(db) {
|
|
28531
28646
|
await sql`DROP INDEX IF EXISTS public.access_log_error_message_trgm_idx`.execute(db);
|
|
28532
28647
|
}
|
|
28533
28648
|
var init__009_access_log_trigram = __esmMin((() => {}));
|
|
28534
28649
|
//#endregion
|
|
28535
28650
|
//#region src/db/migrations/010_cold_store_chunks.ts
|
|
28536
28651
|
var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
28537
|
-
down: () => down$
|
|
28538
|
-
up: () => up$
|
|
28652
|
+
down: () => down$15,
|
|
28653
|
+
up: () => up$15
|
|
28539
28654
|
});
|
|
28540
28655
|
/**
|
|
28541
28656
|
* Cold-store chunk index — Phase 2 (cold-store purge).
|
|
@@ -28567,7 +28682,7 @@ var _010_cold_store_chunks_exports = /* @__PURE__ */ __exportAll({
|
|
|
28567
28682
|
* forever. Adapters that don't opt into per-bucket archival via
|
|
28568
28683
|
* `coldTtlDays` don't insert here either.
|
|
28569
28684
|
*/
|
|
28570
|
-
async function up$
|
|
28685
|
+
async function up$15(db) {
|
|
28571
28686
|
await sql`
|
|
28572
28687
|
CREATE TABLE public.cold_store_chunks (
|
|
28573
28688
|
db TEXT NOT NULL,
|
|
@@ -28594,15 +28709,15 @@ async function up$14(db) {
|
|
|
28594
28709
|
ON public.cold_store_chunks (db, table_name, tenant_id, archived_at DESC)
|
|
28595
28710
|
`.execute(db);
|
|
28596
28711
|
}
|
|
28597
|
-
async function down$
|
|
28712
|
+
async function down$15(db) {
|
|
28598
28713
|
await sql`DROP TABLE IF EXISTS public.cold_store_chunks`.execute(db);
|
|
28599
28714
|
}
|
|
28600
28715
|
var init__010_cold_store_chunks = __esmMin((() => {}));
|
|
28601
28716
|
//#endregion
|
|
28602
28717
|
//#region src/db/migrations/011_drop_source_secrets_notify.ts
|
|
28603
28718
|
var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
28604
|
-
down: () => down$
|
|
28605
|
-
up: () => up$
|
|
28719
|
+
down: () => down$14,
|
|
28720
|
+
up: () => up$14
|
|
28606
28721
|
});
|
|
28607
28722
|
/**
|
|
28608
28723
|
* Drop the `source_secrets_change_trigger` and the
|
|
@@ -28621,11 +28736,11 @@ var _011_drop_source_secrets_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
28621
28736
|
* in `001_initial.ts`. They wake up no consumer until the `WebhookSecretManager`
|
|
28622
28737
|
* is restored, so this migration is safe to roll back.
|
|
28623
28738
|
*/
|
|
28624
|
-
async function up$
|
|
28739
|
+
async function up$14(db) {
|
|
28625
28740
|
await sql`DROP TRIGGER IF EXISTS source_secrets_change_trigger ON public.scoped_secrets`.execute(db);
|
|
28626
28741
|
await sql`DROP FUNCTION IF EXISTS public.notify_source_secrets_change() CASCADE`.execute(db);
|
|
28627
28742
|
}
|
|
28628
|
-
async function down$
|
|
28743
|
+
async function down$14(db) {
|
|
28629
28744
|
await sql`
|
|
28630
28745
|
CREATE OR REPLACE FUNCTION public.notify_source_secrets_change() RETURNS trigger
|
|
28631
28746
|
LANGUAGE plpgsql
|
|
@@ -28665,8 +28780,8 @@ var init__011_drop_source_secrets_notify = __esmMin((() => {}));
|
|
|
28665
28780
|
//#endregion
|
|
28666
28781
|
//#region src/db/migrations/012_peer_credentials_active_uniq.ts
|
|
28667
28782
|
var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
28668
|
-
down: () => down$
|
|
28669
|
-
up: () => up$
|
|
28783
|
+
down: () => down$13,
|
|
28784
|
+
up: () => up$13
|
|
28670
28785
|
});
|
|
28671
28786
|
/**
|
|
28672
28787
|
* Add a partial unique index on `peer_credentials (instance_id) WHERE
|
|
@@ -28692,7 +28807,7 @@ var _012_peer_credentials_active_uniq_exports = /* @__PURE__ */ __exportAll({
|
|
|
28692
28807
|
* `down()` only drops the index; it does NOT undo the dedupe (there's no
|
|
28693
28808
|
* safe way to recreate revoked rows, and the dedupe is monotonic).
|
|
28694
28809
|
*/
|
|
28695
|
-
async function up$
|
|
28810
|
+
async function up$13(db) {
|
|
28696
28811
|
await sql`
|
|
28697
28812
|
UPDATE public.peer_credentials
|
|
28698
28813
|
SET revoked_at = NOW()
|
|
@@ -28710,15 +28825,15 @@ async function up$12(db) {
|
|
|
28710
28825
|
WHERE revoked_at IS NULL
|
|
28711
28826
|
`.execute(db);
|
|
28712
28827
|
}
|
|
28713
|
-
async function down$
|
|
28828
|
+
async function down$13(db) {
|
|
28714
28829
|
await sql`DROP INDEX IF EXISTS public.peer_credentials_active_uniq`.execute(db);
|
|
28715
28830
|
}
|
|
28716
28831
|
var init__012_peer_credentials_active_uniq = __esmMin((() => {}));
|
|
28717
28832
|
//#endregion
|
|
28718
28833
|
//#region src/db/migrations/013_execution_log_bytes.ts
|
|
28719
28834
|
var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
28720
|
-
down: () => down$
|
|
28721
|
-
up: () => up$
|
|
28835
|
+
down: () => down$12,
|
|
28836
|
+
up: () => up$12
|
|
28722
28837
|
});
|
|
28723
28838
|
/**
|
|
28724
28839
|
* Add `log_bytes BIGINT NOT NULL DEFAULT 0` columns to `execution_runs` and
|
|
@@ -28737,7 +28852,7 @@ var _013_execution_log_bytes_exports = /* @__PURE__ */ __exportAll({
|
|
|
28737
28852
|
*
|
|
28738
28853
|
* Idempotent (`ADD COLUMN IF NOT EXISTS`).
|
|
28739
28854
|
*/
|
|
28740
|
-
async function up$
|
|
28855
|
+
async function up$12(db) {
|
|
28741
28856
|
await sql`
|
|
28742
28857
|
ALTER TABLE public.execution_runs
|
|
28743
28858
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
@@ -28747,7 +28862,7 @@ async function up$11(db) {
|
|
|
28747
28862
|
ADD COLUMN IF NOT EXISTS log_bytes BIGINT NOT NULL DEFAULT 0
|
|
28748
28863
|
`.execute(db);
|
|
28749
28864
|
}
|
|
28750
|
-
async function down$
|
|
28865
|
+
async function down$12(db) {
|
|
28751
28866
|
await sql`ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
28752
28867
|
await sql`ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS log_bytes`.execute(db);
|
|
28753
28868
|
}
|
|
@@ -28755,8 +28870,8 @@ var init__013_execution_log_bytes = __esmMin((() => {}));
|
|
|
28755
28870
|
//#endregion
|
|
28756
28871
|
//#region src/db/migrations/014_kici_events_lease_retry.ts
|
|
28757
28872
|
var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
28758
|
-
down: () => down$
|
|
28759
|
-
up: () => up$
|
|
28873
|
+
down: () => down$11,
|
|
28874
|
+
up: () => up$11
|
|
28760
28875
|
});
|
|
28761
28876
|
/**
|
|
28762
28877
|
* Add lease + retry + DLQ columns to `kici_events` so the EventRouter can
|
|
@@ -28790,7 +28905,7 @@ var _014_kici_events_lease_retry_exports = /* @__PURE__ */ __exportAll({
|
|
|
28790
28905
|
*
|
|
28791
28906
|
* Idempotent (`ADD COLUMN IF NOT EXISTS` + `CREATE INDEX IF NOT EXISTS`).
|
|
28792
28907
|
*/
|
|
28793
|
-
async function up$
|
|
28908
|
+
async function up$11(db) {
|
|
28794
28909
|
await sql`
|
|
28795
28910
|
ALTER TABLE public.kici_events
|
|
28796
28911
|
ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ,
|
|
@@ -28821,7 +28936,7 @@ async function up$10(db) {
|
|
|
28821
28936
|
WHERE dlq_at IS NOT NULL
|
|
28822
28937
|
`.execute(db);
|
|
28823
28938
|
}
|
|
28824
|
-
async function down$
|
|
28939
|
+
async function down$11(db) {
|
|
28825
28940
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_dlq`.execute(db);
|
|
28826
28941
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_lease_expired`.execute(db);
|
|
28827
28942
|
await sql`DROP INDEX IF EXISTS public.idx_kici_events_retry_due`.execute(db);
|
|
@@ -28840,8 +28955,8 @@ var init__014_kici_events_lease_retry = __esmMin((() => {}));
|
|
|
28840
28955
|
//#endregion
|
|
28841
28956
|
//#region src/db/migrations/015_org_settings_customer_scoped.ts
|
|
28842
28957
|
var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
28843
|
-
down: () => down$
|
|
28844
|
-
up: () => up$
|
|
28958
|
+
down: () => down$10,
|
|
28959
|
+
up: () => up$10
|
|
28845
28960
|
});
|
|
28846
28961
|
/**
|
|
28847
28962
|
* Org-scope `org_settings` and qualify each glob entry by source.
|
|
@@ -28869,7 +28984,7 @@ var _015_org_settings_customer_scoped_exports = /* @__PURE__ */ __exportAll({
|
|
|
28869
28984
|
* Idempotent: a re-run on an already-migrated DB sees `customer_id` exists
|
|
28870
28985
|
* and the list columns are already jsonb, so it is a no-op.
|
|
28871
28986
|
*/
|
|
28872
|
-
async function up$
|
|
28987
|
+
async function up$10(db) {
|
|
28873
28988
|
if ((await sql`
|
|
28874
28989
|
SELECT EXISTS (
|
|
28875
28990
|
SELECT 1 FROM information_schema.columns
|
|
@@ -28994,7 +29109,7 @@ async function up$9(db) {
|
|
|
28994
29109
|
await sql`DROP TABLE _org_settings_merged`.execute(db);
|
|
28995
29110
|
await sql`DROP TABLE _org_settings_stage`.execute(db);
|
|
28996
29111
|
}
|
|
28997
|
-
async function down$
|
|
29112
|
+
async function down$10(db) {
|
|
28998
29113
|
if (!(await sql`
|
|
28999
29114
|
SELECT EXISTS (
|
|
29000
29115
|
SELECT 1 FROM information_schema.columns
|
|
@@ -29020,8 +29135,8 @@ var init__015_org_settings_customer_scoped = __esmMin((() => {}));
|
|
|
29020
29135
|
//#endregion
|
|
29021
29136
|
//#region src/db/migrations/016_org_settings_allow_http_npm.ts
|
|
29022
29137
|
var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
29023
|
-
down: () => down$
|
|
29024
|
-
up: () => up$
|
|
29138
|
+
down: () => down$9,
|
|
29139
|
+
up: () => up$9
|
|
29025
29140
|
});
|
|
29026
29141
|
/**
|
|
29027
29142
|
* Add `org_settings.allow_http_npm_registries boolean NOT NULL DEFAULT false`.
|
|
@@ -29034,7 +29149,7 @@ var _016_org_settings_allow_http_npm_exports = /* @__PURE__ */ __exportAll({
|
|
|
29034
29149
|
*
|
|
29035
29150
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
29036
29151
|
*/
|
|
29037
|
-
async function up$
|
|
29152
|
+
async function up$9(db) {
|
|
29038
29153
|
if ((await sql`
|
|
29039
29154
|
SELECT EXISTS (
|
|
29040
29155
|
SELECT 1 FROM information_schema.columns
|
|
@@ -29048,7 +29163,7 @@ async function up$8(db) {
|
|
|
29048
29163
|
ADD COLUMN allow_http_npm_registries boolean NOT NULL DEFAULT false
|
|
29049
29164
|
`.execute(db);
|
|
29050
29165
|
}
|
|
29051
|
-
async function down$
|
|
29166
|
+
async function down$9(db) {
|
|
29052
29167
|
await sql`
|
|
29053
29168
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS allow_http_npm_registries
|
|
29054
29169
|
`.execute(db);
|
|
@@ -29057,13 +29172,13 @@ var init__016_org_settings_allow_http_npm = __esmMin((() => {}));
|
|
|
29057
29172
|
//#endregion
|
|
29058
29173
|
//#region src/db/migrations/017_org_id_widen.ts
|
|
29059
29174
|
var _017_org_id_widen_exports = /* @__PURE__ */ __exportAll({
|
|
29060
|
-
down: () => down$
|
|
29061
|
-
up: () => up$
|
|
29175
|
+
down: () => down$8,
|
|
29176
|
+
up: () => up$8
|
|
29062
29177
|
});
|
|
29063
|
-
async function up$
|
|
29178
|
+
async function up$8(db) {
|
|
29064
29179
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(16)`).execute(db);
|
|
29065
29180
|
}
|
|
29066
|
-
async function down$
|
|
29181
|
+
async function down$8(db) {
|
|
29067
29182
|
for (const table of ORG_ID_TABLES$1) await sql.raw(`ALTER TABLE public.${table} ALTER COLUMN org_id TYPE varchar(12)`).execute(db);
|
|
29068
29183
|
}
|
|
29069
29184
|
var ORG_ID_TABLES$1;
|
|
@@ -29082,14 +29197,14 @@ var init__017_org_id_widen = __esmMin((() => {
|
|
|
29082
29197
|
//#endregion
|
|
29083
29198
|
//#region src/db/migrations/018_org_id_prefix_backfill.ts
|
|
29084
29199
|
var _018_org_id_prefix_backfill_exports = /* @__PURE__ */ __exportAll({
|
|
29085
|
-
down: () => down$
|
|
29086
|
-
up: () => up$
|
|
29200
|
+
down: () => down$7,
|
|
29201
|
+
up: () => up$7
|
|
29087
29202
|
});
|
|
29088
|
-
async function up$
|
|
29203
|
+
async function up$7(db) {
|
|
29089
29204
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = 'org_' || org_id WHERE org_id <> 'kici-admin' AND org_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
29090
29205
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = 'org_' || customer_id WHERE customer_id <> 'kici-admin' AND customer_id NOT LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
29091
29206
|
}
|
|
29092
|
-
async function down$
|
|
29207
|
+
async function down$7(db) {
|
|
29093
29208
|
for (const table of CUSTOMER_ID_TABLES) await sql.raw(`UPDATE public.${table} SET customer_id = substring(customer_id from 5) WHERE customer_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
29094
29209
|
for (const table of ORG_ID_TABLES) await sql.raw(`UPDATE public.${table} SET org_id = substring(org_id from 5) WHERE org_id LIKE 'org\\_%' ESCAPE '\\'`).execute(db);
|
|
29095
29210
|
}
|
|
@@ -29115,8 +29230,8 @@ var init__018_org_id_prefix_backfill = __esmMin((() => {
|
|
|
29115
29230
|
//#endregion
|
|
29116
29231
|
//#region src/db/migrations/019_generic_sources_change_notify.ts
|
|
29117
29232
|
var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
29118
|
-
down: () => down$
|
|
29119
|
-
up: () => up$
|
|
29233
|
+
down: () => down$6,
|
|
29234
|
+
up: () => up$6
|
|
29120
29235
|
});
|
|
29121
29236
|
/**
|
|
29122
29237
|
* Add a Postgres trigger on `generic_webhook_sources` that emits
|
|
@@ -29139,7 +29254,7 @@ var _019_generic_sources_change_notify_exports = /* @__PURE__ */ __exportAll({
|
|
|
29139
29254
|
* (`notify_sources_change()` + `sources_change_trigger`, defined in
|
|
29140
29255
|
* `001_initial.ts`).
|
|
29141
29256
|
*/
|
|
29142
|
-
async function up$
|
|
29257
|
+
async function up$6(db) {
|
|
29143
29258
|
await sql`
|
|
29144
29259
|
CREATE FUNCTION public.notify_generic_sources_change() RETURNS trigger
|
|
29145
29260
|
LANGUAGE plpgsql
|
|
@@ -29160,7 +29275,7 @@ async function up$5(db) {
|
|
|
29160
29275
|
FOR EACH ROW EXECUTE FUNCTION public.notify_generic_sources_change()
|
|
29161
29276
|
`.execute(db);
|
|
29162
29277
|
}
|
|
29163
|
-
async function down$
|
|
29278
|
+
async function down$6(db) {
|
|
29164
29279
|
await sql`DROP TRIGGER IF EXISTS generic_sources_change_trigger ON public.generic_webhook_sources`.execute(db);
|
|
29165
29280
|
await sql`DROP FUNCTION IF EXISTS public.notify_generic_sources_change()`.execute(db);
|
|
29166
29281
|
}
|
|
@@ -29168,8 +29283,8 @@ var init__019_generic_sources_change_notify = __esmMin((() => {}));
|
|
|
29168
29283
|
//#endregion
|
|
29169
29284
|
//#region src/db/migrations/020_org_settings_dashboard_write_policy.ts
|
|
29170
29285
|
var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportAll({
|
|
29171
|
-
down: () => down$
|
|
29172
|
-
up: () => up$
|
|
29286
|
+
down: () => down$5,
|
|
29287
|
+
up: () => up$5
|
|
29173
29288
|
});
|
|
29174
29289
|
/**
|
|
29175
29290
|
* Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
|
|
@@ -29184,7 +29299,7 @@ var _020_org_settings_dashboard_write_policy_exports = /* @__PURE__ */ __exportA
|
|
|
29184
29299
|
*
|
|
29185
29300
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
29186
29301
|
*/
|
|
29187
|
-
async function up$
|
|
29302
|
+
async function up$5(db) {
|
|
29188
29303
|
if ((await sql`
|
|
29189
29304
|
SELECT EXISTS (
|
|
29190
29305
|
SELECT 1 FROM information_schema.columns
|
|
@@ -29198,7 +29313,7 @@ async function up$4(db) {
|
|
|
29198
29313
|
ADD COLUMN dashboard_write_policy jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
29199
29314
|
`.execute(db);
|
|
29200
29315
|
}
|
|
29201
|
-
async function down$
|
|
29316
|
+
async function down$5(db) {
|
|
29202
29317
|
await sql`
|
|
29203
29318
|
ALTER TABLE public.org_settings DROP COLUMN IF EXISTS dashboard_write_policy
|
|
29204
29319
|
`.execute(db);
|
|
@@ -29207,8 +29322,8 @@ var init__020_org_settings_dashboard_write_policy = __esmMin((() => {}));
|
|
|
29207
29322
|
//#endregion
|
|
29208
29323
|
//#region src/db/migrations/021_check_run_tracking.ts
|
|
29209
29324
|
var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
29210
|
-
down: () => down$
|
|
29211
|
-
up: () => up$
|
|
29325
|
+
down: () => down$4,
|
|
29326
|
+
up: () => up$4
|
|
29212
29327
|
});
|
|
29213
29328
|
/**
|
|
29214
29329
|
* Add `check_run_tracking` table for HA-safe check-run state persistence.
|
|
@@ -29232,7 +29347,7 @@ var _021_check_run_tracking_exports = /* @__PURE__ */ __exportAll({
|
|
|
29232
29347
|
*
|
|
29233
29348
|
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
29234
29349
|
*/
|
|
29235
|
-
async function up$
|
|
29350
|
+
async function up$4(db) {
|
|
29236
29351
|
if ((await sql`
|
|
29237
29352
|
SELECT EXISTS (
|
|
29238
29353
|
SELECT 1 FROM information_schema.tables
|
|
@@ -29263,15 +29378,15 @@ async function up$3(db) {
|
|
|
29263
29378
|
WHERE run_id IS NOT NULL
|
|
29264
29379
|
`.execute(db);
|
|
29265
29380
|
}
|
|
29266
|
-
async function down$
|
|
29381
|
+
async function down$4(db) {
|
|
29267
29382
|
await sql`DROP TABLE IF EXISTS public.check_run_tracking`.execute(db);
|
|
29268
29383
|
}
|
|
29269
29384
|
var init__021_check_run_tracking = __esmMin((() => {}));
|
|
29270
29385
|
//#endregion
|
|
29271
29386
|
//#region src/db/migrations/022_scaler_manager_state.ts
|
|
29272
29387
|
var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
29273
|
-
down: () => down$
|
|
29274
|
-
up: () => up$
|
|
29388
|
+
down: () => down$3,
|
|
29389
|
+
up: () => up$3
|
|
29275
29390
|
});
|
|
29276
29391
|
/**
|
|
29277
29392
|
* Add three tables persisting `ScalerManager` per-coord state:
|
|
@@ -29298,7 +29413,7 @@ var _022_scaler_manager_state_exports = /* @__PURE__ */ __exportAll({
|
|
|
29298
29413
|
* Idempotent: a re-run on a DB that already has any of these tables
|
|
29299
29414
|
* leaves the existing one alone.
|
|
29300
29415
|
*/
|
|
29301
|
-
async function up$
|
|
29416
|
+
async function up$3(db) {
|
|
29302
29417
|
const tableExists = async (name) => {
|
|
29303
29418
|
return (await sql`
|
|
29304
29419
|
SELECT EXISTS (
|
|
@@ -29348,7 +29463,7 @@ async function up$2(db) {
|
|
|
29348
29463
|
`.execute(db);
|
|
29349
29464
|
}
|
|
29350
29465
|
}
|
|
29351
|
-
async function down$
|
|
29466
|
+
async function down$3(db) {
|
|
29352
29467
|
await sql`DROP TABLE IF EXISTS public.scaler_reservations`.execute(db);
|
|
29353
29468
|
await sql`DROP TABLE IF EXISTS public.scaler_agent_jobs`.execute(db);
|
|
29354
29469
|
await sql`DROP TABLE IF EXISTS public.scaler_spawning_agents`.execute(db);
|
|
@@ -29357,8 +29472,8 @@ var init__022_scaler_manager_state = __esmMin((() => {}));
|
|
|
29357
29472
|
//#endregion
|
|
29358
29473
|
//#region src/db/migrations/023_dispatch_queue_recovery_deadline.ts
|
|
29359
29474
|
var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll({
|
|
29360
|
-
down: () => down$
|
|
29361
|
-
up: () => up$
|
|
29475
|
+
down: () => down$2,
|
|
29476
|
+
up: () => up$2
|
|
29362
29477
|
});
|
|
29363
29478
|
/**
|
|
29364
29479
|
* Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
|
|
@@ -29382,7 +29497,7 @@ var _023_dispatch_queue_recovery_deadline_exports = /* @__PURE__ */ __exportAll(
|
|
|
29382
29497
|
* Idempotent: re-running on a DB that already has either column is a
|
|
29383
29498
|
* no-op.
|
|
29384
29499
|
*/
|
|
29385
|
-
async function up$
|
|
29500
|
+
async function up$2(db) {
|
|
29386
29501
|
const colExists = async (name) => {
|
|
29387
29502
|
return (await sql`
|
|
29388
29503
|
SELECT EXISTS (
|
|
@@ -29407,7 +29522,7 @@ async function up$1(db) {
|
|
|
29407
29522
|
WHERE recovery_deadline IS NOT NULL
|
|
29408
29523
|
`.execute(db);
|
|
29409
29524
|
}
|
|
29410
|
-
async function down$
|
|
29525
|
+
async function down$2(db) {
|
|
29411
29526
|
await sql`DROP INDEX IF EXISTS public.idx_dispatch_queue_recovery_deadline`.execute(db);
|
|
29412
29527
|
await sql`
|
|
29413
29528
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS recovery_agent_id
|
|
@@ -29420,8 +29535,8 @@ var init__023_dispatch_queue_recovery_deadline = __esmMin((() => {}));
|
|
|
29420
29535
|
//#endregion
|
|
29421
29536
|
//#region src/db/migrations/024_dispatch_queue_provisioning_error.ts
|
|
29422
29537
|
var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll({
|
|
29423
|
-
down: () => down,
|
|
29424
|
-
up: () => up
|
|
29538
|
+
down: () => down$1,
|
|
29539
|
+
up: () => up$1
|
|
29425
29540
|
});
|
|
29426
29541
|
/**
|
|
29427
29542
|
* Add `dispatch_queue.last_provisioning_error TEXT` recording the most
|
|
@@ -29437,7 +29552,7 @@ var _024_dispatch_queue_provisioning_error_exports = /* @__PURE__ */ __exportAll
|
|
|
29437
29552
|
*
|
|
29438
29553
|
* Idempotent: re-running on a DB that already has the column is a no-op.
|
|
29439
29554
|
*/
|
|
29440
|
-
async function up(db) {
|
|
29555
|
+
async function up$1(db) {
|
|
29441
29556
|
const colExists = async (name) => {
|
|
29442
29557
|
return (await sql`
|
|
29443
29558
|
SELECT EXISTS (
|
|
@@ -29453,13 +29568,60 @@ async function up(db) {
|
|
|
29453
29568
|
ADD COLUMN last_provisioning_error TEXT
|
|
29454
29569
|
`.execute(db);
|
|
29455
29570
|
}
|
|
29456
|
-
async function down(db) {
|
|
29571
|
+
async function down$1(db) {
|
|
29457
29572
|
await sql`
|
|
29458
29573
|
ALTER TABLE public.dispatch_queue DROP COLUMN IF EXISTS last_provisioning_error
|
|
29459
29574
|
`.execute(db);
|
|
29460
29575
|
}
|
|
29461
29576
|
var init__024_dispatch_queue_provisioning_error = __esmMin((() => {}));
|
|
29462
29577
|
//#endregion
|
|
29578
|
+
//#region src/db/migrations/025_init_failure.ts
|
|
29579
|
+
var _025_init_failure_exports = /* @__PURE__ */ __exportAll({
|
|
29580
|
+
down: () => down,
|
|
29581
|
+
up: () => up
|
|
29582
|
+
});
|
|
29583
|
+
/**
|
|
29584
|
+
* Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
|
|
29585
|
+
*
|
|
29586
|
+
* Presence of this column on a row means the run/job never executed a step
|
|
29587
|
+
* because of an init-phase failure; absence (NULL) means a normal run.
|
|
29588
|
+
* Shape on the wire is `InitFailure` from `@kici-dev/engine`. The dashboard
|
|
29589
|
+
* reads this column directly so it can render the right banner without
|
|
29590
|
+
* round-tripping to the orchestrator (which may be offline).
|
|
29591
|
+
*
|
|
29592
|
+
* Idempotent: re-running on a DB that already has either column is a no-op
|
|
29593
|
+
* for that column.
|
|
29594
|
+
*/
|
|
29595
|
+
async function up(db) {
|
|
29596
|
+
const colExists = async (table, name) => {
|
|
29597
|
+
return (await sql`
|
|
29598
|
+
SELECT EXISTS (
|
|
29599
|
+
SELECT 1 FROM information_schema.columns
|
|
29600
|
+
WHERE table_schema = 'public'
|
|
29601
|
+
AND table_name = ${table}
|
|
29602
|
+
AND column_name = ${name}
|
|
29603
|
+
) AS exists
|
|
29604
|
+
`.execute(db)).rows[0]?.exists ?? false;
|
|
29605
|
+
};
|
|
29606
|
+
if (!await colExists("execution_runs", "init_failure")) await sql`
|
|
29607
|
+
ALTER TABLE public.execution_runs
|
|
29608
|
+
ADD COLUMN init_failure JSONB DEFAULT NULL
|
|
29609
|
+
`.execute(db);
|
|
29610
|
+
if (!await colExists("execution_jobs", "init_failure")) await sql`
|
|
29611
|
+
ALTER TABLE public.execution_jobs
|
|
29612
|
+
ADD COLUMN init_failure JSONB DEFAULT NULL
|
|
29613
|
+
`.execute(db);
|
|
29614
|
+
}
|
|
29615
|
+
async function down(db) {
|
|
29616
|
+
await sql`
|
|
29617
|
+
ALTER TABLE public.execution_jobs DROP COLUMN IF EXISTS init_failure
|
|
29618
|
+
`.execute(db);
|
|
29619
|
+
await sql`
|
|
29620
|
+
ALTER TABLE public.execution_runs DROP COLUMN IF EXISTS init_failure
|
|
29621
|
+
`.execute(db);
|
|
29622
|
+
}
|
|
29623
|
+
var init__025_init_failure = __esmMin((() => {}));
|
|
29624
|
+
//#endregion
|
|
29463
29625
|
//#region src/db/migration-provider.ts
|
|
29464
29626
|
function createMigrationProvider() {
|
|
29465
29627
|
return { async getMigrations() {
|
|
@@ -29487,7 +29649,8 @@ function createMigrationProvider() {
|
|
|
29487
29649
|
"021_check_run_tracking": _021_check_run_tracking_exports,
|
|
29488
29650
|
"022_scaler_manager_state": _022_scaler_manager_state_exports,
|
|
29489
29651
|
"023_dispatch_queue_recovery_deadline": _023_dispatch_queue_recovery_deadline_exports,
|
|
29490
|
-
"024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports
|
|
29652
|
+
"024_dispatch_queue_provisioning_error": _024_dispatch_queue_provisioning_error_exports,
|
|
29653
|
+
"025_init_failure": _025_init_failure_exports
|
|
29491
29654
|
};
|
|
29492
29655
|
} };
|
|
29493
29656
|
}
|
|
@@ -29516,6 +29679,7 @@ var init_migration_provider = __esmMin((() => {
|
|
|
29516
29679
|
init__022_scaler_manager_state();
|
|
29517
29680
|
init__023_dispatch_queue_recovery_deadline();
|
|
29518
29681
|
init__024_dispatch_queue_provisioning_error();
|
|
29682
|
+
init__025_init_failure();
|
|
29519
29683
|
}));
|
|
29520
29684
|
//#endregion
|
|
29521
29685
|
//#region src/db/migrator.ts
|
|
@@ -34217,15 +34381,15 @@ var init_admin_config = __esmMin((() => {
|
|
|
34217
34381
|
function createHealthRoutes$1(deps = {}) {
|
|
34218
34382
|
return createHealthRoutes({
|
|
34219
34383
|
livenessInfo: () => ({
|
|
34220
|
-
version: "0.1.
|
|
34221
|
-
buildDate: "2026-05-
|
|
34222
|
-
buildCommit: "
|
|
34223
|
-
sdkVersion: "0.1.
|
|
34384
|
+
version: "0.1.14",
|
|
34385
|
+
buildDate: "2026-05-30T03:04:39.672Z",
|
|
34386
|
+
buildCommit: "0fc7a43c9",
|
|
34387
|
+
sdkVersion: "0.1.14",
|
|
34224
34388
|
sdkBundleHash: "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128",
|
|
34225
|
-
sharedVersion: "0.1.
|
|
34226
|
-
sharedBundleHash: "
|
|
34227
|
-
engineVersion: "0.1.
|
|
34228
|
-
engineBundleHash: "
|
|
34389
|
+
sharedVersion: "0.1.14",
|
|
34390
|
+
sharedBundleHash: "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4",
|
|
34391
|
+
engineVersion: "0.1.14",
|
|
34392
|
+
engineBundleHash: "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932"
|
|
34229
34393
|
}),
|
|
34230
34394
|
readinessCheck: deps.db ? async () => {
|
|
34231
34395
|
const checks = {};
|
|
@@ -34259,7 +34423,7 @@ function createCapabilitiesRoutes() {
|
|
|
34259
34423
|
const app = new Hono();
|
|
34260
34424
|
app.get("/api/v1/capabilities", (c) => {
|
|
34261
34425
|
const manifest = {
|
|
34262
|
-
orchestratorVersion: "0.1.
|
|
34426
|
+
orchestratorVersion: "0.1.14",
|
|
34263
34427
|
protocolVersion: PROTOCOL_VERSION,
|
|
34264
34428
|
minProtocolVersion: MIN_PROTOCOL_VERSION
|
|
34265
34429
|
};
|
|
@@ -34561,6 +34725,61 @@ var init_certs = __esmMin((() => {
|
|
|
34561
34725
|
WARN_DAYS = 30;
|
|
34562
34726
|
}));
|
|
34563
34727
|
//#endregion
|
|
34728
|
+
//#region src/diagnostics/checks/scaler.ts
|
|
34729
|
+
function truncate(s) {
|
|
34730
|
+
return s.length > MAX_DETAIL_LEN ? `${s.slice(0, MAX_DETAIL_LEN - 1)}…` : s;
|
|
34731
|
+
}
|
|
34732
|
+
async function checkScalerProvisioning(deps) {
|
|
34733
|
+
const start = Date.now();
|
|
34734
|
+
const mgr = deps.scalerManager;
|
|
34735
|
+
const backends = mgr?.getStatus().backends ?? [];
|
|
34736
|
+
if (!mgr || backends.length === 0) return [{
|
|
34737
|
+
name: "scaler",
|
|
34738
|
+
status: "pass",
|
|
34739
|
+
message: "No scaler backends configured",
|
|
34740
|
+
durationMs: Date.now() - start
|
|
34741
|
+
}];
|
|
34742
|
+
const recent = mgr.recentSpawnFailures(SCALER_FAILURE_WINDOW_MS, Date.now());
|
|
34743
|
+
return backends.map((backend) => {
|
|
34744
|
+
const summary = recent.get(backend.name);
|
|
34745
|
+
const durationMs = Date.now() - start;
|
|
34746
|
+
if (!summary || summary.boundCount + summary.unboundCount === 0) return {
|
|
34747
|
+
name: `scaler:${backend.name}`,
|
|
34748
|
+
status: "pass",
|
|
34749
|
+
message: "0 spawn failures in last 5m",
|
|
34750
|
+
details: {
|
|
34751
|
+
windowMs: SCALER_FAILURE_WINDOW_MS,
|
|
34752
|
+
backendType: backend.type,
|
|
34753
|
+
boundCount: 0,
|
|
34754
|
+
unboundCount: 0
|
|
34755
|
+
},
|
|
34756
|
+
durationMs
|
|
34757
|
+
};
|
|
34758
|
+
const total = summary.boundCount + summary.unboundCount;
|
|
34759
|
+
const status = summary.boundCount > 0 ? "fail" : "warn";
|
|
34760
|
+
const message = `${total} spawn failures in last 5m (${summary.boundCount} bound, ${summary.unboundCount} warm-pool; last: ${truncate(summary.lastError)})`;
|
|
34761
|
+
return {
|
|
34762
|
+
name: `scaler:${backend.name}`,
|
|
34763
|
+
status,
|
|
34764
|
+
message,
|
|
34765
|
+
details: {
|
|
34766
|
+
windowMs: SCALER_FAILURE_WINDOW_MS,
|
|
34767
|
+
backendType: summary.backendType,
|
|
34768
|
+
boundCount: summary.boundCount,
|
|
34769
|
+
unboundCount: summary.unboundCount,
|
|
34770
|
+
lastError: summary.lastError,
|
|
34771
|
+
lastAtMs: summary.lastAtMs
|
|
34772
|
+
},
|
|
34773
|
+
durationMs
|
|
34774
|
+
};
|
|
34775
|
+
});
|
|
34776
|
+
}
|
|
34777
|
+
var SCALER_FAILURE_WINDOW_MS, MAX_DETAIL_LEN;
|
|
34778
|
+
var init_scaler$1 = __esmMin((() => {
|
|
34779
|
+
SCALER_FAILURE_WINDOW_MS = 300 * 1e3;
|
|
34780
|
+
MAX_DETAIL_LEN = 120;
|
|
34781
|
+
}));
|
|
34782
|
+
//#endregion
|
|
34564
34783
|
//#region src/diagnostics/checks/index.ts
|
|
34565
34784
|
var defaultChecks;
|
|
34566
34785
|
var init_checks = __esmMin((() => {
|
|
@@ -34570,13 +34789,15 @@ var init_checks = __esmMin((() => {
|
|
|
34570
34789
|
init_disk();
|
|
34571
34790
|
init_config$1();
|
|
34572
34791
|
init_certs();
|
|
34792
|
+
init_scaler$1();
|
|
34573
34793
|
defaultChecks = [
|
|
34574
34794
|
checkDbConnectivity,
|
|
34575
34795
|
checkWsToPlatform,
|
|
34576
34796
|
checkAgentConnectivity,
|
|
34577
34797
|
checkDiskSpace,
|
|
34578
34798
|
checkConfigValidity,
|
|
34579
|
-
checkCertificateExpiry
|
|
34799
|
+
checkCertificateExpiry,
|
|
34800
|
+
checkScalerProvisioning
|
|
34580
34801
|
];
|
|
34581
34802
|
}));
|
|
34582
34803
|
//#endregion
|
|
@@ -34592,14 +34813,14 @@ var init_checks = __esmMin((() => {
|
|
|
34592
34813
|
async function runDiagnostics(deps, checks, options) {
|
|
34593
34814
|
const checksToRun = checks ?? defaultChecks;
|
|
34594
34815
|
const timeoutMs = options?.timeoutMs ?? 5e3;
|
|
34595
|
-
return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).
|
|
34596
|
-
if (result.status === "fulfilled") return result.value;
|
|
34597
|
-
return {
|
|
34816
|
+
return (await Promise.allSettled(checksToRun.map((check) => runWithTimeout(check, deps, timeoutMs)))).flatMap((result, index) => {
|
|
34817
|
+
if (result.status === "fulfilled") return Array.isArray(result.value) ? result.value : [result.value];
|
|
34818
|
+
return [{
|
|
34598
34819
|
name: `Check ${index + 1}`,
|
|
34599
34820
|
status: "fail",
|
|
34600
34821
|
message: `Unexpected error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
|
|
34601
34822
|
durationMs: 0
|
|
34602
|
-
};
|
|
34823
|
+
}];
|
|
34603
34824
|
});
|
|
34604
34825
|
}
|
|
34605
34826
|
/**
|
|
@@ -35696,7 +35917,8 @@ function createApp(deps) {
|
|
|
35696
35917
|
platformUrl: deps.config.platformUrl,
|
|
35697
35918
|
agentRegistry: deps.registry,
|
|
35698
35919
|
config: deps.config,
|
|
35699
|
-
tlsCertPath: deps.config.tlsCertPath
|
|
35920
|
+
tlsCertPath: deps.config.tlsCertPath,
|
|
35921
|
+
scalerManager: deps.scalerManager
|
|
35700
35922
|
}));
|
|
35701
35923
|
{
|
|
35702
35924
|
const peerHandlerRef = deps.peerHandler;
|
|
@@ -38856,6 +39078,55 @@ var init_machine_ledger = __esmMin((() => {
|
|
|
38856
39078
|
};
|
|
38857
39079
|
}));
|
|
38858
39080
|
//#endregion
|
|
39081
|
+
//#region src/scaler/failure-tracker.ts
|
|
39082
|
+
var DEFAULT_MAX_ENTRIES, ScalerFailureTracker;
|
|
39083
|
+
var init_failure_tracker = __esmMin((() => {
|
|
39084
|
+
DEFAULT_MAX_ENTRIES = 256;
|
|
39085
|
+
ScalerFailureTracker = class {
|
|
39086
|
+
records = [];
|
|
39087
|
+
maxEntries;
|
|
39088
|
+
constructor(maxEntries = DEFAULT_MAX_ENTRIES) {
|
|
39089
|
+
this.maxEntries = maxEntries;
|
|
39090
|
+
}
|
|
39091
|
+
/** Record a failure, evicting the oldest entry when over capacity. */
|
|
39092
|
+
record(rec) {
|
|
39093
|
+
this.records.push(rec);
|
|
39094
|
+
if (this.records.length > this.maxEntries) this.records.shift();
|
|
39095
|
+
}
|
|
39096
|
+
/**
|
|
39097
|
+
* Group failures newer than `nowMs - windowMs` by backend instance name.
|
|
39098
|
+
* `nowMs` is injected so callers control the clock (and tests are
|
|
39099
|
+
* deterministic).
|
|
39100
|
+
*/
|
|
39101
|
+
recentByBackend(windowMs, nowMs) {
|
|
39102
|
+
const cutoff = nowMs - windowMs;
|
|
39103
|
+
const out = /* @__PURE__ */ new Map();
|
|
39104
|
+
for (const rec of this.records) {
|
|
39105
|
+
if (rec.timestampMs < cutoff) continue;
|
|
39106
|
+
const existing = out.get(rec.backendName);
|
|
39107
|
+
if (!existing) {
|
|
39108
|
+
out.set(rec.backendName, {
|
|
39109
|
+
backendType: rec.backendType,
|
|
39110
|
+
boundCount: rec.bound ? 1 : 0,
|
|
39111
|
+
unboundCount: rec.bound ? 0 : 1,
|
|
39112
|
+
lastError: rec.detail,
|
|
39113
|
+
lastAtMs: rec.timestampMs
|
|
39114
|
+
});
|
|
39115
|
+
continue;
|
|
39116
|
+
}
|
|
39117
|
+
if (rec.bound) existing.boundCount += 1;
|
|
39118
|
+
else existing.unboundCount += 1;
|
|
39119
|
+
if (rec.timestampMs >= existing.lastAtMs) {
|
|
39120
|
+
existing.lastError = rec.detail;
|
|
39121
|
+
existing.lastAtMs = rec.timestampMs;
|
|
39122
|
+
existing.backendType = rec.backendType;
|
|
39123
|
+
}
|
|
39124
|
+
}
|
|
39125
|
+
return out;
|
|
39126
|
+
}
|
|
39127
|
+
};
|
|
39128
|
+
}));
|
|
39129
|
+
//#endregion
|
|
38859
39130
|
//#region src/scaler/manager.ts
|
|
38860
39131
|
/**
|
|
38861
39132
|
* ScalerManager: central coordinator for all scaler backends.
|
|
@@ -38919,10 +39190,13 @@ var init_manager = __esmMin((() => {
|
|
|
38919
39190
|
init_machine_ledger();
|
|
38920
39191
|
init_prometheus();
|
|
38921
39192
|
init_types$1();
|
|
39193
|
+
init_failure_tracker();
|
|
38922
39194
|
logger$25 = createLogger({ prefix: "scaler" });
|
|
38923
39195
|
ScalerManager = class {
|
|
38924
39196
|
backends = /* @__PURE__ */ new Map();
|
|
38925
39197
|
backendRoles = /* @__PURE__ */ new Map();
|
|
39198
|
+
/** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
|
|
39199
|
+
failureTracker = new ScalerFailureTracker();
|
|
38926
39200
|
globalMaxAgents;
|
|
38927
39201
|
/** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
|
|
38928
39202
|
resourceCaps = /* @__PURE__ */ new Map();
|
|
@@ -39564,6 +39838,13 @@ var init_manager = __esmMin((() => {
|
|
|
39564
39838
|
/**
|
|
39565
39839
|
* Return status summary for metrics and health endpoints.
|
|
39566
39840
|
*/
|
|
39841
|
+
/**
|
|
39842
|
+
* Recent scaler spawn failures grouped per backend instance, for the
|
|
39843
|
+
* diagnose scaler check. `nowMs` is injected by the caller.
|
|
39844
|
+
*/
|
|
39845
|
+
recentSpawnFailures(windowMs, nowMs) {
|
|
39846
|
+
return this.failureTracker.recentByBackend(windowMs, nowMs);
|
|
39847
|
+
}
|
|
39567
39848
|
getStatus() {
|
|
39568
39849
|
const backendStatuses = [];
|
|
39569
39850
|
const enrichedEntries = this.getEnrichedScalerEntries();
|
|
@@ -39850,6 +40131,13 @@ var init_manager = __esmMin((() => {
|
|
|
39850
40131
|
jobId: jobId ?? null,
|
|
39851
40132
|
detail: event.detail
|
|
39852
40133
|
});
|
|
40134
|
+
this.failureTracker.record({
|
|
40135
|
+
backendName: backendName ?? "unknown",
|
|
40136
|
+
backendType: backend,
|
|
40137
|
+
bound: Boolean(jobId),
|
|
40138
|
+
detail: event.detail ?? "",
|
|
40139
|
+
timestampMs: event.timestampMs
|
|
40140
|
+
});
|
|
39853
40141
|
}
|
|
39854
40142
|
if (runId && jobId && this.onScalerEvent) this.onScalerEvent(runId, jobId, event);
|
|
39855
40143
|
else if (this.onScalerEvent) {
|
|
@@ -41605,6 +41893,22 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
41605
41893
|
orgId;
|
|
41606
41894
|
jobQueue;
|
|
41607
41895
|
runs = /* @__PURE__ */ new Map();
|
|
41896
|
+
/**
|
|
41897
|
+
* Per-run async-mutex chain. `onJobStatus` and `addJobsToRun` mutate the same
|
|
41898
|
+
* `run.jobs` Map / `execution_jobs` row; without serialization a job-status
|
|
41899
|
+
* reply that lands mid synthetic→real swap (`addJobsToRun`) clobbers the swap
|
|
41900
|
+
* and the run hangs in `running` forever (see `withRunLock`). The map value is
|
|
41901
|
+
* the tail of the chain for that runId; entries are GC'd when the last holder
|
|
41902
|
+
* releases.
|
|
41903
|
+
*/
|
|
41904
|
+
runLockTails = /* @__PURE__ */ new Map();
|
|
41905
|
+
/**
|
|
41906
|
+
* Tracks which runIds the current async context already holds in
|
|
41907
|
+
* `withRunLock`, so reentrant calls (e.g. onJobStatus → scheduler hook →
|
|
41908
|
+
* dispatchReadyJob → addJobsToRun, all the same runId) bypass re-acquisition
|
|
41909
|
+
* instead of deadlocking. Propagates across awaits via AsyncLocalStorage.
|
|
41910
|
+
*/
|
|
41911
|
+
heldRunLocks = new AsyncLocalStorage();
|
|
41608
41912
|
/** Tracks which runs are test runs for observer broadcasting. */
|
|
41609
41913
|
testRunIds = /* @__PURE__ */ new Set();
|
|
41610
41914
|
/**
|
|
@@ -41759,7 +42063,44 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
41759
42063
|
}
|
|
41760
42064
|
return (await this.db.selectFrom("execution_jobs").select("job_id").where("run_id", "=", runId).where("job_name", "=", jobName).where("job_id", "like", `${prefix}%`).executeTakeFirst())?.job_id;
|
|
41761
42065
|
}
|
|
42066
|
+
/**
|
|
42067
|
+
* Run `fn` while holding a per-run lock, serializing the run-mutating methods
|
|
42068
|
+
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|
|
42069
|
+
* synthetic→real job swap and wedge the run in `running`.
|
|
42070
|
+
*
|
|
42071
|
+
* The lock is **reentrant**: three paths re-enter a locked method within the
|
|
42072
|
+
* same async context for the same runId — onJobStatus → scheduler hook →
|
|
42073
|
+
* dispatchReadyJob → addJobsToRun; onJobStatus → enforceSchedulerInvariant →
|
|
42074
|
+
* onJobStatus; runSchedulerHook → onJobStatus (skip). A non-reentrant mutex
|
|
42075
|
+
* would deadlock on these, so a context that already holds the runId's lock
|
|
42076
|
+
* (tracked via `heldRunLocks`) runs `fn` inline. A genuinely concurrent caller
|
|
42077
|
+
* for the same runId (a separate WS message) is a different async context and
|
|
42078
|
+
* correctly waits. All reentrant paths are same-runId, so there is no
|
|
42079
|
+
* cross-run lock-ordering deadlock.
|
|
42080
|
+
*/
|
|
42081
|
+
async withRunLock(runId, fn) {
|
|
42082
|
+
if (this.heldRunLocks.getStore()?.has(runId)) return fn();
|
|
42083
|
+
const prev = this.runLockTails.get(runId) ?? Promise.resolve();
|
|
42084
|
+
let release;
|
|
42085
|
+
const mine = new Promise((resolve) => {
|
|
42086
|
+
release = resolve;
|
|
42087
|
+
});
|
|
42088
|
+
const chained = prev.then(() => mine);
|
|
42089
|
+
this.runLockTails.set(runId, chained);
|
|
42090
|
+
await prev.catch(() => {});
|
|
42091
|
+
const nextHeld = new Set(this.heldRunLocks.getStore() ?? []);
|
|
42092
|
+
nextHeld.add(runId);
|
|
42093
|
+
try {
|
|
42094
|
+
return await this.heldRunLocks.run(nextHeld, fn);
|
|
42095
|
+
} finally {
|
|
42096
|
+
release();
|
|
42097
|
+
if (this.runLockTails.get(runId) === chained) this.runLockTails.delete(runId);
|
|
42098
|
+
}
|
|
42099
|
+
}
|
|
41762
42100
|
async addJobsToRun(runId, jobs, dispatchedContexts, replaceSyntheticId) {
|
|
42101
|
+
return this.withRunLock(runId, () => this.addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId));
|
|
42102
|
+
}
|
|
42103
|
+
async addJobsToRunImpl(runId, jobs, dispatchedContexts, replaceSyntheticId) {
|
|
41763
42104
|
const run = this.runs.get(runId);
|
|
41764
42105
|
if (!run) {
|
|
41765
42106
|
logger$21.warn("addJobsToRun: run not found in memory", { runId });
|
|
@@ -41854,6 +42195,9 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
41854
42195
|
* fires the onExecutionComplete callback.
|
|
41855
42196
|
*/
|
|
41856
42197
|
async onJobStatus(runId, jobId, state, timestamp, agentId, data) {
|
|
42198
|
+
return this.withRunLock(runId, () => this.onJobStatusImpl(runId, jobId, state, timestamp, agentId, data));
|
|
42199
|
+
}
|
|
42200
|
+
async onJobStatusImpl(runId, jobId, state, timestamp, agentId, data) {
|
|
41857
42201
|
let run = this.runs.get(runId);
|
|
41858
42202
|
let job = run?.jobs.get(jobId);
|
|
41859
42203
|
if (!run) {
|
|
@@ -41974,6 +42318,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
41974
42318
|
* run's in-memory map. Returns the freshly-inserted job entry.
|
|
41975
42319
|
*/
|
|
41976
42320
|
async recoverJobFromDispatchQueue(run, runId, jobId) {
|
|
42321
|
+
const existing = run.jobs.get(jobId);
|
|
42322
|
+
if (existing) return existing;
|
|
41977
42323
|
const jobName = (await this.db.selectFrom("dispatch_queue").select(["job_name"]).where("id", "=", jobId).executeTakeFirst())?.job_name ?? jobId;
|
|
41978
42324
|
run.jobs.set(jobId, {
|
|
41979
42325
|
name: jobName,
|
|
@@ -42008,6 +42354,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42008
42354
|
updateValues.duration_ms = timestamp - job.startedAt;
|
|
42009
42355
|
}
|
|
42010
42356
|
if (data?.error) updateValues.error_message = String(data.error);
|
|
42357
|
+
if (data?.initFailure) updateValues.init_failure = JSON.stringify(data.initFailure);
|
|
42011
42358
|
if (data?.outputs) updateValues.outputs = JSON.stringify(data.outputs);
|
|
42012
42359
|
jobLogBytesTotal = this.jobLogBytes.get(runId)?.get(jobId) ?? 0;
|
|
42013
42360
|
updateValues.log_bytes = jobLogBytesTotal;
|
|
@@ -42126,7 +42473,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42126
42473
|
const completedAtMs = isTerminal ? timestamp : void 0;
|
|
42127
42474
|
const duration = isTerminal && job.startedAt ? timestamp - job.startedAt : void 0;
|
|
42128
42475
|
const errorMsg = isTerminal && data?.error ? String(data.error) : void 0;
|
|
42129
|
-
|
|
42476
|
+
const initFailure = isTerminal && data?.initFailure ? data.initFailure : void 0;
|
|
42477
|
+
this.onJobStatusChange?.(runId, jobId, job.name, state, timestamp, startedAtMs, completedAtMs, duration, errorMsg, job.agentId, job.runsOnLabels, jobLogBytesTotal, initFailure);
|
|
42130
42478
|
}
|
|
42131
42479
|
/**
|
|
42132
42480
|
* Phase 6: reconcile orphaned steps. If a job reaches terminal state but
|
|
@@ -42323,13 +42671,14 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42323
42671
|
* onExecutionStarted but the build subsequently fails. Without this,
|
|
42324
42672
|
* the execution_runs row would stay in a non-terminal state.
|
|
42325
42673
|
*/
|
|
42326
|
-
async onBuildFailed(runId) {
|
|
42674
|
+
async onBuildFailed(runId, initFailure) {
|
|
42327
42675
|
const now = /* @__PURE__ */ new Date();
|
|
42328
|
-
const failureReason = "Build job failed";
|
|
42676
|
+
const failureReason = initFailure?.message ?? "Build job failed";
|
|
42329
42677
|
await this.db.updateTable("execution_runs").set({
|
|
42330
42678
|
status: ExecutionRunStatus.enum.failed,
|
|
42331
42679
|
completed_at: now,
|
|
42332
|
-
failure_reason: failureReason
|
|
42680
|
+
failure_reason: failureReason,
|
|
42681
|
+
...initFailure && { init_failure: JSON.stringify(initFailure) }
|
|
42333
42682
|
}).where("run_id", "=", runId).execute();
|
|
42334
42683
|
await this.db.updateTable("execution_jobs").set({
|
|
42335
42684
|
status: ExecutionJobStatus.enum.failed,
|
|
@@ -42352,7 +42701,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42352
42701
|
parentRunId: run.parentRunId,
|
|
42353
42702
|
originalRunId: run.originalRunId,
|
|
42354
42703
|
triggeredBy: run.triggeredBy
|
|
42355
|
-
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason);
|
|
42704
|
+
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, failureReason, void 0, initFailure);
|
|
42356
42705
|
this.runs.delete(runId);
|
|
42357
42706
|
this.jobLogBytes.delete(runId);
|
|
42358
42707
|
}
|
|
@@ -42365,7 +42714,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42365
42714
|
* Inserts a minimal execution_runs row with status='failed' directly so the E2E
|
|
42366
42715
|
* test (and dashboard) can observe the failure instead of a missing run.
|
|
42367
42716
|
*/
|
|
42368
|
-
async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason) {
|
|
42717
|
+
async onBuildFailedBeforeTracking(runId, workflowName, provider, repoIdentifier, ref, sha, deliveryId, providerContext, routingKey, triggerEvent, commitMessage, failureReason, initFailure) {
|
|
42369
42718
|
const now = /* @__PURE__ */ new Date();
|
|
42370
42719
|
const reason = failureReason ?? "Build job timed out before execution tracking started";
|
|
42371
42720
|
await this.db.insertInto("execution_runs").values({
|
|
@@ -42382,7 +42731,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42382
42731
|
started_at: now,
|
|
42383
42732
|
completed_at: now,
|
|
42384
42733
|
status: ExecutionRunStatus.enum.failed,
|
|
42385
|
-
failure_reason: reason
|
|
42734
|
+
failure_reason: reason,
|
|
42735
|
+
...initFailure && { init_failure: JSON.stringify(initFailure) }
|
|
42386
42736
|
}).execute();
|
|
42387
42737
|
logger$21.info("Created failed execution run (build failed before tracking)", {
|
|
42388
42738
|
runId,
|
|
@@ -42391,6 +42741,55 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42391
42741
|
});
|
|
42392
42742
|
}
|
|
42393
42743
|
/**
|
|
42744
|
+
* Insert a `failed` execution_runs row directly for an init failure that
|
|
42745
|
+
* occurred BEFORE onExecutionStarted ran (so no in-memory state exists
|
|
42746
|
+
* and no jobs were dispatched). Also writes the structured init_failure
|
|
42747
|
+
* signal and fires onExecutionStatusChange so Platform's projection picks
|
|
42748
|
+
* it up via the normal forward path. Idempotent: if a row already exists
|
|
42749
|
+
* for this runId, the insert is a no-op (ON CONFLICT DO NOTHING).
|
|
42750
|
+
*
|
|
42751
|
+
* Closes the silent pre-run-failure gap — without this helper, secret /
|
|
42752
|
+
* install-secret / all-jobs-rejected early-exits in dispatch-matched-workflow
|
|
42753
|
+
* leave no trace on the dashboard.
|
|
42754
|
+
*/
|
|
42755
|
+
async recordInitFailureRun(args) {
|
|
42756
|
+
const now = /* @__PURE__ */ new Date();
|
|
42757
|
+
await this.db.insertInto("execution_runs").values({
|
|
42758
|
+
run_id: args.runId,
|
|
42759
|
+
routing_key: args.routingKey,
|
|
42760
|
+
workflow_name: args.workflowName,
|
|
42761
|
+
provider: args.provider,
|
|
42762
|
+
repo_identifier: args.repoIdentifier,
|
|
42763
|
+
ref: args.ref,
|
|
42764
|
+
sha: args.sha,
|
|
42765
|
+
delivery_id: args.deliveryId,
|
|
42766
|
+
trigger_decision: null,
|
|
42767
|
+
provider_context: JSON.stringify(args.providerContext),
|
|
42768
|
+
started_at: now,
|
|
42769
|
+
completed_at: now,
|
|
42770
|
+
status: ExecutionRunStatus.enum.failed,
|
|
42771
|
+
failure_reason: args.initFailure.message,
|
|
42772
|
+
init_failure: JSON.stringify(args.initFailure)
|
|
42773
|
+
}).onConflict((oc) => oc.column("run_id").doNothing()).execute();
|
|
42774
|
+
executionsTotal.add(1, { status: ExecutionRunStatus.enum.failed });
|
|
42775
|
+
this.onExecutionStatusChange?.(args.runId, ExecutionRunStatus.enum.failed, {
|
|
42776
|
+
workflowName: args.workflowName,
|
|
42777
|
+
provider: args.provider,
|
|
42778
|
+
repoIdentifier: args.repoIdentifier,
|
|
42779
|
+
sha: args.sha,
|
|
42780
|
+
routingKey: args.routingKey,
|
|
42781
|
+
ref: args.ref,
|
|
42782
|
+
triggerEvent: args.triggerEvent,
|
|
42783
|
+
commitMessage: args.commitMessage
|
|
42784
|
+
}, 0, now.getTime(), now.getTime(), 0, args.initFailure.message, 0, args.initFailure);
|
|
42785
|
+
logger$21.info("Recorded init-failure execution run", {
|
|
42786
|
+
runId: args.runId,
|
|
42787
|
+
workflowName: args.workflowName,
|
|
42788
|
+
category: args.initFailure.category,
|
|
42789
|
+
scope: args.initFailure.scope
|
|
42790
|
+
});
|
|
42791
|
+
}
|
|
42792
|
+
/**
|
|
42394
42793
|
* Mark a run as failed immediately with a reason message.
|
|
42395
42794
|
*
|
|
42396
42795
|
* Used when no agents are available to dispatch any jobs (e.g. cron-triggered
|
|
@@ -42398,7 +42797,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42398
42797
|
* Instead of leaving the run in 'running' for OrphanRecovery to catch after 5 min,
|
|
42399
42798
|
* this fails it right away.
|
|
42400
42799
|
*/
|
|
42401
|
-
async failRun(runId, reason) {
|
|
42800
|
+
async failRun(runId, reason, initFailure) {
|
|
42402
42801
|
const now = /* @__PURE__ */ new Date();
|
|
42403
42802
|
const run = this.runs.get(runId);
|
|
42404
42803
|
const durationMs = run ? now.getTime() - run.startedAt : 0;
|
|
@@ -42406,7 +42805,8 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42406
42805
|
status: ExecutionRunStatus.enum.failed,
|
|
42407
42806
|
completed_at: now,
|
|
42408
42807
|
duration_ms: durationMs,
|
|
42409
|
-
failure_reason: reason
|
|
42808
|
+
failure_reason: reason,
|
|
42809
|
+
...initFailure && { init_failure: JSON.stringify(initFailure) }
|
|
42410
42810
|
}).where("run_id", "=", runId).execute();
|
|
42411
42811
|
await this.db.updateTable("execution_jobs").set({
|
|
42412
42812
|
status: ExecutionJobStatus.enum.failed,
|
|
@@ -42430,7 +42830,7 @@ var init_execution_tracker = __esmMin((() => {
|
|
|
42430
42830
|
parentRunId: run.parentRunId,
|
|
42431
42831
|
originalRunId: run.originalRunId,
|
|
42432
42832
|
triggeredBy: run.triggeredBy
|
|
42433
|
-
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason);
|
|
42833
|
+
}, run.jobs.size, run.startedAt, completedAt, completedAt - run.startedAt, reason, void 0, initFailure);
|
|
42434
42834
|
this.runs.delete(runId);
|
|
42435
42835
|
this.jobLogBytes.delete(runId);
|
|
42436
42836
|
}
|
|
@@ -50549,14 +50949,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
50549
50949
|
init_agent_handler();
|
|
50550
50950
|
init_worker_status();
|
|
50551
50951
|
init_agent_heartbeat();
|
|
50552
|
-
ORCHESTRATOR_VERSION$1 = "0.1.
|
|
50553
|
-
WORKER_BUILD_COMMIT = "
|
|
50554
|
-
WORKER_SDK_VERSION = "0.1.
|
|
50952
|
+
ORCHESTRATOR_VERSION$1 = "0.1.14";
|
|
50953
|
+
WORKER_BUILD_COMMIT = "0fc7a43c9";
|
|
50954
|
+
WORKER_SDK_VERSION = "0.1.14";
|
|
50555
50955
|
WORKER_SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
|
|
50556
|
-
WORKER_SHARED_VERSION = "0.1.
|
|
50557
|
-
WORKER_SHARED_BUNDLE_HASH = "
|
|
50558
|
-
WORKER_ENGINE_VERSION = "0.1.
|
|
50559
|
-
WORKER_ENGINE_BUNDLE_HASH = "
|
|
50956
|
+
WORKER_SHARED_VERSION = "0.1.14";
|
|
50957
|
+
WORKER_SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
|
|
50958
|
+
WORKER_ENGINE_VERSION = "0.1.14";
|
|
50959
|
+
WORKER_ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
|
|
50560
50960
|
logger$2 = createLogger({ prefix: "worker" });
|
|
50561
50961
|
DRAIN_TIMEOUT_MS = 3e5;
|
|
50562
50962
|
}));
|
|
@@ -50577,14 +50977,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
50577
50977
|
* Graceful shutdown in reverse order:
|
|
50578
50978
|
* Platform client -> agent WS -> heartbeat -> HTTP -> DB
|
|
50579
50979
|
*/
|
|
50580
|
-
const ORCHESTRATOR_VERSION = "0.1.
|
|
50581
|
-
const BUILD_COMMIT = "
|
|
50582
|
-
const SDK_VERSION = "0.1.
|
|
50980
|
+
const ORCHESTRATOR_VERSION = "0.1.14";
|
|
50981
|
+
const BUILD_COMMIT = "0fc7a43c9";
|
|
50982
|
+
const SDK_VERSION = "0.1.14";
|
|
50583
50983
|
const SDK_BUNDLE_HASH = "675aafe4de03e677785ef2794d8f34951b14e2bd1e6f0ea825953d6c2abfe128";
|
|
50584
|
-
const SHARED_VERSION = "0.1.
|
|
50585
|
-
const SHARED_BUNDLE_HASH = "
|
|
50586
|
-
const ENGINE_VERSION = "0.1.
|
|
50587
|
-
const ENGINE_BUNDLE_HASH = "
|
|
50984
|
+
const SHARED_VERSION = "0.1.14";
|
|
50985
|
+
const SHARED_BUNDLE_HASH = "5a44862f00d382dbac676c0bdbf9e3fb3016d1c4667b60ea4ae6cbf8ac8deaf4";
|
|
50986
|
+
const ENGINE_VERSION = "0.1.14";
|
|
50987
|
+
const ENGINE_BUNDLE_HASH = "8eef210908744b12e19a2b915520298c5dde5abd4a064fa75e87123e237bf932";
|
|
50588
50988
|
const otelSdk = initTelemetry({
|
|
50589
50989
|
serviceName: "kici-orchestrator",
|
|
50590
50990
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
@@ -50662,7 +51062,7 @@ await guardStartup(logger$1, async () => {
|
|
|
50662
51062
|
await bootstrapOrchestrator(config, {
|
|
50663
51063
|
logPrefix: "server",
|
|
50664
51064
|
executionTrackerExtras: () => ({
|
|
50665
|
-
onExecutionStatusChange: (runId, status, context, jobCount, startedAt, completedAt, durationMs, failureReason, logBytes) => {
|
|
51065
|
+
onExecutionStatusChange: (runId, status, context, jobCount, startedAt, completedAt, durationMs, failureReason, logBytes, initFailure) => {
|
|
50666
51066
|
try {
|
|
50667
51067
|
platformClient.send({
|
|
50668
51068
|
type: "execution.status",
|
|
@@ -50684,7 +51084,8 @@ await guardStartup(logger$1, async () => {
|
|
|
50684
51084
|
...completedAt !== void 0 && { completedAt },
|
|
50685
51085
|
...durationMs !== void 0 && { durationMs },
|
|
50686
51086
|
...failureReason !== void 0 && { failureReason },
|
|
50687
|
-
...logBytes !== void 0 && { logBytes }
|
|
51087
|
+
...logBytes !== void 0 && { logBytes },
|
|
51088
|
+
...initFailure && { initFailure }
|
|
50688
51089
|
});
|
|
50689
51090
|
} catch (err) {
|
|
50690
51091
|
logger$1.warn("Failed to forward execution status to Platform", {
|
|
@@ -50694,7 +51095,7 @@ await guardStartup(logger$1, async () => {
|
|
|
50694
51095
|
});
|
|
50695
51096
|
}
|
|
50696
51097
|
},
|
|
50697
|
-
onJobStatusChange: (runId, jobId, jobName, status, timestamp, startedAt, completedAt, durationMs, errorMessage, agentId, runsOnLabels, logBytes) => {
|
|
51098
|
+
onJobStatusChange: (runId, jobId, jobName, status, timestamp, startedAt, completedAt, durationMs, errorMessage, agentId, runsOnLabels, logBytes, initFailure) => {
|
|
50698
51099
|
try {
|
|
50699
51100
|
platformClient.send({
|
|
50700
51101
|
type: "job.status.forward",
|
|
@@ -50711,6 +51112,7 @@ await guardStartup(logger$1, async () => {
|
|
|
50711
51112
|
...logBytes !== void 0 && { logBytes },
|
|
50712
51113
|
...agentId && { agentId },
|
|
50713
51114
|
...runsOnLabels?.length && { runsOnLabels },
|
|
51115
|
+
...initFailure && { initFailure },
|
|
50714
51116
|
orchestratorId: config.instanceId
|
|
50715
51117
|
});
|
|
50716
51118
|
} catch (err) {
|