@kici-dev/orchestrator 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/registry.d.ts +22 -3
- package/dist/cli/commands/shared/container-agent-image-check.d.ts +67 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +7 -7
- package/dist/cli/service/instance/manifest.d.ts +30 -4
- package/dist/cli.js +279 -54
- package/dist/dashboard/handler.d.ts +2 -0
- package/dist/pipeline/dispatch-matched-workflow.d.ts +1 -1
- package/dist/pipeline/global-eval-round.d.ts +0 -8
- package/dist/scaler/bare-metal-backend.d.ts +2 -1
- package/dist/scaler/container-hostconfig.d.ts +9 -0
- package/dist/scaler/warm-pool.d.ts +4 -1
- package/dist/server.js +1165 -974
- package/dist/standalone.js +566 -479
- package/dist/ws/execution-status-frame.d.ts +2 -1
- package/dist/ws/job-status-forward-frame.d.ts +30 -0
- package/dist/ws/outbound-frame-guard.d.ts +4 -0
- package/dist/ws/platform-client.d.ts +11 -1
- package/dist/ws/test-relay-handlers.d.ts +15 -5
- package/dist/ws/test-run-log-cursor.d.ts +18 -0
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +53 -53
package/dist/standalone.js
CHANGED
|
@@ -5,7 +5,7 @@ __cjs_dirname(__filename);
|
|
|
5
5
|
import "node:module";
|
|
6
6
|
import "@kici-dev/shared/load-service-env-file";
|
|
7
7
|
import { AgentDeliveryMode, AgentPlatform, BaseColdStore, ChunkLru, ChunkRequestWaiter, CommandError, PgTableAdapterBase, addLogsToArchive, applyProxyKeepAliveTimeouts, chunkBuffer, closeDatabase, computeBackoffDelay, computeMigrationsHash, createDb, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, createPool, createS3Client, decrypt, decryptJson, depCacheKeyOf, deriveKey, encodeKeySegment, encrypt, enrichRequestContext, formatDuration, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, isPgUniqueViolation, logger, parseDatabaseUrl, redactConfig, requestContext, retryOnBrokenConnection, serializeError, setServiceName, setupGracefulShutdown, shQuote, sha256, stderrTail, stopHttpServer, storeMigrationContentHash, toCommandError, toErrorMessage, trackOpenSockets, validateRequiredTools } from "@kici-dev/shared";
|
|
8
|
-
import { ACTOR_AGENT_SUFFIX, AGENT_API_REQUEST_TIMEOUT_MS, AGENT_FEATURE_LABELS, ALLOWED_SYSTEM_VARS, ARTIFACT_NAME_MAX_LENGTH, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, AgentFailureCategory, ApprovalDecision, ArtifactCompleteAckOutcome, ArtifactDownloadOutcome, ArtifactUploadOutcome, AttestationOrigin, BREAKING_FLOOR, CANONICAL_STATUSES, CONTAINER_BUILD_RUNTIME_LABEL, CacheRefScope, CheckMode, CheckRunConclusion, CheckStepOutcome, CiTrustLevel, ConcurrencyStrategy, ContextDeleteErrorCode, ContextGateRejectReason, ContextType, DASHBOARD_REQUEST_TYPE_SET, DEFAULT_APPROVAL_EXPIRY_SECONDS, DEFAULT_CONCURRENCY_STRATEGY, DEFAULT_HOLD_EXPIRY_SECONDS, DEFERRED_PATHS_SUMMARY_SUFFIX, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, FanoutCause, FanoutError, FleetHostWriteRefusal, ForkPolicy, GLOBAL_EVAL_SKIPS_RESULT_AWARE_LABEL, HeldRunQueueType, HeldRunStatus, HoldScope, HoldType, ID_TOKEN_CLAIM_NAMES, INIT_LABEL, INIT_RUNNER_ROLE_LABEL, INSTALL_JOB_ID_PREFIX, INVALID_JOIN_REQUEST_MESSAGE, InitFailureCategory, InventorySelectorSchema, JOB_IMAGE_RUNTIME_LABEL, JOIN_PROTOCOL_UPGRADE_TOKEN, JOIN_PROTOCOL_V1_REMOVED_MESSAGE, JobRejectReason, JoinErrorCode, JoinRequestKind, KICI_AGENT_ENV_PREFIX, KICI_EVENT_NAME_PREFIX, KNOWN_ROLES, LabelMatcher, LockFileParseError, LogStream, MAX_APPROVAL_EXPIRY_SECONDS, MIN_APPROVAL_EXPIRY_SECONDS, MIN_PROTOCOL_VERSION, MinimumTrustSchema, ORCH_AGENT_CAPABILITIES, OWN_INGRESS_MODES, OrchLogPhase, OrchRole, OrchestratorMode, PEER_MUTUAL_AUTH_REQUIRED_REASON, PLATFORM_CONNECTED_MODES, PLATFORM_TO_ORCH_RECOGNIZED_TYPES, PRIVILEGED_ROOT_LABEL, PROTOCOL_VERSION, PayloadOmittedReason, PeerAuthMode, PeerAuthScheme, PeerForgetOutcome, RegisterableTriggerType, RunFailureClass, RuntimeFact, SCALER_EVENT_NAMES, SCHEMA_VERSION, SECURITY_HOLD_JOB_IDS, SSH_TRANSPORT_CAPABILITY, ScaleDownReason, ScaleDownReason as ScaleDownReason$1, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, ScalerOrphansAction, ScalerReloadOutcome, ScalerScaleDownPayload, ScalerScaleUpPayload, ScalerVmStatus, ScalerVmStopOutcome, ScalerVmTracker, SetupStepType, SourceOrigin, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, TRIGGER_EVENT_TYPES, TimeoutReason, TraceCheck, TriggerSource, TrustTierSchema, UnavailablePathsTrace, VariantKind, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_DISPATCH_ACK_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INTERNAL_ERROR, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PLAN_LIMIT, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_SUPERSEDED_BY_RECONNECT, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WebhookRelayResult, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentLabelOf, agentToOrchestratorMessageSchema, agentTypeLabel, appendChecks, approvalTimeoutSecondsSchema, artifactInvalidNameError, assertValidSecretKey, attestationVerifyStatusSchema, buildEventClaims, buildIdTokenSubject, buildJoinRefusal, buildTrustedPassthroughEnv, canonicalizeCapability, canonicalizeLabel, canonicalizeLabelSet, canonicalizeLabels, canonicalizeMatcher, checkArtifactName, classifyJoinRequest, coerceDispatchInputs, collectDiscriminatorTypes, createContentRequirementsTraceEntry, createDispatchFailureTraceEntry, createGlobalFilterTraceEntry, createWorkflowDecision, deriveOsArchLabels, diffRangeKindSchema, fanoutEnvelopeFields, flattenActor, formatLabelMatcher, getAccessLogColdDays, getSecretAuditLogColdDays, githubIngressPath, githubWebhookPath, globalEvalRoundResultSchema, hostLabel, hostSatisfiesTarget, hostToScalerPlatform, installGateJobId, isFailureStatus, isKnownCapability, isLockDynamicJobFn, isLockParallelStep, isLockStaticJob, isPullRequestFamilyTriggerEvent, isSecurityHoldJobId, isSelfReportedLabel, joinRequestSchema, logPullPlatformToOrchSchema, matchAllWorkflows, matchScopePattern, matchWorkflowsForEvent, matcherSatisfiedBy, materializeFanout, materializeResolvedHosts, materializeResolvedMatrix, matrixEnvelopeFields, mergeOrderedMaps, minAccessLogWarmDays, minSecretAuditLogWarmDays, partitionMatchers, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, platformToOsArchLabels, platformToTaints, reservedEventNamePrefix, resolveDiffRange, resolveRoleLabels, resolveScheduleInputs, resolveSecretsWithProvenance, resourceRequestNestedSchema, runtimeLabel, scalerAgentLabels, scalerLabel, scalerPlatformSchema, scheduleTriggerKey, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, sleep, stringifyActor, testRelayTriggerRequestSchema, testRelayUploadsInitRequestSchema, toIsoString, truncateDecisionsToByteBudget, trustPolicyUpdateSchema, trustedContributorHoldReason, validateScopeName, validateSecretKey, wrapUntrusted } from "@kici-dev/engine";
|
|
8
|
+
import { ACTOR_AGENT_SUFFIX, AGENT_API_REQUEST_TIMEOUT_MS, AGENT_FEATURE_LABELS, ALLOWED_SYSTEM_VARS, ARTIFACT_NAME_MAX_LENGTH, AccessLogAction, AccessLogOutcome, AccessLogSource, AccessLogTargetType, ActorType, AgentFailureCategory, ApprovalDecision, ArtifactCompleteAckOutcome, ArtifactDownloadOutcome, ArtifactUploadOutcome, AttestationOrigin, BREAKING_FLOOR, CANONICAL_STATUSES, CONTAINER_BUILD_RUNTIME_LABEL, CacheRefScope, CheckMode, CheckRunConclusion, CheckStepOutcome, CiTrustLevel, ConcurrencyStrategy, ContextDeleteErrorCode, ContextGateRejectReason, ContextType, DASHBOARD_REQUEST_TYPE_SET, DEFAULT_APPROVAL_EXPIRY_SECONDS, DEFAULT_CONCURRENCY_STRATEGY, DEFAULT_HOLD_EXPIRY_SECONDS, DEFERRED_PATHS_SUMMARY_SUFFIX, EventLogSource, EventLogStatus, ExecutionJobStatus, ExecutionRunStatus, ExecutionStepStatus, FanoutCause, FanoutError, FleetHostWriteRefusal, ForkPolicy, GLOBAL_EVAL_SKIPS_RESULT_AWARE_LABEL, HeldRunQueueType, HeldRunStatus, HoldScope, HoldType, ID_TOKEN_CLAIM_NAMES, INIT_LABEL, INIT_RUNNER_ROLE_LABEL, INSTALL_JOB_ID_PREFIX, INVALID_JOIN_REQUEST_MESSAGE, InitFailureCategory, InventorySelectorSchema, JOB_IMAGE_RUNTIME_LABEL, JOIN_PROTOCOL_UPGRADE_TOKEN, JOIN_PROTOCOL_V1_REMOVED_MESSAGE, JobRejectReason, JoinErrorCode, JoinRequestKind, KICI_AGENT_ENV_PREFIX, KICI_EVENT_NAME_PREFIX, KNOWN_ROLES, LabelMatcher, LockFileParseError, LogStream, MAX_APPROVAL_EXPIRY_SECONDS, MIN_APPROVAL_EXPIRY_SECONDS, MIN_PROTOCOL_VERSION, MinimumTrustSchema, ORCH_AGENT_CAPABILITIES, OWN_INGRESS_MODES, OrchLogPhase, OrchRole, OrchestratorMode, PEER_MUTUAL_AUTH_REQUIRED_REASON, PLATFORM_CONNECTED_MODES, PLATFORM_TO_ORCH_RECOGNIZED_TYPES, PRIVILEGED_ROOT_LABEL, PROTOCOL_VERSION, PayloadOmittedReason, PeerAuthMode, PeerAuthScheme, PeerForgetOutcome, RegisterableTriggerType, RunFailureClass, RuntimeFact, SCALER_EVENT_NAMES, SCHEMA_VERSION, SECURITY_HOLD_JOB_IDS, SSH_TRANSPORT_CAPABILITY, ScaleDownReason, ScaleDownReason as ScaleDownReason$1, ScalerBackendType, ScalerEventType, ScalerEventType as ScalerEventType$1, ScalerOrphansAction, ScalerReloadOutcome, ScalerScaleDownPayload, ScalerScaleUpPayload, ScalerVmStatus, ScalerVmStopOutcome, ScalerVmTracker, SetupStepType, SourceOrigin, SourceSubtype, TERMINAL_JOB_STATES, TERMINAL_RUN_STATES, TEST_RUN_LOG_CURSOR_START, TRIGGER_EVENT_TYPES, TimeoutReason, TraceCheck, TriggerSource, TrustTierSchema, UnavailablePathsTrace, VariantKind, WS_CLOSE_AGENT_AUTH_FAILED, WS_CLOSE_AUTH_TIMEOUT, WS_CLOSE_DISPATCH_ACK_TIMEOUT, WS_CLOSE_GOING_AWAY, WS_CLOSE_HEARTBEAT_TIMEOUT, WS_CLOSE_INTERNAL_ERROR, WS_CLOSE_INVALID_MESSAGE, WS_CLOSE_PLAN_LIMIT, WS_CLOSE_PROTOCOL_ERROR, WS_CLOSE_SUPERSEDED_BY_RECONNECT, WS_CLOSE_UNAUTHORIZED, WS_MAX_PAYLOAD_BYTES, WebhookRelayResult, WsRateLimiter, accessLogWarmSqlCase, agentAuthRequestSchema, agentLabelOf, agentToOrchestratorMessageSchema, agentTypeLabel, appendChecks, approvalTimeoutSecondsSchema, artifactInvalidNameError, assertValidSecretKey, attestationVerifyStatusSchema, buildEventClaims, buildIdTokenSubject, buildJoinRefusal, buildTrustedPassthroughEnv, canonicalizeCapability, canonicalizeLabel, canonicalizeLabelSet, canonicalizeLabels, canonicalizeMatcher, checkArtifactName, classifyJoinRequest, coerceDispatchInputs, collectDiscriminatorTypes, createContentRequirementsTraceEntry, createDispatchFailureTraceEntry, createGlobalFilterTraceEntry, createWorkflowDecision, deriveOsArchLabels, diffRangeKindSchema, fanoutEnvelopeFields, flattenActor, formatLabelMatcher, getAccessLogColdDays, getSecretAuditLogColdDays, githubIngressPath, githubWebhookPath, globalEvalRoundResultSchema, hostLabel, hostSatisfiesTarget, hostToScalerPlatform, installGateJobId, isFailureStatus, isKnownCapability, isLockDynamicJobFn, isLockParallelStep, isLockStaticJob, isPullRequestFamilyTriggerEvent, isSecurityHoldJobId, isSelfReportedLabel, joinRequestSchema, logPullPlatformToOrchSchema, matchAllWorkflows, matchScopePattern, matchWorkflowsForEvent, matcherSatisfiedBy, materializeFanout, materializeResolvedHosts, materializeResolvedMatrix, matrixEnvelopeFields, mergeOrderedMaps, minAccessLogWarmDays, minSecretAuditLogWarmDays, partitionMatchers, peerAuthRequestSchema, peerFromPeerMessageSchema, peerHelloResponseSchema, peerHelloSchema, platformToOsArchLabels, platformToTaints, reservedEventNamePrefix, resolveDiffRange, resolveRoleLabels, resolveScheduleInputs, resolveSecretsWithProvenance, resourceRequestNestedSchema, runtimeLabel, scalerAgentLabels, scalerLabel, scalerPlatformSchema, scheduleTriggerKey, secretAuditLogWarmSqlCase, shouldRecordAccess, shouldRecordSecretResolve, sleep, stringifyActor, testRelayTriggerRequestSchema, testRelayUploadsInitRequestSchema, toIsoString, truncateDecisionsToByteBudget, truncateReasonText, trustPolicyUpdateSchema, trustedContributorHoldReason, validateScopeName, validateSecretKey, wrapUntrusted } from "@kici-dev/engine";
|
|
9
9
|
import { z } from "zod";
|
|
10
10
|
import { X509Certificate, createCipheriv, createDecipheriv, createHash, createHmac, createPrivateKey, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
11
11
|
import { sql } from "kysely";
|
|
@@ -394,9 +394,9 @@ function clusterSentinelKey(prefix) {
|
|
|
394
394
|
const trimmed = prefix.replace(/\/+$/, "");
|
|
395
395
|
return trimmed.length === 0 ? ".kici-cluster-id" : `${trimmed}/.kici-cluster-id`;
|
|
396
396
|
}
|
|
397
|
-
var logger$
|
|
397
|
+
var logger$167, ClusterIdentity;
|
|
398
398
|
var init_cluster_identity = __esmMin((() => {
|
|
399
|
-
logger$
|
|
399
|
+
logger$167 = createLogger({ prefix: "cluster-identity" });
|
|
400
400
|
ClusterIdentity = class {
|
|
401
401
|
deps;
|
|
402
402
|
constructor(deps) {
|
|
@@ -415,11 +415,11 @@ var init_cluster_identity = __esmMin((() => {
|
|
|
415
415
|
*/
|
|
416
416
|
async validateS3Sentinel() {
|
|
417
417
|
if (!this.deps.s3Client || !this.deps.storageBucket) {
|
|
418
|
-
logger$
|
|
418
|
+
logger$167.debug("S3 not configured, skipping sentinel validation");
|
|
419
419
|
return;
|
|
420
420
|
}
|
|
421
421
|
if (this.deps.skipSentinelValidation) {
|
|
422
|
-
logger$
|
|
422
|
+
logger$167.warn("Skipping S3 sentinel validation (KICI_SKIP_S3_SENTINEL_VALIDATION=true)");
|
|
423
423
|
return;
|
|
424
424
|
}
|
|
425
425
|
const clusterId = await this.getClusterId();
|
|
@@ -427,7 +427,7 @@ var init_cluster_identity = __esmMin((() => {
|
|
|
427
427
|
const existing = await this.deps.s3Client.getObject(this.deps.storageBucket, sentinelKey);
|
|
428
428
|
if (existing === null) {
|
|
429
429
|
await this.deps.s3Client.putObject(this.deps.storageBucket, sentinelKey, clusterId);
|
|
430
|
-
logger$
|
|
430
|
+
logger$167.info("Wrote cluster identity sentinel to S3", {
|
|
431
431
|
clusterId,
|
|
432
432
|
bucket: this.deps.storageBucket,
|
|
433
433
|
sentinelKey
|
|
@@ -435,7 +435,7 @@ var init_cluster_identity = __esmMin((() => {
|
|
|
435
435
|
return;
|
|
436
436
|
}
|
|
437
437
|
if (existing.trim() !== clusterId) throw new Error(`Cluster identity mismatch: this orchestrator's database does not match the target pool. DB cluster_id=${clusterId}, S3 sentinel=${existing.trim()} (s3://${this.deps.storageBucket}/${sentinelKey}). Verify database connection configuration, or set KICI_STORAGE_PREFIX to isolate this cluster's sentinel.`);
|
|
438
|
-
logger$
|
|
438
|
+
logger$167.debug("S3 sentinel matches cluster_id", {
|
|
439
439
|
clusterId,
|
|
440
440
|
sentinelKey
|
|
441
441
|
});
|
|
@@ -446,7 +446,7 @@ var init_cluster_identity = __esmMin((() => {
|
|
|
446
446
|
*/
|
|
447
447
|
async validateAtStartup(instanceId) {
|
|
448
448
|
const clusterId = await this.getClusterId();
|
|
449
|
-
logger$
|
|
449
|
+
logger$167.info("Cluster identity validated", {
|
|
450
450
|
clusterId,
|
|
451
451
|
instanceId
|
|
452
452
|
});
|
|
@@ -1419,7 +1419,7 @@ var init_config$5 = __esmMin((() => {
|
|
|
1419
1419
|
*/
|
|
1420
1420
|
function configureJobSecretSealing(keys) {
|
|
1421
1421
|
sealingKeys = keys;
|
|
1422
|
-
if (keys === null) logger$
|
|
1422
|
+
if (keys === null) logger$166.warn("KICI_SECRET_KEY is not set: the secrets of queued and waiting jobs are stored unencrypted in the orchestrator database. Set KICI_SECRET_KEY (or KICI_SECRET_KEY_FILE) to encrypt them with the orchestrator master key.");
|
|
1423
1423
|
}
|
|
1424
1424
|
/** The AAD a stored job's sealed secrets are bound to. */
|
|
1425
1425
|
function jobSecretsAad(runId) {
|
|
@@ -1482,9 +1482,9 @@ function unsealJobConfig(runId, jobConfig, sealed) {
|
|
|
1482
1482
|
...secret
|
|
1483
1483
|
};
|
|
1484
1484
|
}
|
|
1485
|
-
var logger$
|
|
1485
|
+
var logger$166, SEALED_JOB_CONFIG_KEYS, JobSecretsUnsealError, sealingKeys;
|
|
1486
1486
|
var init_job_secret_seal = __esmMin((() => {
|
|
1487
|
-
logger$
|
|
1487
|
+
logger$166 = createLogger({ prefix: "job-secret-seal" });
|
|
1488
1488
|
SEALED_JOB_CONFIG_KEYS = [
|
|
1489
1489
|
"secrets",
|
|
1490
1490
|
"namespacedSecrets",
|
|
@@ -1529,10 +1529,10 @@ async function liveInstanceIds(db, instanceIds, graceMs) {
|
|
|
1529
1529
|
const rows = await db.selectFrom("cluster_instances").select(["instance_id"]).where("instance_id", "in", unique).where("last_heartbeat_at", ">", cutoff).execute();
|
|
1530
1530
|
return new Set(rows.map((r) => r.instance_id));
|
|
1531
1531
|
}
|
|
1532
|
-
var logger$
|
|
1532
|
+
var logger$165, DEFAULT_INSTANCE_HEARTBEAT_MS, InstanceHeartbeat;
|
|
1533
1533
|
var init_instance_heartbeat = __esmMin((() => {
|
|
1534
1534
|
init_run_detached();
|
|
1535
|
-
logger$
|
|
1535
|
+
logger$165 = createLogger({ prefix: "instance-heartbeat" });
|
|
1536
1536
|
DEFAULT_INSTANCE_HEARTBEAT_MS = 1e4;
|
|
1537
1537
|
InstanceHeartbeat = class {
|
|
1538
1538
|
db;
|
|
@@ -1556,10 +1556,10 @@ var init_instance_heartbeat = __esmMin((() => {
|
|
|
1556
1556
|
async start() {
|
|
1557
1557
|
await this.beat({ processStart: true });
|
|
1558
1558
|
this.timer = setInterval(() => {
|
|
1559
|
-
runDetached(logger$
|
|
1559
|
+
runDetached(logger$165, "Coordinator heartbeat", () => this.beat(), { instanceId: this.instanceId });
|
|
1560
1560
|
}, this.intervalMs);
|
|
1561
1561
|
this.timer.unref?.();
|
|
1562
|
-
logger$
|
|
1562
|
+
logger$165.info("Coordinator heartbeat started", {
|
|
1563
1563
|
instanceId: this.instanceId,
|
|
1564
1564
|
intervalMs: this.intervalMs
|
|
1565
1565
|
});
|
|
@@ -1579,7 +1579,7 @@ var init_instance_heartbeat = __esmMin((() => {
|
|
|
1579
1579
|
try {
|
|
1580
1580
|
await this.db.deleteFrom("cluster_instances").where("instance_id", "=", this.instanceId).execute();
|
|
1581
1581
|
} catch (err) {
|
|
1582
|
-
logger$
|
|
1582
|
+
logger$165.warn("Coordinator heartbeat retire failed", {
|
|
1583
1583
|
instanceId: this.instanceId,
|
|
1584
1584
|
error: toErrorMessage(err)
|
|
1585
1585
|
});
|
|
@@ -1607,7 +1607,7 @@ var init_instance_heartbeat = __esmMin((() => {
|
|
|
1607
1607
|
...opts.processStart === true ? { started_at: now } : {}
|
|
1608
1608
|
})).execute();
|
|
1609
1609
|
} catch (err) {
|
|
1610
|
-
logger$
|
|
1610
|
+
logger$165.warn("Coordinator heartbeat write failed", {
|
|
1611
1611
|
instanceId: this.instanceId,
|
|
1612
1612
|
error: toErrorMessage(err)
|
|
1613
1613
|
});
|
|
@@ -1788,9 +1788,9 @@ function parseSelectorColumnForExpiry(v, column) {
|
|
|
1788
1788
|
if (typeof v === "string") try {
|
|
1789
1789
|
const parsed = JSON.parse(v);
|
|
1790
1790
|
if (Array.isArray(parsed)) return parsed;
|
|
1791
|
-
logger$
|
|
1791
|
+
logger$164.warn("dispatch_queue selector column is not a JSON array", { column });
|
|
1792
1792
|
} catch {
|
|
1793
|
-
logger$
|
|
1793
|
+
logger$164.warn("dispatch_queue selector column holds malformed JSON", { column });
|
|
1794
1794
|
}
|
|
1795
1795
|
return [];
|
|
1796
1796
|
}
|
|
@@ -1805,13 +1805,13 @@ function jobPatternsSatisfiedBy(job, labels) {
|
|
|
1805
1805
|
if (job.excludePatterns.some((p) => matcherSatisfiedBy(p, labels))) return false;
|
|
1806
1806
|
return true;
|
|
1807
1807
|
}
|
|
1808
|
-
var logger$
|
|
1808
|
+
var logger$164, OWNER_COLUMN, STOPPED_RUN_DISPATCH_REASON, MAX_DEFERRED_UNOPENABLE, TERMINAL_DISPATCH_STATUSES, JobQueue;
|
|
1809
1809
|
var init_job_queue = __esmMin((() => {
|
|
1810
1810
|
init_agent_fit();
|
|
1811
1811
|
init_config$5();
|
|
1812
1812
|
init_job_secret_seal();
|
|
1813
1813
|
init_instance_heartbeat();
|
|
1814
|
-
logger$
|
|
1814
|
+
logger$164 = createLogger({ prefix: "job-queue" });
|
|
1815
1815
|
OWNER_COLUMN = "dispatch_queue.owner_instance_id";
|
|
1816
1816
|
STOPPED_RUN_DISPATCH_REASON = "the run stopped before this job was dispatched";
|
|
1817
1817
|
MAX_DEFERRED_UNOPENABLE = 1e3;
|
|
@@ -2224,7 +2224,7 @@ var init_job_queue = __esmMin((() => {
|
|
|
2224
2224
|
inserted: false
|
|
2225
2225
|
};
|
|
2226
2226
|
if (await this.expireDispatchOfStoppedRun(id, job.runId)) {
|
|
2227
|
-
logger$
|
|
2227
|
+
logger$164.info("Direct dispatch refused: the run has stopped", {
|
|
2228
2228
|
runId: job.runId,
|
|
2229
2229
|
jobId: id
|
|
2230
2230
|
});
|
|
@@ -2250,7 +2250,7 @@ var init_job_queue = __esmMin((() => {
|
|
|
2250
2250
|
*/
|
|
2251
2251
|
async settleClaimOfStoppedRun(id, runId) {
|
|
2252
2252
|
const settled = await this.expireDispatchOfStoppedRun(id, runId);
|
|
2253
|
-
if (settled) logger$
|
|
2253
|
+
if (settled) logger$164.info("Claimed job settled: its run stopped before it was put back", {
|
|
2254
2254
|
runId,
|
|
2255
2255
|
jobId: id
|
|
2256
2256
|
});
|
|
@@ -3358,7 +3358,7 @@ async function rejectPending(store, hold, reason) {
|
|
|
3358
3358
|
try {
|
|
3359
3359
|
return await store.reject(hold.org_id, hold.id, reason);
|
|
3360
3360
|
} catch (err) {
|
|
3361
|
-
logger$
|
|
3361
|
+
logger$163.info("Cancel of a held run lost the hold to a decision", {
|
|
3362
3362
|
runId: hold.run_id,
|
|
3363
3363
|
heldRunId: hold.id,
|
|
3364
3364
|
error: toErrorMessage(err)
|
|
@@ -3375,7 +3375,7 @@ async function withdrawWorkflowHold(deps, hold, reason, terminalWrite) {
|
|
|
3375
3375
|
try {
|
|
3376
3376
|
await deps.rejectHeldWorkflow(hold, reason, { runHeld: terminalWrite });
|
|
3377
3377
|
} catch (err) {
|
|
3378
|
-
logger$
|
|
3378
|
+
logger$163.error("Failed to withdraw a held run after its hold was rejected by a cancel", {
|
|
3379
3379
|
runId: hold.run_id,
|
|
3380
3380
|
heldRunId: hold.id,
|
|
3381
3381
|
error: toErrorMessage(err)
|
|
@@ -3383,10 +3383,10 @@ async function withdrawWorkflowHold(deps, hold, reason, terminalWrite) {
|
|
|
3383
3383
|
if (terminalWrite) await deps.executionTracker.cancelHeldRun(hold.run_id, reason);
|
|
3384
3384
|
}
|
|
3385
3385
|
}
|
|
3386
|
-
var logger$
|
|
3386
|
+
var logger$163;
|
|
3387
3387
|
var init_cancel_held_run = __esmMin((() => {
|
|
3388
3388
|
init_held_runs();
|
|
3389
|
-
logger$
|
|
3389
|
+
logger$163 = createLogger({ prefix: "cancel-held-run" });
|
|
3390
3390
|
}));
|
|
3391
3391
|
//#endregion
|
|
3392
3392
|
//#region src/cancel/cancel-run.ts
|
|
@@ -3415,7 +3415,7 @@ async function cancelRunWithReason(deps, runId, reason, options = {}) {
|
|
|
3415
3415
|
const force = options.force ?? false;
|
|
3416
3416
|
const runRow = await db.selectFrom("execution_runs").select(["status"]).where("run_id", "=", runId).executeTakeFirst();
|
|
3417
3417
|
if (runRow && TERMINAL_RUN_STATES.has(runRow.status)) {
|
|
3418
|
-
logger$
|
|
3418
|
+
logger$162.info("Cancel ignored: run already terminal", {
|
|
3419
3419
|
runId,
|
|
3420
3420
|
status: runRow.status
|
|
3421
3421
|
});
|
|
@@ -3433,7 +3433,7 @@ async function cancelRunWithReason(deps, runId, reason, options = {}) {
|
|
|
3433
3433
|
rejectHeldWorkflow: deps.rejectHeldWorkflow
|
|
3434
3434
|
}, runId, reason);
|
|
3435
3435
|
if (withdrawal !== "withdrawn") {
|
|
3436
|
-
logger$
|
|
3436
|
+
logger$162.info("Cancel refused: a decision reached the held run first", {
|
|
3437
3437
|
runId,
|
|
3438
3438
|
decision: withdrawal
|
|
3439
3439
|
});
|
|
@@ -3446,7 +3446,7 @@ async function cancelRunWithReason(deps, runId, reason, options = {}) {
|
|
|
3446
3446
|
};
|
|
3447
3447
|
}
|
|
3448
3448
|
await stampWithdrawnAttribution(db, runId, options);
|
|
3449
|
-
logger$
|
|
3449
|
+
logger$162.info("Cancelled held run", { runId });
|
|
3450
3450
|
return {
|
|
3451
3451
|
agentsNotified: 0,
|
|
3452
3452
|
unreachable: 0,
|
|
@@ -3482,7 +3482,7 @@ async function cancelRunWithReason(deps, runId, reason, options = {}) {
|
|
|
3482
3482
|
}
|
|
3483
3483
|
unreachable++;
|
|
3484
3484
|
stillUnwinding.add(jobId);
|
|
3485
|
-
logger$
|
|
3485
|
+
logger$162.warn("Cancel could not reach the coordinator that owns this job", {
|
|
3486
3486
|
runId,
|
|
3487
3487
|
jobId,
|
|
3488
3488
|
ownerInstanceId
|
|
@@ -3516,7 +3516,7 @@ async function cancelRunWithReason(deps, runId, reason, options = {}) {
|
|
|
3516
3516
|
await executionTracker.completeRunIfAllJobsTerminal(runId);
|
|
3517
3517
|
await executionTracker.cancelJoblessRun(runId, reason);
|
|
3518
3518
|
}
|
|
3519
|
-
logger$
|
|
3519
|
+
logger$162.info("Cancelled run", {
|
|
3520
3520
|
runId,
|
|
3521
3521
|
agentsNotified,
|
|
3522
3522
|
unreachable,
|
|
@@ -3544,7 +3544,7 @@ async function withdrawPendingHoldsOfActiveRun(deps, runId, reason) {
|
|
|
3544
3544
|
executionTracker: deps.executionTracker,
|
|
3545
3545
|
rejectHeldWorkflow: deps.rejectHeldWorkflow
|
|
3546
3546
|
}, runId, reason, { terminalWrite: false });
|
|
3547
|
-
logger$
|
|
3547
|
+
logger$162.info("Withdrew the pending approval requests of a cancelled run", {
|
|
3548
3548
|
runId,
|
|
3549
3549
|
withdrawn: withdrawal === "withdrawn",
|
|
3550
3550
|
...withdrawal !== "withdrawn" && { decision: withdrawal }
|
|
@@ -3633,7 +3633,7 @@ function sendLocalCancel(args) {
|
|
|
3633
3633
|
}));
|
|
3634
3634
|
return true;
|
|
3635
3635
|
} catch (err) {
|
|
3636
|
-
logger$
|
|
3636
|
+
logger$162.warn("Failed to send job.cancel to agent; treating job as orphaned", {
|
|
3637
3637
|
runId,
|
|
3638
3638
|
jobId,
|
|
3639
3639
|
agentId,
|
|
@@ -3642,12 +3642,12 @@ function sendLocalCancel(args) {
|
|
|
3642
3642
|
return false;
|
|
3643
3643
|
}
|
|
3644
3644
|
}
|
|
3645
|
-
var logger$
|
|
3645
|
+
var logger$162, RUN_RESUMING_AFTER_APPROVAL_MESSAGE, RUN_REJECTED_BEFORE_CANCEL_MESSAGE, RUN_EXPIRED_BEFORE_CANCEL_MESSAGE, REFUSAL_MESSAGES;
|
|
3646
3646
|
var init_cancel_run = __esmMin((() => {
|
|
3647
3647
|
init_instance_heartbeat();
|
|
3648
3648
|
init_held_runs();
|
|
3649
3649
|
init_cancel_held_run();
|
|
3650
|
-
logger$
|
|
3650
|
+
logger$162 = createLogger({ prefix: "cancel-run" });
|
|
3651
3651
|
RUN_RESUMING_AFTER_APPROVAL_MESSAGE = "Run is resuming after approval; cancel it again once it is running";
|
|
3652
3652
|
RUN_REJECTED_BEFORE_CANCEL_MESSAGE = "Run was rejected before the cancel reached it; the rejection ends it";
|
|
3653
3653
|
RUN_EXPIRED_BEFORE_CANCEL_MESSAGE = "Run approval request expired before the cancel reached it; the expiry ends it";
|
|
@@ -3849,7 +3849,7 @@ async function answerScalerOrphansRequest(manager, req) {
|
|
|
3849
3849
|
}))
|
|
3850
3850
|
};
|
|
3851
3851
|
const results = await manager.stopUntrackedVms(req.vmIds);
|
|
3852
|
-
for (const result of results) logger$
|
|
3852
|
+
for (const result of results) logger$161.info("scaler orphan stop result", {
|
|
3853
3853
|
vmId: result.vmId,
|
|
3854
3854
|
outcome: result.outcome,
|
|
3855
3855
|
pid: result.pid,
|
|
@@ -3861,7 +3861,7 @@ async function answerScalerOrphansRequest(manager, req) {
|
|
|
3861
3861
|
results
|
|
3862
3862
|
};
|
|
3863
3863
|
} catch (err) {
|
|
3864
|
-
logger$
|
|
3864
|
+
logger$161.error("scaler orphan request failed", {
|
|
3865
3865
|
action: req.action,
|
|
3866
3866
|
error: toErrorMessage(err)
|
|
3867
3867
|
});
|
|
@@ -3871,9 +3871,9 @@ async function answerScalerOrphansRequest(manager, req) {
|
|
|
3871
3871
|
};
|
|
3872
3872
|
}
|
|
3873
3873
|
}
|
|
3874
|
-
var logger$
|
|
3874
|
+
var logger$161, MAX_SCALER_ORPHANS_TIMEOUT_MS;
|
|
3875
3875
|
var init_orphan_requests = __esmMin((() => {
|
|
3876
|
-
logger$
|
|
3876
|
+
logger$161 = createLogger({ prefix: "scaler-orphans" });
|
|
3877
3877
|
MAX_SCALER_ORPHANS_TIMEOUT_MS = 24e4;
|
|
3878
3878
|
}));
|
|
3879
3879
|
//#endregion
|
|
@@ -4968,12 +4968,12 @@ function adopterIsLive(peer, nowMs, flapGraceMs) {
|
|
|
4968
4968
|
function clusterViewSufficient(knownCoordinatorPeerCount, connectedCoordinatorPeerCount) {
|
|
4969
4969
|
return knownCoordinatorPeerCount === 0 || connectedCoordinatorPeerCount > 0;
|
|
4970
4970
|
}
|
|
4971
|
-
var logger$
|
|
4971
|
+
var logger$160, PROVISION_OUTCOME_RETENTION_MS, EventProvisionReaper;
|
|
4972
4972
|
var init_event_provision_reaper = __esmMin((() => {
|
|
4973
4973
|
init_prometheus();
|
|
4974
4974
|
init_scaler_events();
|
|
4975
4975
|
init_run_detached();
|
|
4976
|
-
logger$
|
|
4976
|
+
logger$160 = createLogger({ prefix: "event-provision-reaper" });
|
|
4977
4977
|
PROVISION_OUTCOME_RETENTION_MS = 864e5;
|
|
4978
4978
|
EventProvisionReaper = class {
|
|
4979
4979
|
opts;
|
|
@@ -5023,7 +5023,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5023
5023
|
}
|
|
5024
5024
|
onBecomeLeader() {
|
|
5025
5025
|
this.isLeader = true;
|
|
5026
|
-
logger$
|
|
5026
|
+
logger$160.info("Became leader, starting event-provision reaper", { intervalMs: this.opts.bootIntervalMs });
|
|
5027
5027
|
this.schedule(this.opts.bootIntervalMs);
|
|
5028
5028
|
}
|
|
5029
5029
|
onLoseLeadership() {
|
|
@@ -5039,7 +5039,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5039
5039
|
this.unknownOwnerLogged.clear();
|
|
5040
5040
|
this.unseenPastGrace = 0;
|
|
5041
5041
|
this.publishGauges(false);
|
|
5042
|
-
logger$
|
|
5042
|
+
logger$160.info("Lost leadership, stopped event-provision reaper");
|
|
5043
5043
|
}
|
|
5044
5044
|
stop() {
|
|
5045
5045
|
this.onLoseLeadership();
|
|
@@ -5058,7 +5058,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5058
5058
|
try {
|
|
5059
5059
|
windows = await this.opts.resolveWindows();
|
|
5060
5060
|
} catch (err) {
|
|
5061
|
-
logger$
|
|
5061
|
+
logger$160.error("event-provision reap tick could not resolve its windows", { error: toErrorMessage(err) });
|
|
5062
5062
|
this.publishGauges(false);
|
|
5063
5063
|
return;
|
|
5064
5064
|
}
|
|
@@ -5074,7 +5074,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5074
5074
|
async reapProvisions(windows) {
|
|
5075
5075
|
if (!this.opts.canReap()) {
|
|
5076
5076
|
if (!this.reapBlocked) {
|
|
5077
|
-
logger$
|
|
5077
|
+
logger$160.warn("Skipping event-provision reap: this coordinator is connected to none of the coordinator peers it knows");
|
|
5078
5078
|
this.reapBlocked = true;
|
|
5079
5079
|
}
|
|
5080
5080
|
this.strandedSince.clear();
|
|
@@ -5096,7 +5096,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5096
5096
|
try {
|
|
5097
5097
|
candidates = await this.opts.listCandidates(new Date(now - windows.spawnTimeoutMs));
|
|
5098
5098
|
} catch (err) {
|
|
5099
|
-
logger$
|
|
5099
|
+
logger$160.error("event-provision reap tick failed to list candidates", { error: toErrorMessage(err) });
|
|
5100
5100
|
return;
|
|
5101
5101
|
}
|
|
5102
5102
|
if (!this.isLeader) return;
|
|
@@ -5121,14 +5121,14 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5121
5121
|
if (!reason) continue;
|
|
5122
5122
|
const lastAttempt = this.lastAttemptAt.get(candidate.agentId);
|
|
5123
5123
|
if (lastAttempt !== void 0 && now - lastAttempt < windows.reattemptIntervalMs) {
|
|
5124
|
-
logger$
|
|
5124
|
+
logger$160.debug("Skipping a stranded provision still inside its re-attempt window", {
|
|
5125
5125
|
agentId: candidate.agentId,
|
|
5126
5126
|
scalerName: candidate.scalerName,
|
|
5127
5127
|
reason
|
|
5128
5128
|
});
|
|
5129
5129
|
continue;
|
|
5130
5130
|
}
|
|
5131
|
-
logger$
|
|
5131
|
+
logger$160.warn("Reaping stranded event-scaler provision", {
|
|
5132
5132
|
agentId: candidate.agentId,
|
|
5133
5133
|
scalerName: candidate.scalerName,
|
|
5134
5134
|
reason,
|
|
@@ -5139,7 +5139,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5139
5139
|
try {
|
|
5140
5140
|
await this.opts.emitScaleDown(candidate, reason);
|
|
5141
5141
|
} catch (err) {
|
|
5142
|
-
logger$
|
|
5142
|
+
logger$160.error("Failed to tear down a stranded event-scaler provision", {
|
|
5143
5143
|
agentId: candidate.agentId,
|
|
5144
5144
|
scalerName: candidate.scalerName,
|
|
5145
5145
|
error: toErrorMessage(err)
|
|
@@ -5164,9 +5164,9 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5164
5164
|
async purgeClaims(retentionMs) {
|
|
5165
5165
|
try {
|
|
5166
5166
|
const purged = await this.opts.purgeExpiredClaims(new Date(Date.now() - retentionMs));
|
|
5167
|
-
if (purged > 0) logger$
|
|
5167
|
+
if (purged > 0) logger$160.info("Purged expired scaler provisioning claims", { purged });
|
|
5168
5168
|
} catch (err) {
|
|
5169
|
-
logger$
|
|
5169
|
+
logger$160.error("event-provision claim purge failed", { error: toErrorMessage(err) });
|
|
5170
5170
|
}
|
|
5171
5171
|
}
|
|
5172
5172
|
/**
|
|
@@ -5188,9 +5188,9 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5188
5188
|
const retentionMs = Math.max(PROVISION_OUTCOME_RETENTION_MS, 2 * spawnTimeoutMs);
|
|
5189
5189
|
try {
|
|
5190
5190
|
const purged = await this.opts.purgeProvisionOutcomes(new Date(Date.now() - retentionMs));
|
|
5191
|
-
if (purged > 0) logger$
|
|
5191
|
+
if (purged > 0) logger$160.info("Purged expired scaler provision outcomes", { purged });
|
|
5192
5192
|
} catch (err) {
|
|
5193
|
-
logger$
|
|
5193
|
+
logger$160.error("event-provision outcome purge failed", { error: toErrorMessage(err) });
|
|
5194
5194
|
}
|
|
5195
5195
|
}
|
|
5196
5196
|
/**
|
|
@@ -5204,11 +5204,11 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5204
5204
|
reconcileInterval(intervalMs) {
|
|
5205
5205
|
if (!this.isLeader) return;
|
|
5206
5206
|
if (!Number.isFinite(intervalMs) || intervalMs <= 0) {
|
|
5207
|
-
logger$
|
|
5207
|
+
logger$160.warn("Ignoring an unusable event-provision reaper interval", { intervalMs });
|
|
5208
5208
|
return;
|
|
5209
5209
|
}
|
|
5210
5210
|
if (intervalMs === this.scheduledIntervalMs) return;
|
|
5211
|
-
logger$
|
|
5211
|
+
logger$160.info("Rescheduling event-provision reaper", {
|
|
5212
5212
|
fromMs: this.scheduledIntervalMs,
|
|
5213
5213
|
toMs: intervalMs
|
|
5214
5214
|
});
|
|
@@ -5219,7 +5219,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5219
5219
|
if (this.timer) clearInterval(this.timer);
|
|
5220
5220
|
this.scheduledIntervalMs = intervalMs;
|
|
5221
5221
|
this.timer = setInterval(() => {
|
|
5222
|
-
runDetached(logger$
|
|
5222
|
+
runDetached(logger$160, "Event provision reaper tick", () => this.tick());
|
|
5223
5223
|
}, intervalMs);
|
|
5224
5224
|
this.timer.unref?.();
|
|
5225
5225
|
}
|
|
@@ -5261,7 +5261,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5261
5261
|
*/
|
|
5262
5262
|
verdict(candidate, now, windows) {
|
|
5263
5263
|
if (this.opts.isAgentRegistered(candidate.agentId)) {
|
|
5264
|
-
if (this.strandedSince.delete(candidate.agentId)) logger$
|
|
5264
|
+
if (this.strandedSince.delete(candidate.agentId)) logger$160.info("Event-scaler provision seen registered again, absence clock cleared", {
|
|
5265
5265
|
agentId: candidate.agentId,
|
|
5266
5266
|
scalerName: candidate.scalerName
|
|
5267
5267
|
});
|
|
@@ -5270,7 +5270,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5270
5270
|
const since = this.strandedSince.get(candidate.agentId);
|
|
5271
5271
|
if (since === void 0) {
|
|
5272
5272
|
this.strandedSince.set(candidate.agentId, now);
|
|
5273
|
-
logger$
|
|
5273
|
+
logger$160.info("Event-scaler provision registered nowhere, starting absence clock", {
|
|
5274
5274
|
agentId: candidate.agentId,
|
|
5275
5275
|
scalerName: candidate.scalerName,
|
|
5276
5276
|
adoptedBy: candidate.adoptedBy
|
|
@@ -5287,7 +5287,7 @@ var init_event_provision_reaper = __esmMin((() => {
|
|
|
5287
5287
|
if (!candidate.ownerInstanceId) {
|
|
5288
5288
|
if (!this.unknownOwnerLogged.has(candidate.agentId)) {
|
|
5289
5289
|
this.unknownOwnerLogged.add(candidate.agentId);
|
|
5290
|
-
logger$
|
|
5290
|
+
logger$160.warn("Sparing an event-scaler provision whose owner is unknown", {
|
|
5291
5291
|
agentId: candidate.agentId,
|
|
5292
5292
|
scalerName: candidate.scalerName,
|
|
5293
5293
|
spawnedAt: candidate.spawnedAt.toISOString()
|
|
@@ -5441,7 +5441,7 @@ function replyToPeerForgetRequest(msg, handler, send, logFields = {}) {
|
|
|
5441
5441
|
return;
|
|
5442
5442
|
}
|
|
5443
5443
|
handler(msg).then(reply, (err) => {
|
|
5444
|
-
logger$
|
|
5444
|
+
logger$159.error("Error answering a peer forget request", {
|
|
5445
5445
|
...logFields,
|
|
5446
5446
|
instanceId: msg.instanceId,
|
|
5447
5447
|
error: toErrorMessage(err)
|
|
@@ -5491,11 +5491,11 @@ async function forgetAcrossCoordinators(registry, selfInstanceId, instanceId, ti
|
|
|
5491
5491
|
function disconnectedCoordinatorIds(registry, selfInstanceId) {
|
|
5492
5492
|
return registry.getCoordinatorPeers().filter((peer) => !peer.connected && peer.instanceId !== selfInstanceId).map((peer) => peer.instanceId).sort();
|
|
5493
5493
|
}
|
|
5494
|
-
var logger$
|
|
5494
|
+
var logger$159, LOCAL_REFUSALS, PEER_FORGET_COORDINATORS_ONLY, PEER_FORGET_UNHANDLED, PeerLivenessWindowKind, PeerForgetWaiters;
|
|
5495
5495
|
var init_peer_forget = __esmMin((() => {
|
|
5496
5496
|
init_peer_request_waiters();
|
|
5497
5497
|
init_event_provision_reaper();
|
|
5498
|
-
logger$
|
|
5498
|
+
logger$159 = createLogger({ prefix: "peer-forget" });
|
|
5499
5499
|
LOCAL_REFUSALS = /* @__PURE__ */ new Set([
|
|
5500
5500
|
PeerForgetOutcome.enum.connected,
|
|
5501
5501
|
PeerForgetOutcome.enum.recent,
|
|
@@ -5548,7 +5548,7 @@ function replyToScalerOrphansRequest(msg, handler, send, logFields = {}) {
|
|
|
5548
5548
|
return;
|
|
5549
5549
|
}
|
|
5550
5550
|
handler(msg).then(reply, (err) => {
|
|
5551
|
-
logger$
|
|
5551
|
+
logger$158.error("Error answering a scaler orphan request", {
|
|
5552
5552
|
...logFields,
|
|
5553
5553
|
action: msg.action,
|
|
5554
5554
|
error: toErrorMessage(err)
|
|
@@ -5559,10 +5559,10 @@ function replyToScalerOrphansRequest(msg, handler, send, logFields = {}) {
|
|
|
5559
5559
|
});
|
|
5560
5560
|
});
|
|
5561
5561
|
}
|
|
5562
|
-
var logger$
|
|
5562
|
+
var logger$158, SCALER_ORPHANS_UNHANDLED_ERROR, SCALER_ORPHANS_COORDINATORS_ONLY_ERROR, ScalerOrphansForwardFailure, ScalerOrphansWaiters;
|
|
5563
5563
|
var init_scaler_orphans_peer = __esmMin((() => {
|
|
5564
5564
|
init_peer_request_waiters();
|
|
5565
|
-
logger$
|
|
5565
|
+
logger$158 = createLogger({ prefix: "scaler-orphans-peer" });
|
|
5566
5566
|
SCALER_ORPHANS_UNHANDLED_ERROR = "scaler orphan requests are not handled by this peer";
|
|
5567
5567
|
SCALER_ORPHANS_COORDINATORS_ONLY_ERROR = "scaler orphan requests are accepted from coordinators only";
|
|
5568
5568
|
ScalerOrphansForwardFailure = z.enum(["not-connected", "timeout"]);
|
|
@@ -5608,7 +5608,7 @@ function replyToScalerReloadRequest(msg, handler, send, logFields = {}) {
|
|
|
5608
5608
|
...base,
|
|
5609
5609
|
...result
|
|
5610
5610
|
}), (err) => {
|
|
5611
|
-
logger$
|
|
5611
|
+
logger$157.error("Error answering a scaler reload request", {
|
|
5612
5612
|
...logFields,
|
|
5613
5613
|
error: toErrorMessage(err)
|
|
5614
5614
|
});
|
|
@@ -5619,10 +5619,10 @@ function replyToScalerReloadRequest(msg, handler, send, logFields = {}) {
|
|
|
5619
5619
|
});
|
|
5620
5620
|
});
|
|
5621
5621
|
}
|
|
5622
|
-
var logger$
|
|
5622
|
+
var logger$157, SCALER_RELOAD_UNHANDLED_DETAIL, SCALER_RELOAD_COORDINATORS_ONLY_DETAIL, ScalerReloadWaiters;
|
|
5623
5623
|
var init_scaler_reload_peer = __esmMin((() => {
|
|
5624
5624
|
init_peer_request_waiters();
|
|
5625
|
-
logger$
|
|
5625
|
+
logger$157 = createLogger({ prefix: "scaler-reload-peer" });
|
|
5626
5626
|
SCALER_RELOAD_UNHANDLED_DETAIL = "scaler reload requests are not handled by this peer";
|
|
5627
5627
|
SCALER_RELOAD_COORDINATORS_ONLY_DETAIL = "scaler reload requests are accepted from coordinators only";
|
|
5628
5628
|
ScalerReloadWaiters = class extends PeerRequestWaiters {
|
|
@@ -5958,7 +5958,7 @@ var init_join_token = __esmMin((() => {
|
|
|
5958
5958
|
return row ? routingFromRow(row.routing_info, row.role) : null;
|
|
5959
5959
|
}
|
|
5960
5960
|
};
|
|
5961
|
-
})), logger$
|
|
5961
|
+
})), logger$156, NO_AUTH_METHOD_MESSAGE, NO_AUTH_METHOD_REMEDY, PEER_MUTUAL_AUTH_UNSUPPORTED_MESSAGE, DEFAULT_HANDSHAKE_TIMEOUT_MS, PeerDialOrigin$1, SOFTWARE_VERSION$1, PeerClient$1;
|
|
5962
5962
|
var init_peer_client = __esmMin((() => {
|
|
5963
5963
|
init_peer_registry();
|
|
5964
5964
|
init_peer_forget();
|
|
@@ -5967,7 +5967,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
5967
5967
|
init_peer_crypto();
|
|
5968
5968
|
init_join_token();
|
|
5969
5969
|
init_run_detached();
|
|
5970
|
-
logger$
|
|
5970
|
+
logger$156 = createLogger({ prefix: "peer-client" });
|
|
5971
5971
|
NO_AUTH_METHOD_MESSAGE = "No peer auth method: this instance has no credential file and no join token";
|
|
5972
5972
|
NO_AUTH_METHOD_REMEDY = "A coordinator issues its own credential unless an operator revoked it; a worker needs KICI_CLUSTER_JOIN_TOKEN";
|
|
5973
5973
|
PEER_MUTUAL_AUTH_UNSUPPORTED_MESSAGE = "Peer does not support mutual authentication; upgrade every orchestrator in the cluster";
|
|
@@ -5979,7 +5979,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
5979
5979
|
PeerDialOrigin["Discovered"] = "discovered";
|
|
5980
5980
|
return PeerDialOrigin;
|
|
5981
5981
|
}({});
|
|
5982
|
-
SOFTWARE_VERSION$1 = "0.
|
|
5982
|
+
SOFTWARE_VERSION$1 = "0.20.0";
|
|
5983
5983
|
PeerClient$1 = class {
|
|
5984
5984
|
ws = null;
|
|
5985
5985
|
_state = "disconnected";
|
|
@@ -6083,7 +6083,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6083
6083
|
}
|
|
6084
6084
|
connect() {
|
|
6085
6085
|
if (this._state !== "disconnected") {
|
|
6086
|
-
logger$
|
|
6086
|
+
logger$156.warn("connect() called while not disconnected", { state: this._state });
|
|
6087
6087
|
return;
|
|
6088
6088
|
}
|
|
6089
6089
|
this.intentionalDisconnect = false;
|
|
@@ -6270,7 +6270,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6270
6270
|
}
|
|
6271
6271
|
});
|
|
6272
6272
|
} catch (err) {
|
|
6273
|
-
logger$
|
|
6273
|
+
logger$156.error("Failed to create peer WebSocket", { error: toErrorMessage(err) });
|
|
6274
6274
|
this._state = "disconnected";
|
|
6275
6275
|
this.scheduleReconnect();
|
|
6276
6276
|
return;
|
|
@@ -6279,13 +6279,13 @@ var init_peer_client = __esmMin((() => {
|
|
|
6279
6279
|
this.armHandshakeTimer(ws);
|
|
6280
6280
|
this.ws.on("open", () => {
|
|
6281
6281
|
this._state = "handshaking";
|
|
6282
|
-
logger$
|
|
6282
|
+
logger$156.info("Connected to peer, waiting for ECDH handshake", { url: this.url });
|
|
6283
6283
|
});
|
|
6284
6284
|
this.ws.on("message", (data) => {
|
|
6285
6285
|
this.handleMessage(data);
|
|
6286
6286
|
});
|
|
6287
6287
|
this.ws.on("close", (code, reason) => {
|
|
6288
|
-
logger$
|
|
6288
|
+
logger$156.info("Peer connection closed", {
|
|
6289
6289
|
code,
|
|
6290
6290
|
reason: reason.toString(),
|
|
6291
6291
|
targetInstanceId: this._targetInstanceId
|
|
@@ -6303,7 +6303,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6303
6303
|
if (!this.intentionalDisconnect) this.scheduleReconnect();
|
|
6304
6304
|
});
|
|
6305
6305
|
this.ws.on("error", (err) => {
|
|
6306
|
-
logger$
|
|
6306
|
+
logger$156.error("Peer WebSocket error", { error: err.message });
|
|
6307
6307
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
|
|
6308
6308
|
});
|
|
6309
6309
|
}
|
|
@@ -6323,19 +6323,19 @@ var init_peer_client = __esmMin((() => {
|
|
|
6323
6323
|
try {
|
|
6324
6324
|
decrypted = decryptMessage(raw, this.sessionKey);
|
|
6325
6325
|
} catch {
|
|
6326
|
-
logger$
|
|
6326
|
+
logger$156.warn("Failed to decrypt message from peer");
|
|
6327
6327
|
return;
|
|
6328
6328
|
}
|
|
6329
6329
|
let parsed;
|
|
6330
6330
|
try {
|
|
6331
6331
|
parsed = JSON.parse(decrypted);
|
|
6332
6332
|
} catch {
|
|
6333
|
-
logger$
|
|
6333
|
+
logger$156.warn("Malformed JSON from peer");
|
|
6334
6334
|
return;
|
|
6335
6335
|
}
|
|
6336
6336
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
6337
6337
|
if (!msgResult.success) {
|
|
6338
|
-
logger$
|
|
6338
|
+
logger$156.warn("Invalid message from peer", { errors: msgResult.error.issues });
|
|
6339
6339
|
return;
|
|
6340
6340
|
}
|
|
6341
6341
|
this.routeMessage(msgResult.data);
|
|
@@ -6345,7 +6345,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6345
6345
|
this.handshakeTimer = setTimeout(() => {
|
|
6346
6346
|
this.handshakeTimer = null;
|
|
6347
6347
|
if (this.ws !== ws || this._state === "connected") return;
|
|
6348
|
-
logger$
|
|
6348
|
+
logger$156.warn("Peer handshake timed out", {
|
|
6349
6349
|
url: this.url,
|
|
6350
6350
|
state: this._state,
|
|
6351
6351
|
timeoutMs: this.handshakeTimeoutMs
|
|
@@ -6376,7 +6376,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6376
6376
|
}
|
|
6377
6377
|
/** Close a connection whose handshake broke the protocol. */
|
|
6378
6378
|
abortHandshake(reason) {
|
|
6379
|
-
logger$
|
|
6379
|
+
logger$156.warn("Closing peer connection during the handshake", {
|
|
6380
6380
|
url: this.url,
|
|
6381
6381
|
reason
|
|
6382
6382
|
});
|
|
@@ -6387,7 +6387,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6387
6387
|
}
|
|
6388
6388
|
/** Close a connection whose server failed mutual authentication. */
|
|
6389
6389
|
failMutualAuth(failure, closeCode) {
|
|
6390
|
-
logger$
|
|
6390
|
+
logger$156.error("Peer failed mutual authentication", {
|
|
6391
6391
|
url: this.url,
|
|
6392
6392
|
failure,
|
|
6393
6393
|
origin: this.origin
|
|
@@ -6417,7 +6417,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6417
6417
|
}
|
|
6418
6418
|
const authSchemes = hello.data.authSchemes ?? [];
|
|
6419
6419
|
if (!authSchemes.includes(PeerAuthScheme.enum["mutual-v2"])) {
|
|
6420
|
-
logger$
|
|
6420
|
+
logger$156.error(PEER_MUTUAL_AUTH_UNSUPPORTED_MESSAGE, {
|
|
6421
6421
|
url: this.url,
|
|
6422
6422
|
advertised: authSchemes
|
|
6423
6423
|
});
|
|
@@ -6435,7 +6435,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6435
6435
|
try {
|
|
6436
6436
|
handshakeKey = deriveSessionKey(ecdh.privateKey, serverPub, nonce);
|
|
6437
6437
|
} catch (err) {
|
|
6438
|
-
logger$
|
|
6438
|
+
logger$156.error("ECDH key derivation failed", { error: toErrorMessage(err) });
|
|
6439
6439
|
this.abortHandshake("Key derivation failed");
|
|
6440
6440
|
return;
|
|
6441
6441
|
}
|
|
@@ -6455,7 +6455,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6455
6455
|
}));
|
|
6456
6456
|
this._state = "authenticating";
|
|
6457
6457
|
this.sendAuthRequest().catch((err) => {
|
|
6458
|
-
logger$
|
|
6458
|
+
logger$156.error("Failed to send auth request", { error: toErrorMessage(err) });
|
|
6459
6459
|
this.abortHandshake("Auth request failed");
|
|
6460
6460
|
});
|
|
6461
6461
|
}
|
|
@@ -6526,13 +6526,13 @@ var init_peer_client = __esmMin((() => {
|
|
|
6526
6526
|
/** The server proved itself: register the peer and start the connected session. */
|
|
6527
6527
|
completeAcceptance(msg) {
|
|
6528
6528
|
this.clearHandshakeTimer();
|
|
6529
|
-
if (msg.softwareVersion) logger$
|
|
6529
|
+
if (msg.softwareVersion) logger$156.info("Coordinator software version", {
|
|
6530
6530
|
localVersion: SOFTWARE_VERSION$1,
|
|
6531
6531
|
coordinatorVersion: msg.softwareVersion
|
|
6532
6532
|
});
|
|
6533
6533
|
this._targetInstanceId = msg.instanceId;
|
|
6534
6534
|
this.onAuthenticated?.(msg.instanceId);
|
|
6535
|
-
logger$
|
|
6535
|
+
logger$156.info("Peer auth accepted", {
|
|
6536
6536
|
targetInstanceId: msg.instanceId,
|
|
6537
6537
|
agentCount: msg.agents?.length ?? 0,
|
|
6538
6538
|
scalerBackends: msg.scalerCapacity?.length ?? 0
|
|
@@ -6585,7 +6585,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6585
6585
|
* deletion would not help.
|
|
6586
6586
|
*/
|
|
6587
6587
|
handleAuthRejected(reason) {
|
|
6588
|
-
logger$
|
|
6588
|
+
logger$156.error("Peer auth rejected", { reason });
|
|
6589
6589
|
const wasJoiner = this.isJoiner;
|
|
6590
6590
|
const joinComplete = this.joinComplete;
|
|
6591
6591
|
const provedCredential = this.lastProvedCredential;
|
|
@@ -6601,15 +6601,15 @@ var init_peer_client = __esmMin((() => {
|
|
|
6601
6601
|
finish();
|
|
6602
6602
|
return;
|
|
6603
6603
|
}
|
|
6604
|
-
runDetached(logger$
|
|
6604
|
+
runDetached(logger$156, "Peer auth rejection handling", async () => {
|
|
6605
6605
|
try {
|
|
6606
6606
|
const action = await this.authCoordinator.reportRejection(provedCredential, reason);
|
|
6607
|
-
logger$
|
|
6607
|
+
logger$156.info("Coordinator rejection action", {
|
|
6608
6608
|
reason,
|
|
6609
6609
|
action
|
|
6610
6610
|
});
|
|
6611
6611
|
} catch (err) {
|
|
6612
|
-
logger$
|
|
6612
|
+
logger$156.warn("Coordinator rejection handling failed", {
|
|
6613
6613
|
error: toErrorMessage(err),
|
|
6614
6614
|
reason
|
|
6615
6615
|
});
|
|
@@ -6654,7 +6654,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6654
6654
|
tokenRouting = parsed.routingB64;
|
|
6655
6655
|
psk = Buffer.from(tokenHashOf(parsed.secretHex), "hex");
|
|
6656
6656
|
} catch (err) {
|
|
6657
|
-
logger$
|
|
6657
|
+
logger$156.error("KICI_CLUSTER_JOIN_TOKEN is not a valid join token", { error: toErrorMessage(err) });
|
|
6658
6658
|
decision.complete(null);
|
|
6659
6659
|
this.abortHandshake("Invalid join token");
|
|
6660
6660
|
return;
|
|
@@ -6669,10 +6669,10 @@ var init_peer_client = __esmMin((() => {
|
|
|
6669
6669
|
targetUrl: this.url,
|
|
6670
6670
|
remedy: NO_AUTH_METHOD_REMEDY
|
|
6671
6671
|
};
|
|
6672
|
-
if (this.noAuthWarned) logger$
|
|
6672
|
+
if (this.noAuthWarned) logger$156.debug(NO_AUTH_METHOD_MESSAGE, meta);
|
|
6673
6673
|
else {
|
|
6674
6674
|
this.noAuthWarned = true;
|
|
6675
|
-
logger$
|
|
6675
|
+
logger$156.warn(NO_AUTH_METHOD_MESSAGE, meta);
|
|
6676
6676
|
}
|
|
6677
6677
|
if (this.ws.readyState === WebSocket.OPEN) this.ws.close(1e3, "No auth method");
|
|
6678
6678
|
return;
|
|
@@ -6691,7 +6691,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6691
6691
|
clientProof,
|
|
6692
6692
|
mode
|
|
6693
6693
|
};
|
|
6694
|
-
logger$
|
|
6694
|
+
logger$156.info("Sending mutual-v2 auth request", {
|
|
6695
6695
|
targetUrl: this.url,
|
|
6696
6696
|
mode,
|
|
6697
6697
|
...mode === PeerAuthMode.enum.token && { reason: "no-credential-file" }
|
|
@@ -6750,7 +6750,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6750
6750
|
break;
|
|
6751
6751
|
case "job.reroute":
|
|
6752
6752
|
this.onJobReroute(msg).catch((err) => {
|
|
6753
|
-
logger$
|
|
6753
|
+
logger$156.error("Error handling job reroute", { error: toErrorMessage(err) });
|
|
6754
6754
|
});
|
|
6755
6755
|
break;
|
|
6756
6756
|
case "job.reroute.ack": {
|
|
@@ -6758,7 +6758,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6758
6758
|
if (waiter) {
|
|
6759
6759
|
clearTimeout(waiter.timer);
|
|
6760
6760
|
this.ackWaiters.delete(msg.messageId);
|
|
6761
|
-
if (!msg.accepted) logger$
|
|
6761
|
+
if (!msg.accepted) logger$156.info("Reroute ACK rejected by peer", {
|
|
6762
6762
|
targetInstanceId: this._targetInstanceId,
|
|
6763
6763
|
reason: msg.reason
|
|
6764
6764
|
});
|
|
@@ -6768,7 +6768,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6768
6768
|
}
|
|
6769
6769
|
case "job.progress":
|
|
6770
6770
|
if (this._targetInstanceId === null) {
|
|
6771
|
-
logger$
|
|
6771
|
+
logger$156.warn("Dropping job.progress received before peer authentication completed");
|
|
6772
6772
|
break;
|
|
6773
6773
|
}
|
|
6774
6774
|
this.onJobProgress(msg, this._targetInstanceId, (out) => this.send(out));
|
|
@@ -6795,7 +6795,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6795
6795
|
if (this.onPeerCacheUploadRequest) this.onPeerCacheUploadRequest(msg).then((response) => {
|
|
6796
6796
|
this.send(response);
|
|
6797
6797
|
}).catch((err) => {
|
|
6798
|
-
logger$
|
|
6798
|
+
logger$156.error("Error handling cache upload request", { error: toErrorMessage(err) });
|
|
6799
6799
|
});
|
|
6800
6800
|
break;
|
|
6801
6801
|
case "peer.cache.upload.response": {
|
|
@@ -6808,7 +6808,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6808
6808
|
break;
|
|
6809
6809
|
}
|
|
6810
6810
|
case "peer.auth.request":
|
|
6811
|
-
logger$
|
|
6811
|
+
logger$156.warn("Unexpected peer.auth.request on outgoing connection");
|
|
6812
6812
|
break;
|
|
6813
6813
|
case "raft.vote.response":
|
|
6814
6814
|
this.onRaftVoteResponse?.(msg);
|
|
@@ -6840,7 +6840,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6840
6840
|
handler(msg).then((result) => {
|
|
6841
6841
|
sendReply(result);
|
|
6842
6842
|
}).catch((err) => {
|
|
6843
|
-
logger$
|
|
6843
|
+
logger$156.error("Error executing peer config reload", { error: toErrorMessage(err) });
|
|
6844
6844
|
sendReply({
|
|
6845
6845
|
success: false,
|
|
6846
6846
|
errors: [toErrorMessage(err)]
|
|
@@ -6867,7 +6867,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6867
6867
|
break;
|
|
6868
6868
|
}
|
|
6869
6869
|
case "peer.logs.collect.request":
|
|
6870
|
-
runDetached(logger$
|
|
6870
|
+
runDetached(logger$156, "Peer logs collect request", () => this.onLogsCollectRequest?.(msg, (out) => this.send(out)), {
|
|
6871
6871
|
messageType: msg.type,
|
|
6872
6872
|
peerId: this._targetInstanceId
|
|
6873
6873
|
});
|
|
@@ -6900,7 +6900,7 @@ var init_peer_client = __esmMin((() => {
|
|
|
6900
6900
|
this.cancelReconnect();
|
|
6901
6901
|
const delay = this.getReconnectDelay();
|
|
6902
6902
|
this.reconnectAttempts++;
|
|
6903
|
-
logger$
|
|
6903
|
+
logger$156.info("Scheduling peer reconnect", {
|
|
6904
6904
|
attempt: this.reconnectAttempts,
|
|
6905
6905
|
delayMs: Math.round(delay)
|
|
6906
6906
|
});
|
|
@@ -7302,12 +7302,12 @@ function deferred() {
|
|
|
7302
7302
|
resolve
|
|
7303
7303
|
};
|
|
7304
7304
|
}
|
|
7305
|
-
var logger$
|
|
7305
|
+
var logger$155, DEFAULT_JOIN_WAIT_TIMEOUT_MS, MAX_DECIDE_ITERATIONS, RejectionAction, PeerAuthCoordinator$1;
|
|
7306
7306
|
var init_peer_auth_coordinator = __esmMin((() => {
|
|
7307
7307
|
init_peer_credentials();
|
|
7308
7308
|
init_run_detached();
|
|
7309
7309
|
init_coordinator_credential();
|
|
7310
|
-
logger$
|
|
7310
|
+
logger$155 = createLogger({ prefix: "peer-auth-coordinator" });
|
|
7311
7311
|
DEFAULT_JOIN_WAIT_TIMEOUT_MS = 1e4;
|
|
7312
7312
|
MAX_DECIDE_ITERATIONS = 3;
|
|
7313
7313
|
RejectionAction = /* @__PURE__ */ function(RejectionAction) {
|
|
@@ -7353,7 +7353,7 @@ var init_peer_auth_coordinator = __esmMin((() => {
|
|
|
7353
7353
|
try {
|
|
7354
7354
|
cred = await readCredentialFile(this.credentialFile);
|
|
7355
7355
|
} catch (err) {
|
|
7356
|
-
logger$
|
|
7356
|
+
logger$155.warn("Ignoring unreadable peer credential file", {
|
|
7357
7357
|
instanceId: this.instanceId,
|
|
7358
7358
|
path: this.credentialFile,
|
|
7359
7359
|
error: toErrorMessage(err)
|
|
@@ -7420,7 +7420,7 @@ var init_peer_auth_coordinator = __esmMin((() => {
|
|
|
7420
7420
|
}
|
|
7421
7421
|
makeComplete(join) {
|
|
7422
7422
|
return (issued) => {
|
|
7423
|
-
runDetached(logger$
|
|
7423
|
+
runDetached(logger$155, "Peer token-join completion", () => this.withLock(async () => {
|
|
7424
7424
|
if (issued) await writeCredentialFile(this.credentialFile, issued);
|
|
7425
7425
|
if (this.inFlightJoin === join) this.inFlightJoin = null;
|
|
7426
7426
|
join.resolve(issued);
|
|
@@ -7434,7 +7434,7 @@ var init_peer_auth_coordinator = __esmMin((() => {
|
|
|
7434
7434
|
const timeout = new Promise((res) => {
|
|
7435
7435
|
timer = setTimeout(() => {
|
|
7436
7436
|
if (this.inFlightJoin === join) this.inFlightJoin = null;
|
|
7437
|
-
logger$
|
|
7437
|
+
logger$155.warn("In-flight peer token-join timed out; waiter will retry", { instanceId: this.instanceId });
|
|
7438
7438
|
res();
|
|
7439
7439
|
}, this.joinWaitTimeoutMs);
|
|
7440
7440
|
});
|
|
@@ -7445,7 +7445,7 @@ var init_peer_auth_coordinator = __esmMin((() => {
|
|
|
7445
7445
|
return this.withLock(async () => {
|
|
7446
7446
|
const cred = await this.readValidCredential();
|
|
7447
7447
|
if (cred && cred.credential !== provedCredential) {
|
|
7448
|
-
logger$
|
|
7448
|
+
logger$155.info("Credential refreshed by sibling; retrying credential auth", {
|
|
7449
7449
|
instanceId: this.instanceId,
|
|
7450
7450
|
reason
|
|
7451
7451
|
});
|
|
@@ -7454,7 +7454,7 @@ var init_peer_auth_coordinator = __esmMin((() => {
|
|
|
7454
7454
|
if (cred && this.corroborateRejection) {
|
|
7455
7455
|
const verdict = await this.corroborateRejection(cred.credential);
|
|
7456
7456
|
if (verdict !== "not-held") {
|
|
7457
|
-
logger$
|
|
7457
|
+
logger$155.warn("A peer rejected a credential this cluster holds as valid; keeping the credential file", {
|
|
7458
7458
|
instanceId: this.instanceId,
|
|
7459
7459
|
reason,
|
|
7460
7460
|
verdict
|
|
@@ -7464,12 +7464,12 @@ var init_peer_auth_coordinator = __esmMin((() => {
|
|
|
7464
7464
|
}
|
|
7465
7465
|
try {
|
|
7466
7466
|
await unlink(this.credentialFile);
|
|
7467
|
-
logger$
|
|
7467
|
+
logger$155.warn("Deleted stale credential file after server rejection", {
|
|
7468
7468
|
instanceId: this.instanceId,
|
|
7469
7469
|
reason
|
|
7470
7470
|
});
|
|
7471
7471
|
} catch (err) {
|
|
7472
|
-
if (err.code !== "ENOENT") logger$
|
|
7472
|
+
if (err.code !== "ENOENT") logger$155.warn("Failed to delete stale credential file", {
|
|
7473
7473
|
instanceId: this.instanceId,
|
|
7474
7474
|
path: this.credentialFile
|
|
7475
7475
|
});
|
|
@@ -7592,7 +7592,7 @@ function createPeerHandler(deps) {
|
|
|
7592
7592
|
const connected = peerRegistry.getConnectedWorkerPeers().length;
|
|
7593
7593
|
if (shouldAdmitWorker(role, ceiling, connected)) return true;
|
|
7594
7594
|
const reason = `Plan limit: this organization allows ${(ceiling ?? 0) + 1} orchestrator(s), coordinators and workers combined`;
|
|
7595
|
-
logger$
|
|
7595
|
+
logger$154.warn("Worker join refused by plan ceiling", {
|
|
7596
7596
|
peerInstanceId,
|
|
7597
7597
|
connected,
|
|
7598
7598
|
ceiling
|
|
@@ -7633,7 +7633,7 @@ function createPeerHandler(deps) {
|
|
|
7633
7633
|
break;
|
|
7634
7634
|
case "job.reroute":
|
|
7635
7635
|
onJobReroute(msg).catch((err) => {
|
|
7636
|
-
logger$
|
|
7636
|
+
logger$154.error("Error handling job reroute from peer", { error: toErrorMessage(err) });
|
|
7637
7637
|
});
|
|
7638
7638
|
break;
|
|
7639
7639
|
case "job.reroute.ack": {
|
|
@@ -7670,7 +7670,7 @@ function createPeerHandler(deps) {
|
|
|
7670
7670
|
if (onPeerCacheUploadRequest) onPeerCacheUploadRequest(msg, conn.peerInstanceId).then((response) => {
|
|
7671
7671
|
sendEncryptedMessage(conn.ws, conn.sessionKey, response);
|
|
7672
7672
|
}).catch((err) => {
|
|
7673
|
-
logger$
|
|
7673
|
+
logger$154.error("Error handling cache upload request from peer", {
|
|
7674
7674
|
peerId: conn.peerInstanceId,
|
|
7675
7675
|
error: toErrorMessage(err)
|
|
7676
7676
|
});
|
|
@@ -7691,13 +7691,13 @@ function createPeerHandler(deps) {
|
|
|
7691
7691
|
});
|
|
7692
7692
|
break;
|
|
7693
7693
|
case "peer.cache.upload.response":
|
|
7694
|
-
logger$
|
|
7694
|
+
logger$154.warn("Unexpected peer.cache.upload.response on incoming connection");
|
|
7695
7695
|
break;
|
|
7696
7696
|
case "peer.auth.request":
|
|
7697
|
-
logger$
|
|
7697
|
+
logger$154.warn("Unexpected peer.auth.request after authentication");
|
|
7698
7698
|
break;
|
|
7699
7699
|
case "peer.auth.response":
|
|
7700
|
-
logger$
|
|
7700
|
+
logger$154.warn("Unexpected peer.auth.response on incoming connection");
|
|
7701
7701
|
break;
|
|
7702
7702
|
case "raft.vote.response":
|
|
7703
7703
|
onRaftVoteResponse?.(msg);
|
|
@@ -7729,7 +7729,7 @@ function createPeerHandler(deps) {
|
|
|
7729
7729
|
handler(msg).then((result) => {
|
|
7730
7730
|
sendResponse(result);
|
|
7731
7731
|
}).catch((err) => {
|
|
7732
|
-
logger$
|
|
7732
|
+
logger$154.error("Error executing peer config reload", {
|
|
7733
7733
|
peerId: conn.peerInstanceId,
|
|
7734
7734
|
error: toErrorMessage(err)
|
|
7735
7735
|
});
|
|
@@ -7754,7 +7754,7 @@ function createPeerHandler(deps) {
|
|
|
7754
7754
|
sendEncryptedMessage(conn.ws, conn.sessionKey, response);
|
|
7755
7755
|
};
|
|
7756
7756
|
if (conn.authenticatedRole !== "coordinator") {
|
|
7757
|
-
logger$
|
|
7757
|
+
logger$154.warn("Refused a scaler orphan request from a peer that is not a coordinator", {
|
|
7758
7758
|
peerId: conn.peerInstanceId,
|
|
7759
7759
|
role: conn.authenticatedRole,
|
|
7760
7760
|
declaredRole: peerRegistry.getPeer(conn.peerInstanceId)?.role ?? "unknown"
|
|
@@ -7778,7 +7778,7 @@ function createPeerHandler(deps) {
|
|
|
7778
7778
|
sendEncryptedMessage(conn.ws, conn.sessionKey, response);
|
|
7779
7779
|
};
|
|
7780
7780
|
if (conn.authenticatedRole !== "coordinator") {
|
|
7781
|
-
logger$
|
|
7781
|
+
logger$154.warn("Refused a scaler reload request from a peer that is not a coordinator", {
|
|
7782
7782
|
peerId: conn.peerInstanceId,
|
|
7783
7783
|
role: conn.authenticatedRole
|
|
7784
7784
|
});
|
|
@@ -7806,7 +7806,7 @@ function createPeerHandler(deps) {
|
|
|
7806
7806
|
});
|
|
7807
7807
|
};
|
|
7808
7808
|
if (conn.authenticatedRole !== "coordinator") {
|
|
7809
|
-
logger$
|
|
7809
|
+
logger$154.warn("Refused a peer forget request from a peer that is not a coordinator", {
|
|
7810
7810
|
peerId: conn.peerInstanceId,
|
|
7811
7811
|
role: conn.authenticatedRole
|
|
7812
7812
|
});
|
|
@@ -7831,7 +7831,7 @@ function createPeerHandler(deps) {
|
|
|
7831
7831
|
...resolved
|
|
7832
7832
|
});
|
|
7833
7833
|
}).catch((err) => {
|
|
7834
|
-
logger$
|
|
7834
|
+
logger$154.error("Error resolving peer cluster-settings request", {
|
|
7835
7835
|
peerId: conn.peerInstanceId,
|
|
7836
7836
|
error: toErrorMessage(err)
|
|
7837
7837
|
});
|
|
@@ -7839,7 +7839,7 @@ function createPeerHandler(deps) {
|
|
|
7839
7839
|
break;
|
|
7840
7840
|
}
|
|
7841
7841
|
case "peer.logs.collect.request":
|
|
7842
|
-
runDetached(logger$
|
|
7842
|
+
runDetached(logger$154, "Peer logs collect request", () => onLogsCollectRequest?.(msg, (out) => {
|
|
7843
7843
|
sendEncryptedMessage(conn.ws, conn.sessionKey, out);
|
|
7844
7844
|
return true;
|
|
7845
7845
|
}), {
|
|
@@ -7856,7 +7856,7 @@ function createPeerHandler(deps) {
|
|
|
7856
7856
|
function handleConnection(ws, remoteIp) {
|
|
7857
7857
|
const ip = remoteIp ?? "unknown";
|
|
7858
7858
|
if (isRateLimited(ip)) {
|
|
7859
|
-
logger$
|
|
7859
|
+
logger$154.warn("Rate limited peer connection attempt", { ip });
|
|
7860
7860
|
ws.close(WS_CLOSE_UNAUTHORIZED, "Rate limited");
|
|
7861
7861
|
return;
|
|
7862
7862
|
}
|
|
@@ -7870,7 +7870,7 @@ function createPeerHandler(deps) {
|
|
|
7870
7870
|
let ownConn = null;
|
|
7871
7871
|
const authTimer = setTimeout(() => {
|
|
7872
7872
|
if (!authenticated) {
|
|
7873
|
-
logger$
|
|
7873
|
+
logger$154.warn("Peer auth timeout, closing connection");
|
|
7874
7874
|
ws.close(WS_CLOSE_AUTH_TIMEOUT, "Auth timeout");
|
|
7875
7875
|
}
|
|
7876
7876
|
}, authTimeoutMs);
|
|
@@ -7895,7 +7895,7 @@ function createPeerHandler(deps) {
|
|
|
7895
7895
|
}
|
|
7896
7896
|
if (!authenticated) {
|
|
7897
7897
|
if (authInFlight) {
|
|
7898
|
-
logger$
|
|
7898
|
+
logger$154.warn("Peer sent a frame while its authentication was in flight", { ip });
|
|
7899
7899
|
fail(WS_CLOSE_INVALID_MESSAGE, "Frame during authentication");
|
|
7900
7900
|
return;
|
|
7901
7901
|
}
|
|
@@ -7913,7 +7913,7 @@ function createPeerHandler(deps) {
|
|
|
7913
7913
|
ownConn = registerInboundPeer(ws, authMsg, accepted);
|
|
7914
7914
|
}).catch((err) => {
|
|
7915
7915
|
authInFlight = false;
|
|
7916
|
-
logger$
|
|
7916
|
+
logger$154.error("Unexpected error during auth handling", { error: toErrorMessage(err) });
|
|
7917
7917
|
fail(WS_CLOSE_UNAUTHORIZED, "Auth error");
|
|
7918
7918
|
});
|
|
7919
7919
|
return;
|
|
@@ -7923,19 +7923,19 @@ function createPeerHandler(deps) {
|
|
|
7923
7923
|
try {
|
|
7924
7924
|
decrypted = decryptMessage(raw, sessionKey);
|
|
7925
7925
|
} catch {
|
|
7926
|
-
logger$
|
|
7926
|
+
logger$154.warn("Failed to decrypt message from authenticated peer");
|
|
7927
7927
|
return;
|
|
7928
7928
|
}
|
|
7929
7929
|
let parsed;
|
|
7930
7930
|
try {
|
|
7931
7931
|
parsed = JSON.parse(decrypted);
|
|
7932
7932
|
} catch {
|
|
7933
|
-
logger$
|
|
7933
|
+
logger$154.warn("Malformed JSON from peer connection");
|
|
7934
7934
|
return;
|
|
7935
7935
|
}
|
|
7936
7936
|
const msgResult = peerFromPeerMessageSchema.safeParse(parsed);
|
|
7937
7937
|
if (!msgResult.success) {
|
|
7938
|
-
logger$
|
|
7938
|
+
logger$154.warn("Invalid message from peer", { errors: msgResult.error.issues });
|
|
7939
7939
|
return;
|
|
7940
7940
|
}
|
|
7941
7941
|
const conn = connections.get(peerInstanceId);
|
|
@@ -7945,7 +7945,7 @@ function createPeerHandler(deps) {
|
|
|
7945
7945
|
closed = true;
|
|
7946
7946
|
clearTimeout(authTimer);
|
|
7947
7947
|
if (peerInstanceId && ownConn) {
|
|
7948
|
-
logger$
|
|
7948
|
+
logger$154.info("Peer disconnected", { peerInstanceId });
|
|
7949
7949
|
if (ownConn.heartbeatTimer) {
|
|
7950
7950
|
clearInterval(ownConn.heartbeatTimer);
|
|
7951
7951
|
ownConn.heartbeatTimer = null;
|
|
@@ -7957,7 +7957,7 @@ function createPeerHandler(deps) {
|
|
|
7957
7957
|
}
|
|
7958
7958
|
});
|
|
7959
7959
|
ws.on("error", (err) => {
|
|
7960
|
-
logger$
|
|
7960
|
+
logger$154.error("Peer connection error", { error: toErrorMessage(err) });
|
|
7961
7961
|
});
|
|
7962
7962
|
}
|
|
7963
7963
|
/** The peer's plaintext hello.response: derive K_hs and the transcript hash, or close. */
|
|
@@ -7966,12 +7966,12 @@ function createPeerHandler(deps) {
|
|
|
7966
7966
|
try {
|
|
7967
7967
|
parsed = JSON.parse(raw);
|
|
7968
7968
|
} catch {
|
|
7969
|
-
logger$
|
|
7969
|
+
logger$154.warn("Malformed JSON during ECDH handshake");
|
|
7970
7970
|
return null;
|
|
7971
7971
|
}
|
|
7972
7972
|
const helloResp = peerHelloResponseSchema.safeParse(parsed);
|
|
7973
7973
|
if (!helloResp.success) {
|
|
7974
|
-
logger$
|
|
7974
|
+
logger$154.warn("Expected peer.hello.response, got invalid message");
|
|
7975
7975
|
fail(WS_CLOSE_INVALID_MESSAGE, "Expected hello response");
|
|
7976
7976
|
return null;
|
|
7977
7977
|
}
|
|
@@ -7987,7 +7987,7 @@ function createPeerHandler(deps) {
|
|
|
7987
7987
|
})
|
|
7988
7988
|
};
|
|
7989
7989
|
} catch (err) {
|
|
7990
|
-
logger$
|
|
7990
|
+
logger$154.warn("ECDH key derivation failed", { error: toErrorMessage(err) });
|
|
7991
7991
|
fail(WS_CLOSE_INVALID_MESSAGE, "Key derivation failed");
|
|
7992
7992
|
return null;
|
|
7993
7993
|
}
|
|
@@ -8003,7 +8003,7 @@ function createPeerHandler(deps) {
|
|
|
8003
8003
|
try {
|
|
8004
8004
|
decrypted = decryptMessage(raw, hs.handshakeKey);
|
|
8005
8005
|
} catch (err) {
|
|
8006
|
-
logger$
|
|
8006
|
+
logger$154.warn("Failed to decrypt auth request", { error: toErrorMessage(err) });
|
|
8007
8007
|
recordFailedAuth(ip);
|
|
8008
8008
|
fail(WS_CLOSE_UNAUTHORIZED, "Decryption failed");
|
|
8009
8009
|
return null;
|
|
@@ -8012,13 +8012,13 @@ function createPeerHandler(deps) {
|
|
|
8012
8012
|
try {
|
|
8013
8013
|
authParsed = JSON.parse(decrypted);
|
|
8014
8014
|
} catch {
|
|
8015
|
-
logger$
|
|
8015
|
+
logger$154.warn("Malformed JSON in decrypted auth request");
|
|
8016
8016
|
fail(WS_CLOSE_INVALID_MESSAGE, "Invalid auth format");
|
|
8017
8017
|
return null;
|
|
8018
8018
|
}
|
|
8019
8019
|
if (isLegacyPeerAuthRequest(authParsed)) {
|
|
8020
8020
|
const legacyId = authParsed.instanceId;
|
|
8021
|
-
logger$
|
|
8021
|
+
logger$154.warn("Peer used an authentication scheme this release no longer accepts; upgrade every orchestrator in the cluster", {
|
|
8022
8022
|
peerInstanceId: typeof legacyId === "string" ? legacyId : null,
|
|
8023
8023
|
ip
|
|
8024
8024
|
});
|
|
@@ -8033,7 +8033,7 @@ function createPeerHandler(deps) {
|
|
|
8033
8033
|
}
|
|
8034
8034
|
const authMsg = peerAuthRequestSchema.safeParse(authParsed);
|
|
8035
8035
|
if (!authMsg.success) {
|
|
8036
|
-
logger$
|
|
8036
|
+
logger$154.warn("Invalid peer.auth.request format", { errors: authMsg.error.issues });
|
|
8037
8037
|
fail(WS_CLOSE_INVALID_MESSAGE, "Invalid auth request");
|
|
8038
8038
|
return null;
|
|
8039
8039
|
}
|
|
@@ -8042,7 +8042,7 @@ function createPeerHandler(deps) {
|
|
|
8042
8042
|
/** Register an authenticated inbound peer and start its heartbeats under K_app. */
|
|
8043
8043
|
function registerInboundPeer(ws, authMsg, accepted) {
|
|
8044
8044
|
const peerInstanceId = authMsg.instanceId;
|
|
8045
|
-
logger$
|
|
8045
|
+
logger$154.info("Peer authenticated", { peerInstanceId });
|
|
8046
8046
|
peerRegistry.addPeer({
|
|
8047
8047
|
instanceId: peerInstanceId,
|
|
8048
8048
|
connectionId: randomUUID(),
|
|
@@ -8120,7 +8120,7 @@ function createPeerHandler(deps) {
|
|
|
8120
8120
|
*/
|
|
8121
8121
|
async function handleAuth(ws, hs, authMsg, ip) {
|
|
8122
8122
|
if (authMsg.protocolVersion < MIN_PROTOCOL_VERSION) {
|
|
8123
|
-
logger$
|
|
8123
|
+
logger$154.warn("Peer protocol version below minimum", {
|
|
8124
8124
|
peerInstanceId: authMsg.instanceId,
|
|
8125
8125
|
received: authMsg.protocolVersion,
|
|
8126
8126
|
minimum: MIN_PROTOCOL_VERSION
|
|
@@ -8136,7 +8136,7 @@ function createPeerHandler(deps) {
|
|
|
8136
8136
|
ws.close(WS_CLOSE_PROTOCOL_ERROR, "Unsupported protocol version");
|
|
8137
8137
|
return null;
|
|
8138
8138
|
}
|
|
8139
|
-
if (authMsg.softwareVersion) logger$
|
|
8139
|
+
if (authMsg.softwareVersion) logger$154.info("Peer software version", {
|
|
8140
8140
|
peerInstanceId: authMsg.instanceId,
|
|
8141
8141
|
localVersion: SOFTWARE_VERSION,
|
|
8142
8142
|
remoteVersion: authMsg.softwareVersion
|
|
@@ -8161,14 +8161,14 @@ function createPeerHandler(deps) {
|
|
|
8161
8161
|
async function authenticateCredential(ws, hs, authMsg, clientProof, ip) {
|
|
8162
8162
|
const stored = await credentialStore.findByInstanceId(authMsg.instanceId);
|
|
8163
8163
|
if (!stored) {
|
|
8164
|
-
logger$
|
|
8164
|
+
logger$154.warn("Peer credential not found", {
|
|
8165
8165
|
peerInstanceId: authMsg.instanceId,
|
|
8166
8166
|
authPath: "credential-proof"
|
|
8167
8167
|
});
|
|
8168
8168
|
return rejectAuth(ws, hs, "Unknown credential", ip, authMsg.instanceId);
|
|
8169
8169
|
}
|
|
8170
8170
|
if (stored.revokedAt) {
|
|
8171
|
-
logger$
|
|
8171
|
+
logger$154.warn("Peer credential revoked", {
|
|
8172
8172
|
peerInstanceId: authMsg.instanceId,
|
|
8173
8173
|
authPath: "credential-proof",
|
|
8174
8174
|
revokedAt: stored.revokedAt.toISOString()
|
|
@@ -8177,7 +8177,7 @@ function createPeerHandler(deps) {
|
|
|
8177
8177
|
}
|
|
8178
8178
|
const psk = Buffer.from(stored.credentialHash, "hex");
|
|
8179
8179
|
if (!proofMatches$1(expectedClientProof(hs, authMsg, psk, ""), clientProof)) {
|
|
8180
|
-
logger$
|
|
8180
|
+
logger$154.warn("Peer credential proof invalid", {
|
|
8181
8181
|
peerInstanceId: authMsg.instanceId,
|
|
8182
8182
|
authPath: "credential-proof"
|
|
8183
8183
|
});
|
|
@@ -8213,7 +8213,7 @@ function createPeerHandler(deps) {
|
|
|
8213
8213
|
return proofMatches$1(expectedClientProof(hs, authMsg, Buffer.from(tokenHash, "hex"), routingB64), clientProof);
|
|
8214
8214
|
});
|
|
8215
8215
|
if (resolved.status !== ResolvedJoinTokenStatus.enum.live) {
|
|
8216
|
-
logger$
|
|
8216
|
+
logger$154.warn("Peer token proof matched no live join token", {
|
|
8217
8217
|
peerInstanceId: authMsg.instanceId,
|
|
8218
8218
|
status: resolved.status
|
|
8219
8219
|
});
|
|
@@ -8222,7 +8222,7 @@ function createPeerHandler(deps) {
|
|
|
8222
8222
|
const grant = await claimTokenGrant(resolved.tokenHash, authMsg.instanceId);
|
|
8223
8223
|
if (!grant) return rejectAuth(ws, hs, "Invalid token", ip, authMsg.instanceId);
|
|
8224
8224
|
if (!acceptedRoles.includes(grant.role)) {
|
|
8225
|
-
logger$
|
|
8225
|
+
logger$154.warn("Peer role mismatch", {
|
|
8226
8226
|
peerInstanceId: authMsg.instanceId,
|
|
8227
8227
|
accepted: acceptedRoles,
|
|
8228
8228
|
actual: grant.role,
|
|
@@ -8238,7 +8238,7 @@ function createPeerHandler(deps) {
|
|
|
8238
8238
|
routingKeys: [grant.routingKey],
|
|
8239
8239
|
sourceTokenHash: resolved.tokenHash
|
|
8240
8240
|
});
|
|
8241
|
-
logger$
|
|
8241
|
+
logger$154.info(grant.trigger === "token-join" ? "Peer credential issued via token join" : "Peer idempotent token retry accepted", {
|
|
8242
8242
|
peerInstanceId: authMsg.instanceId,
|
|
8243
8243
|
trigger: grant.trigger,
|
|
8244
8244
|
tokenFingerprint: tokenFingerprint(resolved.tokenHash),
|
|
@@ -8266,7 +8266,7 @@ function createPeerHandler(deps) {
|
|
|
8266
8266
|
};
|
|
8267
8267
|
} catch (err) {
|
|
8268
8268
|
if (!isTokenAlreadyUsedError(err)) {
|
|
8269
|
-
logger$
|
|
8269
|
+
logger$154.warn("Peer token validation failed", {
|
|
8270
8270
|
peerInstanceId,
|
|
8271
8271
|
tokenFingerprint: tokenFingerprint(tokenHash),
|
|
8272
8272
|
error: toErrorMessage(err)
|
|
@@ -8276,7 +8276,7 @@ function createPeerHandler(deps) {
|
|
|
8276
8276
|
}
|
|
8277
8277
|
const existing = await credentialStore.findByInstanceId(peerInstanceId);
|
|
8278
8278
|
if (!existing || existing.revokedAt || existing.sourceTokenHash !== tokenHash) {
|
|
8279
|
-
logger$
|
|
8279
|
+
logger$154.warn("Peer token was already used by another instance", {
|
|
8280
8280
|
peerInstanceId,
|
|
8281
8281
|
tokenFingerprint: tokenFingerprint(tokenHash)
|
|
8282
8282
|
});
|
|
@@ -8440,7 +8440,7 @@ function createPeerHandler(deps) {
|
|
|
8440
8440
|
closeAllInbound
|
|
8441
8441
|
};
|
|
8442
8442
|
}
|
|
8443
|
-
var logger$
|
|
8443
|
+
var logger$154, SOFTWARE_VERSION, ADVERTISED_PEER_AUTH_SCHEMES, MAX_TOKEN_CANDIDATES, RATE_LIMIT_MAX, RATE_LIMIT_WINDOW_MS;
|
|
8444
8444
|
var init_peer_handler = __esmMin((() => {
|
|
8445
8445
|
init_peer_registry();
|
|
8446
8446
|
init_peer_forget();
|
|
@@ -8449,8 +8449,8 @@ var init_peer_handler = __esmMin((() => {
|
|
|
8449
8449
|
init_peer_crypto();
|
|
8450
8450
|
init_join_token();
|
|
8451
8451
|
init_run_detached();
|
|
8452
|
-
logger$
|
|
8453
|
-
SOFTWARE_VERSION = "0.
|
|
8452
|
+
logger$154 = createLogger({ prefix: "peer-handler" });
|
|
8453
|
+
SOFTWARE_VERSION = "0.20.0";
|
|
8454
8454
|
ADVERTISED_PEER_AUTH_SCHEMES = [PeerAuthScheme.enum["mutual-v2"]];
|
|
8455
8455
|
MAX_TOKEN_CANDIDATES = 8;
|
|
8456
8456
|
RATE_LIMIT_MAX = 5;
|
|
@@ -8478,9 +8478,9 @@ var init_peer_handler = __esmMin((() => {
|
|
|
8478
8478
|
* heartbeat (peer.heartbeat every 30s). Raft heartbeat is purely for leader
|
|
8479
8479
|
* liveness detection.
|
|
8480
8480
|
*/
|
|
8481
|
-
var logger$
|
|
8481
|
+
var logger$153, RaftNode;
|
|
8482
8482
|
var init_raft = __esmMin((() => {
|
|
8483
|
-
logger$
|
|
8483
|
+
logger$153 = createLogger({ prefix: "raft" });
|
|
8484
8484
|
RaftNode = class {
|
|
8485
8485
|
instanceId;
|
|
8486
8486
|
stateStore;
|
|
@@ -8534,7 +8534,7 @@ var init_raft = __esmMin((() => {
|
|
|
8534
8534
|
this.role = "follower";
|
|
8535
8535
|
this.stopped = false;
|
|
8536
8536
|
this.startedAt = Date.now();
|
|
8537
|
-
logger$
|
|
8537
|
+
logger$153.info("Raft node started", {
|
|
8538
8538
|
instanceId: this.instanceId,
|
|
8539
8539
|
currentTerm: this.currentTerm,
|
|
8540
8540
|
leaderId: this.leaderId
|
|
@@ -8562,7 +8562,7 @@ var init_raft = __esmMin((() => {
|
|
|
8562
8562
|
votedFor: this.votedFor,
|
|
8563
8563
|
leaderId: this.leaderId
|
|
8564
8564
|
});
|
|
8565
|
-
logger$
|
|
8565
|
+
logger$153.info("Raft node stopped", {
|
|
8566
8566
|
instanceId: this.instanceId,
|
|
8567
8567
|
wasLeader
|
|
8568
8568
|
});
|
|
@@ -8588,11 +8588,11 @@ var init_raft = __esmMin((() => {
|
|
|
8588
8588
|
handlePeerLeaving(instanceId) {
|
|
8589
8589
|
if (this.stopped) return;
|
|
8590
8590
|
if (this.role === "leader") {
|
|
8591
|
-
logger$
|
|
8591
|
+
logger$153.debug("Ignoring peer.leaving (we are leader)", { leavingPeer: instanceId });
|
|
8592
8592
|
return;
|
|
8593
8593
|
}
|
|
8594
8594
|
if (instanceId === this.leaderId) {
|
|
8595
|
-
logger$
|
|
8595
|
+
logger$153.info("Current leader is leaving, starting immediate election", {
|
|
8596
8596
|
leavingPeer: instanceId,
|
|
8597
8597
|
term: this.currentTerm
|
|
8598
8598
|
});
|
|
@@ -8600,7 +8600,7 @@ var init_raft = __esmMin((() => {
|
|
|
8600
8600
|
this.consecutiveUnansweredElections = 0;
|
|
8601
8601
|
this.clearElectionTimer();
|
|
8602
8602
|
this.startElection();
|
|
8603
|
-
} else logger$
|
|
8603
|
+
} else logger$153.debug("Non-leader peer leaving", { leavingPeer: instanceId });
|
|
8604
8604
|
}
|
|
8605
8605
|
/** Restart the election timer with a randomized timeout. */
|
|
8606
8606
|
resetElectionTimer() {
|
|
@@ -8647,7 +8647,7 @@ var init_raft = __esmMin((() => {
|
|
|
8647
8647
|
this.votedFor = this.instanceId;
|
|
8648
8648
|
this.role = "candidate";
|
|
8649
8649
|
this.votesReceived = /* @__PURE__ */ new Set([this.instanceId]);
|
|
8650
|
-
logger$
|
|
8650
|
+
logger$153.info("Starting election", {
|
|
8651
8651
|
instanceId: this.instanceId,
|
|
8652
8652
|
term: this.currentTerm
|
|
8653
8653
|
});
|
|
@@ -8656,12 +8656,12 @@ var init_raft = __esmMin((() => {
|
|
|
8656
8656
|
votedFor: this.votedFor,
|
|
8657
8657
|
leaderId: this.leaderId
|
|
8658
8658
|
}).catch((err) => {
|
|
8659
|
-
logger$
|
|
8659
|
+
logger$153.error("Failed to persist election state", { error: toErrorMessage(err) });
|
|
8660
8660
|
});
|
|
8661
8661
|
const connectedPeerCount = this.peerRegistry.getConnectedCoordinatorPeerCount();
|
|
8662
8662
|
if (connectedPeerCount === 0) {
|
|
8663
8663
|
if (this.gracePeriodMs > 0 && Date.now() - this.startedAt < this.gracePeriodMs) {
|
|
8664
|
-
logger$
|
|
8664
|
+
logger$153.info("Dormant mode: deferring self-election (grace period active)", {
|
|
8665
8665
|
instanceId: this.instanceId,
|
|
8666
8666
|
elapsedMs: Date.now() - this.startedAt,
|
|
8667
8667
|
gracePeriodMs: this.gracePeriodMs
|
|
@@ -8669,7 +8669,7 @@ var init_raft = __esmMin((() => {
|
|
|
8669
8669
|
this.resetElectionTimer();
|
|
8670
8670
|
return;
|
|
8671
8671
|
}
|
|
8672
|
-
logger$
|
|
8672
|
+
logger$153.info("No peers connected, self-electing as leader", {
|
|
8673
8673
|
instanceId: this.instanceId,
|
|
8674
8674
|
term: this.currentTerm
|
|
8675
8675
|
});
|
|
@@ -8677,7 +8677,7 @@ var init_raft = __esmMin((() => {
|
|
|
8677
8677
|
return;
|
|
8678
8678
|
}
|
|
8679
8679
|
if (this.consecutiveUnansweredElections >= 2) {
|
|
8680
|
-
logger$
|
|
8680
|
+
logger$153.warn("No vote responses after consecutive elections, forcing self-election", {
|
|
8681
8681
|
instanceId: this.instanceId,
|
|
8682
8682
|
term: this.currentTerm,
|
|
8683
8683
|
consecutiveUnanswered: this.consecutiveUnansweredElections,
|
|
@@ -8715,14 +8715,14 @@ var init_raft = __esmMin((() => {
|
|
|
8715
8715
|
votedFor: this.votedFor,
|
|
8716
8716
|
leaderId: this.leaderId
|
|
8717
8717
|
}).catch((err) => {
|
|
8718
|
-
logger$
|
|
8718
|
+
logger$153.error("Failed to persist vote", { error: toErrorMessage(err) });
|
|
8719
8719
|
});
|
|
8720
8720
|
this.resetElectionTimer();
|
|
8721
|
-
logger$
|
|
8721
|
+
logger$153.info("Vote granted", {
|
|
8722
8722
|
candidateId: msg.candidateId,
|
|
8723
8723
|
term: msg.term
|
|
8724
8724
|
});
|
|
8725
|
-
} else logger$
|
|
8725
|
+
} else logger$153.info("Vote denied", {
|
|
8726
8726
|
candidateId: msg.candidateId,
|
|
8727
8727
|
term: msg.term,
|
|
8728
8728
|
currentTerm: this.currentTerm,
|
|
@@ -8745,7 +8745,7 @@ var init_raft = __esmMin((() => {
|
|
|
8745
8745
|
this.consecutiveUnansweredElections = 0;
|
|
8746
8746
|
if (msg.voteGranted) {
|
|
8747
8747
|
this.votesReceived.add(msg.voterId);
|
|
8748
|
-
logger$
|
|
8748
|
+
logger$153.info("Vote received", {
|
|
8749
8749
|
from: msg.voterId,
|
|
8750
8750
|
term: msg.term,
|
|
8751
8751
|
totalVotes: this.votesReceived.size
|
|
@@ -8766,7 +8766,7 @@ var init_raft = __esmMin((() => {
|
|
|
8766
8766
|
this.leaderId = this.instanceId;
|
|
8767
8767
|
this.consecutiveUnansweredElections = 0;
|
|
8768
8768
|
this.clearElectionTimer();
|
|
8769
|
-
logger$
|
|
8769
|
+
logger$153.info("Became leader", {
|
|
8770
8770
|
instanceId: this.instanceId,
|
|
8771
8771
|
term: this.currentTerm
|
|
8772
8772
|
});
|
|
@@ -8776,7 +8776,7 @@ var init_raft = __esmMin((() => {
|
|
|
8776
8776
|
votedFor: this.votedFor,
|
|
8777
8777
|
leaderId: this.leaderId
|
|
8778
8778
|
}).catch((err) => {
|
|
8779
|
-
logger$
|
|
8779
|
+
logger$153.error("Failed to persist leader state", { error: toErrorMessage(err) });
|
|
8780
8780
|
});
|
|
8781
8781
|
this.onBecomeLeader();
|
|
8782
8782
|
}
|
|
@@ -8795,16 +8795,16 @@ var init_raft = __esmMin((() => {
|
|
|
8795
8795
|
votedFor: this.votedFor,
|
|
8796
8796
|
leaderId: this.leaderId
|
|
8797
8797
|
}).catch((err) => {
|
|
8798
|
-
logger$
|
|
8798
|
+
logger$153.error("Failed to persist step-down state", { error: toErrorMessage(err) });
|
|
8799
8799
|
});
|
|
8800
8800
|
if (wasLeader) {
|
|
8801
|
-
logger$
|
|
8801
|
+
logger$153.info("Lost leadership", {
|
|
8802
8802
|
instanceId: this.instanceId,
|
|
8803
8803
|
newTerm,
|
|
8804
8804
|
newLeaderId
|
|
8805
8805
|
});
|
|
8806
8806
|
this.onLoseLeadership();
|
|
8807
|
-
} else logger$
|
|
8807
|
+
} else logger$153.info("Stepped down", {
|
|
8808
8808
|
instanceId: this.instanceId,
|
|
8809
8809
|
newTerm,
|
|
8810
8810
|
newLeaderId
|
|
@@ -8938,9 +8938,9 @@ function shouldDeferReroutedJob(job, peerRegistry, opts) {
|
|
|
8938
8938
|
const flapGraceMs = opts?.flapGraceMs ?? 12e4;
|
|
8939
8939
|
return nowMs - peer.lastHeartbeatAt <= flapGraceMs;
|
|
8940
8940
|
}
|
|
8941
|
-
var init_rerouted_job_guard = __esmMin((() => {})), logger$
|
|
8941
|
+
var init_rerouted_job_guard = __esmMin((() => {})), logger$152, ConcurrencyQueueManager;
|
|
8942
8942
|
var init_queue_manager = __esmMin((() => {
|
|
8943
|
-
logger$
|
|
8943
|
+
logger$152 = createLogger({ prefix: "concurrency-queue" });
|
|
8944
8944
|
ConcurrencyQueueManager = class ConcurrencyQueueManager {
|
|
8945
8945
|
db;
|
|
8946
8946
|
constructor(db) {
|
|
@@ -9009,7 +9009,7 @@ var init_queue_manager = __esmMin((() => {
|
|
|
9009
9009
|
status: "completed",
|
|
9010
9010
|
completed_at: /* @__PURE__ */ new Date()
|
|
9011
9011
|
}).where("id", "in", leaked.map((r) => r.id)).execute();
|
|
9012
|
-
logger$
|
|
9012
|
+
logger$152.info("Released concurrency slots held by finished runs", {
|
|
9013
9013
|
groupKey,
|
|
9014
9014
|
routingKey,
|
|
9015
9015
|
released: leaked.length
|
|
@@ -9024,7 +9024,7 @@ var init_queue_manager = __esmMin((() => {
|
|
|
9024
9024
|
routing_key: opts.routingKey,
|
|
9025
9025
|
status: "queued"
|
|
9026
9026
|
}).execute();
|
|
9027
|
-
logger$
|
|
9027
|
+
logger$152.info("Job enqueued for concurrency group", {
|
|
9028
9028
|
groupKey: opts.groupKey,
|
|
9029
9029
|
routingKey: opts.routingKey,
|
|
9030
9030
|
runId: opts.runId,
|
|
@@ -9058,7 +9058,7 @@ var init_queue_manager = __esmMin((() => {
|
|
|
9058
9058
|
status: "cancelled",
|
|
9059
9059
|
completed_at: /* @__PURE__ */ new Date()
|
|
9060
9060
|
}).where("run_id", "=", runId).where("status", "=", "queued").execute();
|
|
9061
|
-
logger$
|
|
9061
|
+
logger$152.info("Cancelled queued concurrency entries", { runId });
|
|
9062
9062
|
}
|
|
9063
9063
|
/** Called when a job finishes (success/failed/cancelled). */
|
|
9064
9064
|
async markCompleted(runId, groupKey, routingKey) {
|
|
@@ -9085,11 +9085,11 @@ var init_queue_manager = __esmMin((() => {
|
|
|
9085
9085
|
}));
|
|
9086
9086
|
//#endregion
|
|
9087
9087
|
//#region src/cluster/orphan-recovery.ts
|
|
9088
|
-
var logger$
|
|
9088
|
+
var logger$151, DEFAULT_STALE_THRESHOLD_MS, DEFAULT_JOB_STUCK_THRESHOLD_MS, OrphanRecovery;
|
|
9089
9089
|
var init_orphan_recovery = __esmMin((() => {
|
|
9090
9090
|
init_rerouted_job_guard();
|
|
9091
9091
|
init_queue_manager();
|
|
9092
|
-
logger$
|
|
9092
|
+
logger$151 = createLogger({ prefix: "orphan-recovery" });
|
|
9093
9093
|
DEFAULT_STALE_THRESHOLD_MS = 3e5;
|
|
9094
9094
|
DEFAULT_JOB_STUCK_THRESHOLD_MS = 18e4;
|
|
9095
9095
|
OrphanRecovery = class {
|
|
@@ -9118,14 +9118,14 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
9118
9118
|
start() {
|
|
9119
9119
|
if (this.scanTimer) return;
|
|
9120
9120
|
this.releaseLeakedConcurrencySlots().catch((err) => {
|
|
9121
|
-
logger$
|
|
9121
|
+
logger$151.error("Concurrency slot reconcile failed", { error: toErrorMessage(err) });
|
|
9122
9122
|
});
|
|
9123
9123
|
this.scanTimer = setInterval(() => {
|
|
9124
9124
|
this.scanForOrphans().catch((err) => {
|
|
9125
|
-
logger$
|
|
9125
|
+
logger$151.error("Orphan recovery scan failed", { error: toErrorMessage(err) });
|
|
9126
9126
|
});
|
|
9127
9127
|
}, this.scanIntervalMs);
|
|
9128
|
-
logger$
|
|
9128
|
+
logger$151.info("Orphan recovery started", {
|
|
9129
9129
|
scanIntervalMs: this.scanIntervalMs,
|
|
9130
9130
|
staleThresholdMs: this.staleThresholdMs
|
|
9131
9131
|
});
|
|
@@ -9135,7 +9135,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
9135
9135
|
clearInterval(this.scanTimer);
|
|
9136
9136
|
this.scanTimer = null;
|
|
9137
9137
|
}
|
|
9138
|
-
logger$
|
|
9138
|
+
logger$151.info("Orphan recovery stopped");
|
|
9139
9139
|
}
|
|
9140
9140
|
async scanForOrphans() {
|
|
9141
9141
|
if (!this.raft.isLeader()) return;
|
|
@@ -9149,7 +9149,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
9149
9149
|
"sha"
|
|
9150
9150
|
]).where("status", "=", ExecutionRunStatus.enum.running).where("started_at", "<", staleThreshold).execute();
|
|
9151
9151
|
if (staleRuns.length === 0) return;
|
|
9152
|
-
logger$
|
|
9152
|
+
logger$151.info("Found potentially orphaned runs", { count: staleRuns.length });
|
|
9153
9153
|
for (const run of staleRuns) await this.recoverRun(run);
|
|
9154
9154
|
}
|
|
9155
9155
|
/**
|
|
@@ -9170,15 +9170,15 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
9170
9170
|
status: "completed",
|
|
9171
9171
|
completed_at: /* @__PURE__ */ new Date()
|
|
9172
9172
|
}).where("id", "in", leaked.map((r) => r.id)).execute();
|
|
9173
|
-
logger$
|
|
9173
|
+
logger$151.info("Released concurrency slots held by finished runs", { released: leaked.length });
|
|
9174
9174
|
}
|
|
9175
9175
|
async recoverRun(run) {
|
|
9176
9176
|
if (this.isCoordinatorStillConnected(run.routing_key)) return;
|
|
9177
9177
|
if (await this.hasRecentProgress(run.run_id)) {
|
|
9178
|
-
logger$
|
|
9178
|
+
logger$151.debug("Skipping orphan recovery: run is still making progress", { runId: run.run_id });
|
|
9179
9179
|
return;
|
|
9180
9180
|
}
|
|
9181
|
-
logger$
|
|
9181
|
+
logger$151.info("Recovering orphan run", {
|
|
9182
9182
|
runId: run.run_id,
|
|
9183
9183
|
routingKey: run.routing_key,
|
|
9184
9184
|
workflowName: run.workflow_name
|
|
@@ -9196,7 +9196,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
9196
9196
|
completed_at: /* @__PURE__ */ new Date(),
|
|
9197
9197
|
failure_reason: "No jobs found for orphaned run"
|
|
9198
9198
|
}).where("run_id", "=", run.run_id).where("status", "=", ExecutionRunStatus.enum.running).execute();
|
|
9199
|
-
logger$
|
|
9199
|
+
logger$151.info("Finalized orphan run with no jobs", { runId: run.run_id });
|
|
9200
9200
|
this.executionTracker.emitInfraEvent(run.run_id, "orchestrator.run.orphan_recovered", { metadata: {
|
|
9201
9201
|
routingKey: run.routing_key,
|
|
9202
9202
|
reason: "No jobs found"
|
|
@@ -9208,7 +9208,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
9208
9208
|
if (TERMINAL_JOB_STATES.has(job.status)) continue;
|
|
9209
9209
|
const flapGraceMs = await this.clusterSettings.getNumber("reroute_flap_grace_ms", this.rerouteFlapGraceFallbackMs);
|
|
9210
9210
|
if (shouldDeferReroutedJob(job, this.peerRegistry, { flapGraceMs })) {
|
|
9211
|
-
logger$
|
|
9211
|
+
logger$151.info("Deferring orphan-fail for rerouted job; worker peer connected or recently seen", {
|
|
9212
9212
|
runId: run.run_id,
|
|
9213
9213
|
jobId: job.job_id,
|
|
9214
9214
|
peer: job.rerouted_to_peer
|
|
@@ -9232,7 +9232,7 @@ var init_orphan_recovery = __esmMin((() => {
|
|
|
9232
9232
|
reason: errorMsg
|
|
9233
9233
|
}
|
|
9234
9234
|
});
|
|
9235
|
-
logger$
|
|
9235
|
+
logger$151.info("Marked stuck job as failed", {
|
|
9236
9236
|
runId: run.run_id,
|
|
9237
9237
|
jobId: job.job_id,
|
|
9238
9238
|
jobName: job.job_name
|
|
@@ -9374,10 +9374,10 @@ function agentSatisfiesMandatoryLabels(agent, requiredLabels) {
|
|
|
9374
9374
|
const required = new Set(requiredLabels);
|
|
9375
9375
|
return mandatory.every((m) => required.has(m));
|
|
9376
9376
|
}
|
|
9377
|
-
var logger$
|
|
9377
|
+
var logger$150, DEFAULT_ACK_TIMEOUT_MS, DEFAULT_MAX_HOPS, DEFAULT_REROUTE_SPAWN_WINDOW_MS, DEFAULT_REROUTE_SPAWN_RETRY, NAK_BACKOFF_BASE_MS, NAK_BACKOFF_MAX_MS, RunCoordinator;
|
|
9378
9378
|
var init_coordinator = __esmMin((() => {
|
|
9379
9379
|
init_worker_eviction();
|
|
9380
|
-
logger$
|
|
9380
|
+
logger$150 = createLogger({ prefix: "coordinator" });
|
|
9381
9381
|
DEFAULT_ACK_TIMEOUT_MS = 15e3;
|
|
9382
9382
|
DEFAULT_MAX_HOPS = 3;
|
|
9383
9383
|
DEFAULT_REROUTE_SPAWN_WINDOW_MS = 9e4;
|
|
@@ -9442,7 +9442,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9442
9442
|
this.closePeerViaHandler?.(id, code, reason);
|
|
9443
9443
|
}
|
|
9444
9444
|
};
|
|
9445
|
-
this.workerEviction = new WorkerEviction(this.peerRegistry, evictionHooks, logger$
|
|
9445
|
+
this.workerEviction = new WorkerEviction(this.peerRegistry, evictionHooks, logger$150);
|
|
9446
9446
|
}
|
|
9447
9447
|
/** In-flight rerouted jobs currently attributed to a peer. */
|
|
9448
9448
|
runningJobCountForPeer(instanceId) {
|
|
@@ -9538,7 +9538,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9538
9538
|
}));
|
|
9539
9539
|
await Promise.all(reroutePromises);
|
|
9540
9540
|
}
|
|
9541
|
-
logger$
|
|
9541
|
+
logger$150.info("Jobs routed", {
|
|
9542
9542
|
runId: runContext.runId,
|
|
9543
9543
|
local: result.localJobs.length,
|
|
9544
9544
|
rerouted: result.reroutedJobs.length,
|
|
@@ -9601,7 +9601,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9601
9601
|
runId: msg.runId
|
|
9602
9602
|
});
|
|
9603
9603
|
} catch (err) {
|
|
9604
|
-
logger$
|
|
9604
|
+
logger$150.warn("Failed to create check runs for rerouted job", {
|
|
9605
9605
|
runId: msg.runId,
|
|
9606
9606
|
error: toErrorMessage(err)
|
|
9607
9607
|
});
|
|
@@ -9619,7 +9619,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9619
9619
|
jobName: msg.jobName
|
|
9620
9620
|
}], msg.routingKey);
|
|
9621
9621
|
} catch (err) {
|
|
9622
|
-
logger$
|
|
9622
|
+
logger$150.warn("Failed to register rerouted execution for tracking", {
|
|
9623
9623
|
runId: msg.runId,
|
|
9624
9624
|
error: toErrorMessage(err)
|
|
9625
9625
|
});
|
|
@@ -9642,7 +9642,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9642
9642
|
tracked.windowTimer = void 0;
|
|
9643
9643
|
}
|
|
9644
9644
|
if (isSupersededTerminal) {
|
|
9645
|
-
logger$
|
|
9645
|
+
logger$150.warn("Ignoring stray terminal job update from superseded peer", {
|
|
9646
9646
|
runId: msg.runId,
|
|
9647
9647
|
jobId: msg.jobId,
|
|
9648
9648
|
state: msg.state,
|
|
@@ -9665,7 +9665,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9665
9665
|
state: msg.state
|
|
9666
9666
|
});
|
|
9667
9667
|
}).catch((err) => {
|
|
9668
|
-
logger$
|
|
9668
|
+
logger$150.error("Failed to track peer job progress", {
|
|
9669
9669
|
error: toErrorMessage(err),
|
|
9670
9670
|
runId: msg.runId,
|
|
9671
9671
|
jobId: msg.jobId,
|
|
@@ -9683,7 +9683,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9683
9683
|
if (drainedPeerId) this.fireIdleIfDrained(drainedPeerId);
|
|
9684
9684
|
}
|
|
9685
9685
|
}
|
|
9686
|
-
logger$
|
|
9686
|
+
logger$150.debug("Peer job progress", {
|
|
9687
9687
|
runId: msg.runId,
|
|
9688
9688
|
jobId: msg.jobId,
|
|
9689
9689
|
kind: msg.kind,
|
|
@@ -9740,7 +9740,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9740
9740
|
onRelayedSpawnFailure(msg, tracked) {
|
|
9741
9741
|
const { runId, jobId } = msg;
|
|
9742
9742
|
if (msg.final === void 0) {
|
|
9743
|
-
logger$
|
|
9743
|
+
logger$150.info("Worker relayed scaler.failed without a retry verdict — waiting out the spawn window", {
|
|
9744
9744
|
runId,
|
|
9745
9745
|
jobId,
|
|
9746
9746
|
detail: msg.detail
|
|
@@ -9750,7 +9750,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9750
9750
|
if (!msg.final) {
|
|
9751
9751
|
tracked.spawnFailures += 1;
|
|
9752
9752
|
if (tracked.spawnFailures < tracked.spawnRetry.maxAttempts) {
|
|
9753
|
-
logger$
|
|
9753
|
+
logger$150.info("Worker retrying spawn for a rerouted job — spawn window re-armed", {
|
|
9754
9754
|
runId,
|
|
9755
9755
|
jobId,
|
|
9756
9756
|
attempt: tracked.spawnFailures,
|
|
@@ -9758,7 +9758,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9758
9758
|
detail: msg.detail
|
|
9759
9759
|
});
|
|
9760
9760
|
this.rearmAfterSpawnRetry(runId, jobId, tracked).catch((err) => {
|
|
9761
|
-
logger$
|
|
9761
|
+
logger$150.error("Reroute spawn-window re-arm failed", {
|
|
9762
9762
|
runId,
|
|
9763
9763
|
jobId,
|
|
9764
9764
|
error: toErrorMessage(err)
|
|
@@ -9767,13 +9767,13 @@ var init_coordinator = __esmMin((() => {
|
|
|
9767
9767
|
return;
|
|
9768
9768
|
}
|
|
9769
9769
|
}
|
|
9770
|
-
logger$
|
|
9770
|
+
logger$150.warn("Worker exhausted spawn retries for a rerouted job — re-dispatching", {
|
|
9771
9771
|
runId,
|
|
9772
9772
|
jobId,
|
|
9773
9773
|
detail: msg.detail
|
|
9774
9774
|
});
|
|
9775
9775
|
this.handleRerouteSpawnTimeout(runId, jobId).catch((err) => {
|
|
9776
|
-
logger$
|
|
9776
|
+
logger$150.error("Reroute exhausted-retries handler failed", {
|
|
9777
9777
|
runId,
|
|
9778
9778
|
jobId,
|
|
9779
9779
|
error: toErrorMessage(err)
|
|
@@ -9794,7 +9794,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9794
9794
|
*/
|
|
9795
9795
|
onPeerJobComplete(runId, jobId, status, timestamp, data) {
|
|
9796
9796
|
if (this.executionTracker) this.executionTracker.onJobStatus(runId, jobId, status, timestamp, void 0, data).catch((err) => {
|
|
9797
|
-
logger$
|
|
9797
|
+
logger$150.error("Failed to track peer job completion", {
|
|
9798
9798
|
error: toErrorMessage(err),
|
|
9799
9799
|
runId,
|
|
9800
9800
|
jobId
|
|
@@ -9835,13 +9835,13 @@ var init_coordinator = __esmMin((() => {
|
|
|
9835
9835
|
jobId,
|
|
9836
9836
|
reason
|
|
9837
9837
|
};
|
|
9838
|
-
if (!this.sendToPeer(peerId, cancelMsg)) logger$
|
|
9838
|
+
if (!this.sendToPeer(peerId, cancelMsg)) logger$150.warn("Failed to send cancel to peer", {
|
|
9839
9839
|
peerId,
|
|
9840
9840
|
runId,
|
|
9841
9841
|
jobId
|
|
9842
9842
|
});
|
|
9843
9843
|
}
|
|
9844
|
-
logger$
|
|
9844
|
+
logger$150.info("Cancel propagated to peers", {
|
|
9845
9845
|
runId,
|
|
9846
9846
|
reason,
|
|
9847
9847
|
peerCount: peerJobs.size
|
|
@@ -9855,7 +9855,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9855
9855
|
if (this.staleEvictionTimer) clearInterval(this.staleEvictionTimer);
|
|
9856
9856
|
this.staleEvictionTimer = setInterval(() => {
|
|
9857
9857
|
const evicted = this.peerRegistry.evictStalePeers(staleTimeoutMs);
|
|
9858
|
-
if (evicted.length > 0) logger$
|
|
9858
|
+
if (evicted.length > 0) logger$150.warn("Evicted stale peers", {
|
|
9859
9859
|
evicted,
|
|
9860
9860
|
count: evicted.length
|
|
9861
9861
|
});
|
|
@@ -9899,7 +9899,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9899
9899
|
if (triedConnections.includes(peer.instanceId)) continue;
|
|
9900
9900
|
const nakEntry = this.nakTracker.get(peer.instanceId);
|
|
9901
9901
|
if (nakEntry && nakEntry.backoffUntil > now) {
|
|
9902
|
-
logger$
|
|
9902
|
+
logger$150.debug("Skipping peer in NAK backoff", {
|
|
9903
9903
|
peerId: peer.instanceId,
|
|
9904
9904
|
jobName: job.jobName,
|
|
9905
9905
|
backoffRemainingMs: nakEntry.backoffUntil - now
|
|
@@ -9908,7 +9908,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9908
9908
|
}
|
|
9909
9909
|
const transport = this.rerouteTransport(peer.instanceId);
|
|
9910
9910
|
if (transport === null) {
|
|
9911
|
-
logger$
|
|
9911
|
+
logger$150.warn("No link to peer (found in registry but no connection)", {
|
|
9912
9912
|
peerId: peer.instanceId,
|
|
9913
9913
|
jobName: job.jobName
|
|
9914
9914
|
});
|
|
@@ -9926,7 +9926,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9926
9926
|
triedConnections,
|
|
9927
9927
|
spawnRetry
|
|
9928
9928
|
});
|
|
9929
|
-
logger$
|
|
9929
|
+
logger$150.info("Job rerouted to peer", {
|
|
9930
9930
|
runId: runContext.runId,
|
|
9931
9931
|
jobName: job.jobName,
|
|
9932
9932
|
jobId,
|
|
@@ -9944,7 +9944,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9944
9944
|
count: nakCount,
|
|
9945
9945
|
backoffUntil: Date.now() + backoffMs
|
|
9946
9946
|
});
|
|
9947
|
-
logger$
|
|
9947
|
+
logger$150.warn("Peer NAKed job", {
|
|
9948
9948
|
peerId: peer.instanceId,
|
|
9949
9949
|
jobName: job.jobName,
|
|
9950
9950
|
nakCount,
|
|
@@ -9961,7 +9961,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
9961
9961
|
noCapacityResult(job, labelSets) {
|
|
9962
9962
|
const peersWithLabels = this.peerRegistry.findPeersWithLabels(labelSets);
|
|
9963
9963
|
const allPeers = this.peerRegistry.getConnectedPeers();
|
|
9964
|
-
logger$
|
|
9964
|
+
logger$150.debug("Reroute failed — peer registry state", {
|
|
9965
9965
|
jobName: job.jobName,
|
|
9966
9966
|
requiredLabels: labelSets,
|
|
9967
9967
|
connectedPeers: allPeers.map((p) => ({
|
|
@@ -10084,7 +10084,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10084
10084
|
armRerouteWindow(runId, jobId, windowMs) {
|
|
10085
10085
|
return setTimeout(() => {
|
|
10086
10086
|
this.handleRerouteSpawnTimeout(runId, jobId).catch((err) => {
|
|
10087
|
-
logger$
|
|
10087
|
+
logger$150.error("Reroute spawn-window handler failed", {
|
|
10088
10088
|
runId,
|
|
10089
10089
|
jobId,
|
|
10090
10090
|
error: toErrorMessage(err)
|
|
@@ -10106,7 +10106,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10106
10106
|
try {
|
|
10107
10107
|
return await this.executionTracker.hasJobStarted(runId, jobId);
|
|
10108
10108
|
} catch (err) {
|
|
10109
|
-
logger$
|
|
10109
|
+
logger$150.warn("Could not read rerouted job state; treating the spawn window as elapsed", {
|
|
10110
10110
|
runId,
|
|
10111
10111
|
jobId,
|
|
10112
10112
|
peerId: tracked.peerId,
|
|
@@ -10133,7 +10133,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10133
10133
|
try {
|
|
10134
10134
|
terminal = await this.executionTracker?.isJobTerminal(runId, jobId) ?? false;
|
|
10135
10135
|
} catch (err) {
|
|
10136
|
-
logger$
|
|
10136
|
+
logger$150.warn("Could not read rerouted job state; keeping the tracking entry", {
|
|
10137
10137
|
runId,
|
|
10138
10138
|
jobId,
|
|
10139
10139
|
peerId: tracked.peerId,
|
|
@@ -10143,7 +10143,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10143
10143
|
if (terminal) {
|
|
10144
10144
|
runJobs.delete(jobId);
|
|
10145
10145
|
if (runJobs.size === 0) this.reroutedJobs.delete(runId);
|
|
10146
|
-
logger$
|
|
10146
|
+
logger$150.info("Rerouted job reached a terminal state on its peer — tracking released", {
|
|
10147
10147
|
runId,
|
|
10148
10148
|
jobId,
|
|
10149
10149
|
jobName: tracked.jobName,
|
|
@@ -10153,7 +10153,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10153
10153
|
return;
|
|
10154
10154
|
}
|
|
10155
10155
|
tracked.windowTimer = this.armRerouteWindow(runId, jobId, await this.getRerouteSpawnWindowMs(tracked.job));
|
|
10156
|
-
logger$
|
|
10156
|
+
logger$150.info("Rerouted job is running on its peer — spawn-window backstop disarmed", {
|
|
10157
10157
|
runId,
|
|
10158
10158
|
jobId,
|
|
10159
10159
|
jobName: tracked.jobName,
|
|
@@ -10181,7 +10181,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10181
10181
|
if (tracked.windowTimer) clearTimeout(tracked.windowTimer);
|
|
10182
10182
|
runJobs.delete(jobId);
|
|
10183
10183
|
if (runJobs.size === 0) this.reroutedJobs.delete(runId);
|
|
10184
|
-
logger$
|
|
10184
|
+
logger$150.warn("Rerouted job produced no progress — re-dispatching", {
|
|
10185
10185
|
runId,
|
|
10186
10186
|
jobId,
|
|
10187
10187
|
jobName: tracked.jobName,
|
|
@@ -10195,7 +10195,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10195
10195
|
}
|
|
10196
10196
|
/** Best-effort cancel a single job on a peer (double-execution guard). */
|
|
10197
10197
|
cancelPeerJob(peerId, runId, jobId, reason) {
|
|
10198
|
-
if (!this.cancelJobOnPeer(peerId, runId, jobId, reason)) logger$
|
|
10198
|
+
if (!this.cancelJobOnPeer(peerId, runId, jobId, reason)) logger$150.warn("Failed to send re-dispatch cancel to peer", {
|
|
10199
10199
|
peerId,
|
|
10200
10200
|
runId,
|
|
10201
10201
|
jobId
|
|
@@ -10274,7 +10274,7 @@ var init_coordinator = __esmMin((() => {
|
|
|
10274
10274
|
if (result.status === "queued-no-backend") await this.dispatcher.cancelQueuedJob(result.jobId, "reroute re-dispatch unroutable");
|
|
10275
10275
|
return false;
|
|
10276
10276
|
}
|
|
10277
|
-
logger$
|
|
10277
|
+
logger$150.info("Rerouted job re-dispatched locally", {
|
|
10278
10278
|
runId: runContext.runId,
|
|
10279
10279
|
jobId,
|
|
10280
10280
|
jobName: job.jobName
|
|
@@ -10283,13 +10283,13 @@ var init_coordinator = __esmMin((() => {
|
|
|
10283
10283
|
}
|
|
10284
10284
|
/** Mark a job failed when no backend (peer or local) can run it after re-dispatch. */
|
|
10285
10285
|
async failUnroutableJob(runId, jobId, jobName) {
|
|
10286
|
-
logger$
|
|
10286
|
+
logger$150.error("Rerouted job unroutable after spawn failure — failing it", {
|
|
10287
10287
|
runId,
|
|
10288
10288
|
jobId,
|
|
10289
10289
|
jobName
|
|
10290
10290
|
});
|
|
10291
10291
|
if (this.executionTracker) await this.executionTracker.onJobStatus(runId, jobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, { reason: "reroute re-dispatch found no available backend after spawn failure" }).catch((err) => {
|
|
10292
|
-
logger$
|
|
10292
|
+
logger$150.error("Failed to record unroutable-job failure", {
|
|
10293
10293
|
runId,
|
|
10294
10294
|
jobId,
|
|
10295
10295
|
error: toErrorMessage(err)
|
|
@@ -11175,10 +11175,10 @@ function satisfiesMandatoryLabels(agent, requiredLabels) {
|
|
|
11175
11175
|
for (const label of agent.mandatoryLabels) if (!requiredSet.has(label)) return false;
|
|
11176
11176
|
return true;
|
|
11177
11177
|
}
|
|
11178
|
-
var BUSY_HOLD_MAX_MS, logger$
|
|
11178
|
+
var BUSY_HOLD_MAX_MS, logger$149, AgentRegistry;
|
|
11179
11179
|
var init_registry = __esmMin((() => {
|
|
11180
11180
|
BUSY_HOLD_MAX_MS = 3e4;
|
|
11181
|
-
logger$
|
|
11181
|
+
logger$149 = createLogger({ prefix: "agent-registry" });
|
|
11182
11182
|
AgentRegistry = class {
|
|
11183
11183
|
agents = /* @__PURE__ */ new Map();
|
|
11184
11184
|
/** label -> agentIds, keyed on the canonical label. */
|
|
@@ -11340,7 +11340,7 @@ var init_registry = __esmMin((() => {
|
|
|
11340
11340
|
arch,
|
|
11341
11341
|
instanceId,
|
|
11342
11342
|
properties: metadata?.properties
|
|
11343
|
-
}).catch((err) => logger$
|
|
11343
|
+
}).catch((err) => logger$149.warn("host_roster upsert failed (best-effort)", {
|
|
11344
11344
|
agentId,
|
|
11345
11345
|
error: toErrorMessage(err)
|
|
11346
11346
|
}));
|
|
@@ -11396,7 +11396,7 @@ var init_registry = __esmMin((() => {
|
|
|
11396
11396
|
entry.ws.close(WS_CLOSE_AGENT_AUTH_FAILED, "Token revoked");
|
|
11397
11397
|
} catch {}
|
|
11398
11398
|
this.disconnectTriage?.(agentId).catch((err) => {
|
|
11399
|
-
logger$
|
|
11399
|
+
logger$149.error("Job triage failed for an agent dropped by token revocation", {
|
|
11400
11400
|
agentId,
|
|
11401
11401
|
error: toErrorMessage(err)
|
|
11402
11402
|
});
|
|
@@ -11589,8 +11589,11 @@ var init_registry = __esmMin((() => {
|
|
|
11589
11589
|
return this.agents.has(agentId);
|
|
11590
11590
|
}
|
|
11591
11591
|
/**
|
|
11592
|
-
* Condemn an agent
|
|
11593
|
-
*
|
|
11592
|
+
* Condemn an agent: it stays registered and keeps finishing its current work,
|
|
11593
|
+
* but takes no new dispatches.
|
|
11594
|
+
*
|
|
11595
|
+
* Called before a warm-pool destroy is issued. An agent that reports its own
|
|
11596
|
+
* drain goes through {@link markDrainReported} instead.
|
|
11594
11597
|
*
|
|
11595
11598
|
* @returns the agent's `activeJobs`, so the caller can see a job claimed between its
|
|
11596
11599
|
* idle check and this mark and back out; `undefined` when not registered
|
|
@@ -11651,10 +11654,23 @@ var init_registry = __esmMin((() => {
|
|
|
11651
11654
|
const entry = this.agents.get(agentId);
|
|
11652
11655
|
if (entry?.busyHeldUntil !== void 0 && now >= entry.busyHeldUntil) delete entry.busyHeldUntil;
|
|
11653
11656
|
}
|
|
11654
|
-
/**
|
|
11657
|
+
/**
|
|
11658
|
+
* The agent reported on `agent.status` that it takes no new job: a drain, or a
|
|
11659
|
+
* single-job agent that has accepted its job. Terminal for this connection.
|
|
11660
|
+
*/
|
|
11661
|
+
markDrainReported(agentId) {
|
|
11662
|
+
const entry = this.agents.get(agentId);
|
|
11663
|
+
if (!entry) return;
|
|
11664
|
+
entry.draining = true;
|
|
11665
|
+
entry.drainReported = true;
|
|
11666
|
+
}
|
|
11667
|
+
/**
|
|
11668
|
+
* Undo {@link markDraining} — a destroy that was rejected leaves the agent alive.
|
|
11669
|
+
* An agent that reported its own drain stays draining.
|
|
11670
|
+
*/
|
|
11655
11671
|
clearDraining(agentId) {
|
|
11656
11672
|
const entry = this.agents.get(agentId);
|
|
11657
|
-
if (entry) entry.draining =
|
|
11673
|
+
if (entry) entry.draining = entry.drainReported === true;
|
|
11658
11674
|
}
|
|
11659
11675
|
get(agentId) {
|
|
11660
11676
|
return this.agents.get(agentId);
|
|
@@ -12252,9 +12268,9 @@ var init_leader_gated_scheduler = __esmMin((() => {
|
|
|
12252
12268
|
* Driven by the leader-only host-roster reaper's tick (one timer, one leader), so it owns no
|
|
12253
12269
|
* timer — just a testable `scan()`.
|
|
12254
12270
|
*/
|
|
12255
|
-
var logger$
|
|
12271
|
+
var logger$148, RebootDeadlineSweep;
|
|
12256
12272
|
var init_reboot_deadline_sweep = __esmMin((() => {
|
|
12257
|
-
logger$
|
|
12273
|
+
logger$148 = createLogger({ prefix: "reboot-deadline-sweep" });
|
|
12258
12274
|
RebootDeadlineSweep = class {
|
|
12259
12275
|
rosterStore;
|
|
12260
12276
|
constructor(deps) {
|
|
@@ -12265,22 +12281,22 @@ var init_reboot_deadline_sweep = __esmMin((() => {
|
|
|
12265
12281
|
const expired = await this.rosterStore.listExpiredRebootPending(Date.now());
|
|
12266
12282
|
for (const agentId of expired) {
|
|
12267
12283
|
await this.rosterStore.clearRebootPending(agentId);
|
|
12268
|
-
logger$
|
|
12284
|
+
logger$148.warn("Host did not return after reboot within deadline; cleared reboot-pending", { agentId });
|
|
12269
12285
|
}
|
|
12270
12286
|
} catch (err) {
|
|
12271
|
-
logger$
|
|
12287
|
+
logger$148.error("Reboot deadline sweep error", { error: toErrorMessage(err) });
|
|
12272
12288
|
}
|
|
12273
12289
|
}
|
|
12274
12290
|
};
|
|
12275
12291
|
}));
|
|
12276
12292
|
//#endregion
|
|
12277
12293
|
//#region src/agent/host-roster-reaper.ts
|
|
12278
|
-
var logger$
|
|
12294
|
+
var logger$147, HostRosterReaper;
|
|
12279
12295
|
var init_host_roster_reaper = __esmMin((() => {
|
|
12280
12296
|
init_leader_gated_scheduler();
|
|
12281
12297
|
init_reboot_deadline_sweep();
|
|
12282
12298
|
init_run_detached();
|
|
12283
|
-
logger$
|
|
12299
|
+
logger$147 = createLogger({ prefix: "host-roster-reaper" });
|
|
12284
12300
|
HostRosterReaper = class {
|
|
12285
12301
|
store;
|
|
12286
12302
|
ttlMs;
|
|
@@ -12299,12 +12315,12 @@ var init_host_roster_reaper = __esmMin((() => {
|
|
|
12299
12315
|
this.scheduler = new LeaderGatedScheduler({
|
|
12300
12316
|
name: "host roster reaper",
|
|
12301
12317
|
intervalMs: this.scanIntervalMs,
|
|
12302
|
-
logger: logger$
|
|
12318
|
+
logger: logger$147,
|
|
12303
12319
|
tick: () => this.tick()
|
|
12304
12320
|
});
|
|
12305
12321
|
}
|
|
12306
12322
|
onBecomeLeader() {
|
|
12307
|
-
runDetached(logger$
|
|
12323
|
+
runDetached(logger$147, "Host roster reaper start", () => this.scheduler.onBecomeLeader());
|
|
12308
12324
|
}
|
|
12309
12325
|
onLoseLeadership() {
|
|
12310
12326
|
this.scheduler.onLoseLeadership();
|
|
@@ -12316,10 +12332,10 @@ var init_host_roster_reaper = __esmMin((() => {
|
|
|
12316
12332
|
async tick() {
|
|
12317
12333
|
if (!this.scheduler.isLeader) return;
|
|
12318
12334
|
const deleted = await this.store.reapEphemeralPastTtl(this.ttlMs);
|
|
12319
|
-
if (deleted > 0) logger$
|
|
12335
|
+
if (deleted > 0) logger$147.info("Reaped expired ephemeral hosts", { deleted });
|
|
12320
12336
|
const unreachable = await this.store.countStaticUnreachable(this.graceMs);
|
|
12321
12337
|
this.setUnreachableGauge(unreachable);
|
|
12322
|
-
if (unreachable > 0) logger$
|
|
12338
|
+
if (unreachable > 0) logger$147.warn("Declared hosts unreachable", { count: unreachable });
|
|
12323
12339
|
await this.rebootSweep.scan();
|
|
12324
12340
|
}
|
|
12325
12341
|
};
|
|
@@ -12346,16 +12362,16 @@ async function pruneExpiredLogs(storage, ttlDays, now = /* @__PURE__ */ new Date
|
|
|
12346
12362
|
try {
|
|
12347
12363
|
return await storage.deleteMany(expired);
|
|
12348
12364
|
} catch (err) {
|
|
12349
|
-
logger$
|
|
12365
|
+
logger$146.error("Failed to bulk-delete expired log objects", {
|
|
12350
12366
|
count: expired.length,
|
|
12351
12367
|
error: toErrorMessage(err)
|
|
12352
12368
|
});
|
|
12353
12369
|
return 0;
|
|
12354
12370
|
}
|
|
12355
12371
|
}
|
|
12356
|
-
var logger$
|
|
12372
|
+
var logger$146, LOG_ROOT_PREFIX;
|
|
12357
12373
|
var init_log_retention = __esmMin((() => {
|
|
12358
|
-
logger$
|
|
12374
|
+
logger$146 = createLogger({ prefix: "log-retention" });
|
|
12359
12375
|
LOG_ROOT_PREFIX = "executions/";
|
|
12360
12376
|
}));
|
|
12361
12377
|
//#endregion
|
|
@@ -12752,14 +12768,14 @@ function buildJobFailureDescription(data) {
|
|
|
12752
12768
|
if (data.error) return `Job error: ${data.error}`;
|
|
12753
12769
|
return "Job failed";
|
|
12754
12770
|
}
|
|
12755
|
-
var logger$
|
|
12771
|
+
var logger$145, PROGRESS_DEBOUNCE_MS, CheckRunReporter;
|
|
12756
12772
|
var init_check_run_reporter = __esmMin((() => {
|
|
12757
12773
|
init_auth();
|
|
12758
12774
|
init_prometheus();
|
|
12759
12775
|
init_reduced_privilege_note();
|
|
12760
12776
|
init_check_run_summary();
|
|
12761
12777
|
init_run_detached();
|
|
12762
|
-
logger$
|
|
12778
|
+
logger$145 = createLogger({ prefix: "check-run-reporter" });
|
|
12763
12779
|
PROGRESS_DEBOUNCE_MS = 5e3;
|
|
12764
12780
|
CheckRunReporter = class {
|
|
12765
12781
|
deps;
|
|
@@ -12847,7 +12863,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
12847
12863
|
*/
|
|
12848
12864
|
setPending(opts) {
|
|
12849
12865
|
this.doSetPending(opts).catch((err) => {
|
|
12850
|
-
logger$
|
|
12866
|
+
logger$145.error("Failed to set pending check run", {
|
|
12851
12867
|
error: toErrorMessage(err),
|
|
12852
12868
|
provider: opts.provider,
|
|
12853
12869
|
owner: opts.owner,
|
|
@@ -12867,7 +12883,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
12867
12883
|
/** Called when a job reaches a terminal state. Fire-and-forget. */
|
|
12868
12884
|
updateJobStatus(opts) {
|
|
12869
12885
|
this.doUpdateJobStatus(opts).catch((err) => {
|
|
12870
|
-
logger$
|
|
12886
|
+
logger$145.error("Failed to update job check run", {
|
|
12871
12887
|
error: toErrorMessage(err),
|
|
12872
12888
|
provider: opts.provider,
|
|
12873
12889
|
owner: opts.owner,
|
|
@@ -12884,7 +12900,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
12884
12900
|
*/
|
|
12885
12901
|
updateWorkflowStatus(opts) {
|
|
12886
12902
|
this.doUpdateWorkflowStatus(opts).catch((err) => {
|
|
12887
|
-
logger$
|
|
12903
|
+
logger$145.error("Failed to update workflow check run", {
|
|
12888
12904
|
error: toErrorMessage(err),
|
|
12889
12905
|
provider: opts.provider,
|
|
12890
12906
|
owner: opts.owner,
|
|
@@ -12915,12 +12931,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
12915
12931
|
*/
|
|
12916
12932
|
async completeUndispatchedCheckRuns(opts) {
|
|
12917
12933
|
if (opts.provider !== "github") {
|
|
12918
|
-
logger$
|
|
12934
|
+
logger$145.debug("Check runs not supported for provider, skipping", { provider: opts.provider });
|
|
12919
12935
|
return;
|
|
12920
12936
|
}
|
|
12921
12937
|
const githubConfig = this.resolveGithubConfig(opts.routingKey);
|
|
12922
12938
|
if (!githubConfig || !opts.installationId) {
|
|
12923
|
-
logger$
|
|
12939
|
+
logger$145.debug("GitHub config or installationId missing, skipping undispatched completion", {
|
|
12924
12940
|
hasConfig: !!githubConfig,
|
|
12925
12941
|
hasInstallationId: !!opts.installationId
|
|
12926
12942
|
});
|
|
@@ -12943,7 +12959,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
12943
12959
|
const key = this.checkRunKey(opts.owner, opts.repo, opts.sha, target.name);
|
|
12944
12960
|
const checkRunId = await this.resolveCheckRunId(key);
|
|
12945
12961
|
if (!checkRunId) {
|
|
12946
|
-
logger$
|
|
12962
|
+
logger$145.debug("Check run ID not found for undispatched completion, skipping", { key });
|
|
12947
12963
|
continue;
|
|
12948
12964
|
}
|
|
12949
12965
|
if (this.terminalSent.has(key)) continue;
|
|
@@ -12963,7 +12979,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
12963
12979
|
}, key, opts.runId);
|
|
12964
12980
|
completed++;
|
|
12965
12981
|
}
|
|
12966
|
-
if (completed > 0) logger$
|
|
12982
|
+
if (completed > 0) logger$145.info("Completed check runs for a workflow that never dispatched", {
|
|
12967
12983
|
owner: opts.owner,
|
|
12968
12984
|
repo: opts.repo,
|
|
12969
12985
|
sha: opts.sha,
|
|
@@ -12983,7 +12999,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
12983
12999
|
*/
|
|
12984
13000
|
cleanupStaleCheckRuns(opts) {
|
|
12985
13001
|
this.doCleanupStaleCheckRuns(opts).catch((err) => {
|
|
12986
|
-
logger$
|
|
13002
|
+
logger$145.error("Failed to clean up stale check runs", {
|
|
12987
13003
|
error: toErrorMessage(err),
|
|
12988
13004
|
provider: opts.provider,
|
|
12989
13005
|
owner: opts.owner,
|
|
@@ -13000,7 +13016,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13000
13016
|
*/
|
|
13001
13017
|
updateStepProgress(opts) {
|
|
13002
13018
|
this.doUpdateStepProgress(opts).catch((err) => {
|
|
13003
|
-
logger$
|
|
13019
|
+
logger$145.error("Failed to update step progress", {
|
|
13004
13020
|
error: toErrorMessage(err),
|
|
13005
13021
|
provider: opts.provider,
|
|
13006
13022
|
workflowName: opts.workflowName,
|
|
@@ -13018,10 +13034,10 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13018
13034
|
setBuildPending(opts) {
|
|
13019
13035
|
const buildCheckName = `kici/${this.workflowLabel(opts)}/setup`;
|
|
13020
13036
|
const key = this.checkRunKey(opts.owner, opts.repo, opts.sha, buildCheckName);
|
|
13021
|
-
runDetached(logger$
|
|
13037
|
+
runDetached(logger$145, "Build creation marker write", () => this.persistBuildCreationPending(key, opts.runId), { runId: opts.runId });
|
|
13022
13038
|
this.trackRunKey(opts.runId, key);
|
|
13023
13039
|
const creation = this.doSetBuildPending(opts).catch((err) => {
|
|
13024
|
-
logger$
|
|
13040
|
+
logger$145.error("Failed to set build pending check run", {
|
|
13025
13041
|
error: toErrorMessage(err),
|
|
13026
13042
|
provider: opts.provider,
|
|
13027
13043
|
owner: opts.owner,
|
|
@@ -13031,12 +13047,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13031
13047
|
});
|
|
13032
13048
|
});
|
|
13033
13049
|
this.pendingBuildCreations.set(key, creation);
|
|
13034
|
-
runDetached(logger$
|
|
13050
|
+
runDetached(logger$145, "Build creation tracking cleanup", () => creation.finally(() => this.pendingBuildCreations.delete(key)), { runId: opts.runId });
|
|
13035
13051
|
}
|
|
13036
13052
|
/** Called when the build job finishes (success, failure, or cancellation). Fire-and-forget. */
|
|
13037
13053
|
setBuildComplete(opts) {
|
|
13038
13054
|
this.doSetBuildComplete(opts).catch((err) => {
|
|
13039
|
-
logger$
|
|
13055
|
+
logger$145.error("Failed to set build complete check run", {
|
|
13040
13056
|
error: toErrorMessage(err),
|
|
13041
13057
|
provider: opts.provider,
|
|
13042
13058
|
owner: opts.owner,
|
|
@@ -13082,7 +13098,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13082
13098
|
*/
|
|
13083
13099
|
async recoverState() {
|
|
13084
13100
|
if (!this.deps.trackingStore) return;
|
|
13085
|
-
logger$
|
|
13101
|
+
logger$145.info("CheckRunReporter recovered (DB-backed state lookups enabled)");
|
|
13086
13102
|
}
|
|
13087
13103
|
trackRunKey(runId, key) {
|
|
13088
13104
|
if (!runId) return;
|
|
@@ -13117,7 +13133,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13117
13133
|
await this.deps.trackingStore.setCheckRunId(this.parseKey(key), checkRunId, runId);
|
|
13118
13134
|
if (runId) await this.deps.trackingStore.markBuildCreationComplete(this.parseKey(key));
|
|
13119
13135
|
} catch (err) {
|
|
13120
|
-
logger$
|
|
13136
|
+
logger$145.warn("Failed to persist check_run_tracking row; cache-only fallback", {
|
|
13121
13137
|
key,
|
|
13122
13138
|
error: toErrorMessage(err)
|
|
13123
13139
|
});
|
|
@@ -13142,7 +13158,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13142
13158
|
if (state?.checkRunId !== void 0) this.checkRunIds.set(key, state.checkRunId);
|
|
13143
13159
|
return state?.checkRunId;
|
|
13144
13160
|
} catch (err) {
|
|
13145
|
-
logger$
|
|
13161
|
+
logger$145.warn("Failed to read check_run_tracking row; treating as miss", {
|
|
13146
13162
|
key,
|
|
13147
13163
|
error: toErrorMessage(err)
|
|
13148
13164
|
});
|
|
@@ -13155,7 +13171,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13155
13171
|
try {
|
|
13156
13172
|
await this.deps.trackingStore.setStepProgress(this.parseKey(key), steps, runId);
|
|
13157
13173
|
} catch (err) {
|
|
13158
|
-
logger$
|
|
13174
|
+
logger$145.warn("Failed to persist step-progress; cache-only fallback", {
|
|
13159
13175
|
key,
|
|
13160
13176
|
error: toErrorMessage(err)
|
|
13161
13177
|
});
|
|
@@ -13168,7 +13184,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13168
13184
|
try {
|
|
13169
13185
|
await this.deps.trackingStore.markInProgressSent(this.parseKey(key), runId);
|
|
13170
13186
|
} catch (err) {
|
|
13171
|
-
logger$
|
|
13187
|
+
logger$145.warn("Failed to persist in-progress-sent marker; cache-only fallback", {
|
|
13172
13188
|
key,
|
|
13173
13189
|
error: toErrorMessage(err)
|
|
13174
13190
|
});
|
|
@@ -13180,7 +13196,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13180
13196
|
try {
|
|
13181
13197
|
await this.deps.trackingStore.markBuildCreationPending(this.parseKey(key), runId);
|
|
13182
13198
|
} catch (err) {
|
|
13183
|
-
logger$
|
|
13199
|
+
logger$145.warn("Failed to persist build-creation-pending marker", {
|
|
13184
13200
|
key,
|
|
13185
13201
|
error: toErrorMessage(err)
|
|
13186
13202
|
});
|
|
@@ -13246,12 +13262,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13246
13262
|
}
|
|
13247
13263
|
async doSetPending(opts) {
|
|
13248
13264
|
if (opts.provider !== "github") {
|
|
13249
|
-
logger$
|
|
13265
|
+
logger$145.warn("Check runs not supported for provider, skipping", { provider: opts.provider });
|
|
13250
13266
|
return;
|
|
13251
13267
|
}
|
|
13252
13268
|
const githubConfig = this.resolveGithubConfig(opts.routingKey);
|
|
13253
13269
|
if (!githubConfig || !opts.installationId) {
|
|
13254
|
-
logger$
|
|
13270
|
+
logger$145.debug("GitHub config or installationId missing, skipping check run", {
|
|
13255
13271
|
hasConfig: !!githubConfig,
|
|
13256
13272
|
hasInstallationId: !!opts.installationId
|
|
13257
13273
|
});
|
|
@@ -13302,12 +13318,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13302
13318
|
}
|
|
13303
13319
|
async doUpdateJobStatus(opts) {
|
|
13304
13320
|
if (opts.provider !== "github") {
|
|
13305
|
-
logger$
|
|
13321
|
+
logger$145.warn("Check runs not supported for provider, skipping", { provider: opts.provider });
|
|
13306
13322
|
return;
|
|
13307
13323
|
}
|
|
13308
13324
|
const githubConfig = this.resolveGithubConfig(opts.routingKey);
|
|
13309
13325
|
if (!githubConfig || !opts.installationId) {
|
|
13310
|
-
logger$
|
|
13326
|
+
logger$145.debug("GitHub config or installationId missing, skipping check run update");
|
|
13311
13327
|
return;
|
|
13312
13328
|
}
|
|
13313
13329
|
const label = this.workflowLabel(opts);
|
|
@@ -13315,7 +13331,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13315
13331
|
const key = this.checkRunKey(opts.owner, opts.repo, opts.sha, checkName);
|
|
13316
13332
|
const checkRunId = await this.resolveCheckRunId(key);
|
|
13317
13333
|
if (!checkRunId) {
|
|
13318
|
-
logger$
|
|
13334
|
+
logger$145.warn("Check run ID not found for job update, skipping", { key });
|
|
13319
13335
|
return;
|
|
13320
13336
|
}
|
|
13321
13337
|
const pendingTimer = this.progressTimers.get(key);
|
|
@@ -13394,19 +13410,19 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13394
13410
|
}
|
|
13395
13411
|
async doUpdateWorkflowStatus(opts) {
|
|
13396
13412
|
if (opts.provider !== "github") {
|
|
13397
|
-
logger$
|
|
13413
|
+
logger$145.warn("Check runs not supported for provider, skipping", { provider: opts.provider });
|
|
13398
13414
|
return;
|
|
13399
13415
|
}
|
|
13400
13416
|
const githubConfig = this.resolveGithubConfig(opts.routingKey);
|
|
13401
13417
|
if (!githubConfig || !opts.installationId) {
|
|
13402
|
-
logger$
|
|
13418
|
+
logger$145.debug("GitHub config or installationId missing, skipping check run update");
|
|
13403
13419
|
return;
|
|
13404
13420
|
}
|
|
13405
13421
|
const label = this.workflowLabel(opts);
|
|
13406
13422
|
const key = this.checkRunKey(opts.owner, opts.repo, opts.sha, `kici/${label}`);
|
|
13407
13423
|
const checkRunId = await this.resolveCheckRunId(key);
|
|
13408
13424
|
if (!checkRunId) {
|
|
13409
|
-
logger$
|
|
13425
|
+
logger$145.warn("Check run ID not found for workflow update, skipping", { key });
|
|
13410
13426
|
return;
|
|
13411
13427
|
}
|
|
13412
13428
|
const octokit = createInstallationOctokit(githubConfig, opts.installationId);
|
|
@@ -13431,12 +13447,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13431
13447
|
}
|
|
13432
13448
|
async doCleanupStaleCheckRuns(opts) {
|
|
13433
13449
|
if (opts.provider !== "github") {
|
|
13434
|
-
logger$
|
|
13450
|
+
logger$145.debug("Stale check run cleanup not supported for provider, skipping", { provider: opts.provider });
|
|
13435
13451
|
return;
|
|
13436
13452
|
}
|
|
13437
13453
|
const githubConfig = this.resolveGithubConfig(opts.routingKey);
|
|
13438
13454
|
if (!githubConfig) {
|
|
13439
|
-
logger$
|
|
13455
|
+
logger$145.debug("GitHub config not available for stale cleanup, skipping", { routingKey: opts.routingKey });
|
|
13440
13456
|
return;
|
|
13441
13457
|
}
|
|
13442
13458
|
const appOctokit = new Octokit({
|
|
@@ -13454,7 +13470,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13454
13470
|
});
|
|
13455
13471
|
installationId = data.id;
|
|
13456
13472
|
} catch (err) {
|
|
13457
|
-
logger$
|
|
13473
|
+
logger$145.warn("Could not find installation for repo, skipping stale cleanup", {
|
|
13458
13474
|
owner: opts.owner,
|
|
13459
13475
|
repo: opts.repo,
|
|
13460
13476
|
error: toErrorMessage(err)
|
|
@@ -13490,14 +13506,14 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13490
13506
|
});
|
|
13491
13507
|
cleanedCount++;
|
|
13492
13508
|
} catch (updateErr) {
|
|
13493
|
-
logger$
|
|
13509
|
+
logger$145.warn("Failed to update stale check run", {
|
|
13494
13510
|
checkRunId: checkRun.id,
|
|
13495
13511
|
checkName: checkRun.name,
|
|
13496
13512
|
error: toErrorMessage(updateErr)
|
|
13497
13513
|
});
|
|
13498
13514
|
}
|
|
13499
13515
|
if (cleanedCount > 0) {
|
|
13500
|
-
logger$
|
|
13516
|
+
logger$145.info("Cleaned up stale check runs", {
|
|
13501
13517
|
owner: opts.owner,
|
|
13502
13518
|
repo: opts.repo,
|
|
13503
13519
|
sha: opts.sha,
|
|
@@ -13507,7 +13523,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13507
13523
|
githubCheckRunTotal.add(cleanedCount, { operation: "stale_cleanup" });
|
|
13508
13524
|
}
|
|
13509
13525
|
} catch (err) {
|
|
13510
|
-
logger$
|
|
13526
|
+
logger$145.error("Failed to list check runs for stale cleanup", {
|
|
13511
13527
|
owner: opts.owner,
|
|
13512
13528
|
repo: opts.repo,
|
|
13513
13529
|
sha: opts.sha,
|
|
@@ -13575,7 +13591,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13575
13591
|
const timer = setTimeout(() => {
|
|
13576
13592
|
this.progressTimers.delete(key);
|
|
13577
13593
|
this.flushProgress(key, opts).catch((err) => {
|
|
13578
|
-
logger$
|
|
13594
|
+
logger$145.error("Failed to flush step progress", {
|
|
13579
13595
|
error: toErrorMessage(err),
|
|
13580
13596
|
key
|
|
13581
13597
|
});
|
|
@@ -13613,12 +13629,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13613
13629
|
}
|
|
13614
13630
|
async doSetBuildPending(opts) {
|
|
13615
13631
|
if (opts.provider !== "github") {
|
|
13616
|
-
logger$
|
|
13632
|
+
logger$145.warn("Check runs not supported for provider, skipping", { provider: opts.provider });
|
|
13617
13633
|
return;
|
|
13618
13634
|
}
|
|
13619
13635
|
const githubConfig = this.resolveGithubConfig(opts.routingKey);
|
|
13620
13636
|
if (!githubConfig || !opts.installationId) {
|
|
13621
|
-
logger$
|
|
13637
|
+
logger$145.debug("GitHub config or installationId missing, skipping build check run", {
|
|
13622
13638
|
hasConfig: !!githubConfig,
|
|
13623
13639
|
hasInstallationId: !!opts.installationId
|
|
13624
13640
|
});
|
|
@@ -13648,12 +13664,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13648
13664
|
}
|
|
13649
13665
|
async doSetBuildComplete(opts) {
|
|
13650
13666
|
if (opts.provider !== "github") {
|
|
13651
|
-
logger$
|
|
13667
|
+
logger$145.warn("Check runs not supported for provider, skipping", { provider: opts.provider });
|
|
13652
13668
|
return;
|
|
13653
13669
|
}
|
|
13654
13670
|
const githubConfig = this.resolveGithubConfig(opts.routingKey);
|
|
13655
13671
|
if (!githubConfig || !opts.installationId) {
|
|
13656
|
-
logger$
|
|
13672
|
+
logger$145.debug("GitHub config or installationId missing, skipping build check run update");
|
|
13657
13673
|
return;
|
|
13658
13674
|
}
|
|
13659
13675
|
const label = this.workflowLabel(opts);
|
|
@@ -13663,7 +13679,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13663
13679
|
if (pending) await pending;
|
|
13664
13680
|
const checkRunId = await this.resolveCheckRunId(key);
|
|
13665
13681
|
if (!checkRunId) {
|
|
13666
|
-
logger$
|
|
13682
|
+
logger$145.warn("Check run ID not found for build update, skipping", { key });
|
|
13667
13683
|
return;
|
|
13668
13684
|
}
|
|
13669
13685
|
const octokit = createInstallationOctokit(githubConfig, opts.installationId);
|
|
@@ -13716,7 +13732,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13716
13732
|
};
|
|
13717
13733
|
default: {
|
|
13718
13734
|
const unexpected = status;
|
|
13719
|
-
logger$
|
|
13735
|
+
logger$145.warn("Unexpected terminal job status mapping a build conclusion", { status: unexpected });
|
|
13720
13736
|
return {
|
|
13721
13737
|
conclusion: CheckRunConclusion.enum.failure,
|
|
13722
13738
|
description: customDescription ?? "Build finished with an unrecognised status"
|
|
@@ -13756,7 +13772,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13756
13772
|
};
|
|
13757
13773
|
default: {
|
|
13758
13774
|
const unexpected = state;
|
|
13759
|
-
logger$
|
|
13775
|
+
logger$145.warn("Unexpected terminal job status mapping a job conclusion", { status: unexpected });
|
|
13760
13776
|
return {
|
|
13761
13777
|
conclusion: CheckRunConclusion.enum.failure,
|
|
13762
13778
|
description: customDescription ?? "Job finished with an unrecognised status"
|
|
@@ -13796,7 +13812,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13796
13812
|
};
|
|
13797
13813
|
default: {
|
|
13798
13814
|
const unexpected = status;
|
|
13799
|
-
logger$
|
|
13815
|
+
logger$145.warn("Unexpected terminal job status mapping a workflow conclusion", { status: unexpected });
|
|
13800
13816
|
return {
|
|
13801
13817
|
conclusion: CheckRunConclusion.enum.failure,
|
|
13802
13818
|
description: customDescription ?? "Workflow finished with an unrecognised status"
|
|
@@ -13823,12 +13839,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13823
13839
|
if (error.status === 403) {
|
|
13824
13840
|
const rateRemaining = error.response?.headers?.["x-ratelimit-remaining"];
|
|
13825
13841
|
const rateReset = error.response?.headers?.["x-ratelimit-reset"];
|
|
13826
|
-
logger$
|
|
13842
|
+
logger$145.error("GitHub API 403 creating check run", {
|
|
13827
13843
|
name: params.name,
|
|
13828
13844
|
rateRemaining,
|
|
13829
13845
|
rateReset
|
|
13830
13846
|
});
|
|
13831
|
-
} else logger$
|
|
13847
|
+
} else logger$145.error("GitHub API error creating check run", {
|
|
13832
13848
|
name: params.name,
|
|
13833
13849
|
status: error.status,
|
|
13834
13850
|
error: toErrorMessage(err)
|
|
@@ -13866,12 +13882,12 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13866
13882
|
if (error.status === 403) {
|
|
13867
13883
|
const rateRemaining = error.response?.headers?.["x-ratelimit-remaining"];
|
|
13868
13884
|
const rateReset = error.response?.headers?.["x-ratelimit-reset"];
|
|
13869
|
-
logger$
|
|
13885
|
+
logger$145.error("GitHub API 403 updating check run", {
|
|
13870
13886
|
checkRunId: params.check_run_id,
|
|
13871
13887
|
rateRemaining,
|
|
13872
13888
|
rateReset
|
|
13873
13889
|
});
|
|
13874
|
-
} else logger$
|
|
13890
|
+
} else logger$145.error("GitHub API error updating check run", {
|
|
13875
13891
|
checkRunId: params.check_run_id,
|
|
13876
13892
|
status: error.status,
|
|
13877
13893
|
error: toErrorMessage(err)
|
|
@@ -13880,7 +13896,7 @@ var init_check_run_reporter = __esmMin((() => {
|
|
|
13880
13896
|
if (patchAccepted && params.status === "completed" && trackingKey && this.deps.trackingStore) try {
|
|
13881
13897
|
await this.deps.trackingStore.markTerminalSent(this.parseKey(trackingKey), trackingRunId);
|
|
13882
13898
|
} catch (err) {
|
|
13883
|
-
logger$
|
|
13899
|
+
logger$145.warn("Failed to stamp terminal_sent_at; continuing", {
|
|
13884
13900
|
key: trackingKey,
|
|
13885
13901
|
error: toErrorMessage(err)
|
|
13886
13902
|
});
|
|
@@ -13915,7 +13931,7 @@ function reportJobCheckRunCompletion(deps, input) {
|
|
|
13915
13931
|
try {
|
|
13916
13932
|
postJobCheckRunCompletion(deps, input);
|
|
13917
13933
|
} catch (err) {
|
|
13918
|
-
logger$
|
|
13934
|
+
logger$144.error("Failed to report job check-run completion", {
|
|
13919
13935
|
error: toErrorMessage(err),
|
|
13920
13936
|
runId: input.runId,
|
|
13921
13937
|
jobId: input.jobId,
|
|
@@ -13927,7 +13943,7 @@ function postJobCheckRunCompletion(deps, input) {
|
|
|
13927
13943
|
if (!isTerminalJobStatus(input.status)) return;
|
|
13928
13944
|
const execContext = deps.getExecutionContext(input.runId);
|
|
13929
13945
|
if (!execContext) {
|
|
13930
|
-
logger$
|
|
13946
|
+
logger$144.debug("No execution context for job check-run completion, skipping", {
|
|
13931
13947
|
runId: input.runId,
|
|
13932
13948
|
jobId: input.jobId
|
|
13933
13949
|
});
|
|
@@ -13955,10 +13971,10 @@ function postJobCheckRunCompletion(deps, input) {
|
|
|
13955
13971
|
runId: input.runId
|
|
13956
13972
|
});
|
|
13957
13973
|
}
|
|
13958
|
-
var logger$
|
|
13974
|
+
var logger$144;
|
|
13959
13975
|
var init_job_check_run_completion = __esmMin((() => {
|
|
13960
13976
|
init_check_run_reporter();
|
|
13961
|
-
logger$
|
|
13977
|
+
logger$144 = createLogger({ prefix: "check-run-completion" });
|
|
13962
13978
|
}));
|
|
13963
13979
|
//#endregion
|
|
13964
13980
|
//#region src/queue/terminalize-unroutable.ts
|
|
@@ -14066,7 +14082,7 @@ async function terminalizeUnroutableJob(deps, job) {
|
|
|
14066
14082
|
}
|
|
14067
14083
|
return job.runId;
|
|
14068
14084
|
} catch (err) {
|
|
14069
|
-
logger$
|
|
14085
|
+
logger$143.error("Failed to forward never-dispatched job status", {
|
|
14070
14086
|
runId: job.runId,
|
|
14071
14087
|
jobName: job.jobName,
|
|
14072
14088
|
error: toErrorMessage(err)
|
|
@@ -14074,11 +14090,11 @@ async function terminalizeUnroutableJob(deps, job) {
|
|
|
14074
14090
|
return null;
|
|
14075
14091
|
}
|
|
14076
14092
|
}
|
|
14077
|
-
var logger$
|
|
14093
|
+
var logger$143, GENERIC_MESSAGE;
|
|
14078
14094
|
var init_terminalize_unroutable = __esmMin((() => {
|
|
14079
14095
|
init_job_check_run_completion();
|
|
14080
14096
|
init_agent_fit();
|
|
14081
|
-
logger$
|
|
14097
|
+
logger$143 = createLogger({ prefix: "terminalize-unroutable" });
|
|
14082
14098
|
GENERIC_MESSAGE = "Queue timeout expired (job was never dispatched to an agent)";
|
|
14083
14099
|
}));
|
|
14084
14100
|
//#endregion
|
|
@@ -14235,7 +14251,7 @@ async function pruneExpiredHistory(deps) {
|
|
|
14235
14251
|
const r = await db.selectFrom("execution_runs").select(({ fn }) => fn.countAll().as("n")).where("created_at", "<", cutoff).where("status", "in", TERMINAL_RUN_STATUS_LIST).executeTakeFirst();
|
|
14236
14252
|
summary.pending.execution_runs = Number(r?.n ?? 0);
|
|
14237
14253
|
}
|
|
14238
|
-
logger$
|
|
14254
|
+
logger$142.warn("Database retention is announced but not yet deleting; the first deletion runs 7 days after the announce stamp. Set KICI_RUN_RETENTION_DAYS / KICI_AUDIT_RETENTION_DAYS / KICI_PROVENANCE_RETENTION_DAYS / KICI_HELD_RUN_RETENTION_DAYS to 0 to disable.", {
|
|
14239
14255
|
windows,
|
|
14240
14256
|
wouldDelete: summary.pending
|
|
14241
14257
|
});
|
|
@@ -14254,7 +14270,7 @@ async function pruneExpiredHistory(deps) {
|
|
|
14254
14270
|
summary.deleted.pending_attestations = await pruneByAge(db, "pending_attestations", "created_at", cutoff, batchSize);
|
|
14255
14271
|
}
|
|
14256
14272
|
if (windows.heldRunRetentionDays > 0) summary.deleted.held_runs = await pruneHeldRuns(db, cutoffFor(now, windows.heldRunRetentionDays), batchSize);
|
|
14257
|
-
if (Object.values(summary.deleted).reduce((a, b) => a + b, 0) > 0) logger$
|
|
14273
|
+
if (Object.values(summary.deleted).reduce((a, b) => a + b, 0) > 0) logger$142.info("Pruned expired history", {
|
|
14258
14274
|
deleted: summary.deleted,
|
|
14259
14275
|
windows
|
|
14260
14276
|
});
|
|
@@ -14278,7 +14294,7 @@ async function pruneExpiredHistorySafely(deps) {
|
|
|
14278
14294
|
try {
|
|
14279
14295
|
return await pruneExpiredHistory(deps);
|
|
14280
14296
|
} catch (err) {
|
|
14281
|
-
logger$
|
|
14297
|
+
logger$142.error("Failed to prune expired history", { error: toErrorMessage(err) });
|
|
14282
14298
|
return {
|
|
14283
14299
|
deleted: {},
|
|
14284
14300
|
pending: {},
|
|
@@ -14286,11 +14302,11 @@ async function pruneExpiredHistorySafely(deps) {
|
|
|
14286
14302
|
};
|
|
14287
14303
|
}
|
|
14288
14304
|
}
|
|
14289
|
-
var logger$
|
|
14305
|
+
var logger$142, TERMINAL_RUN_STATUS_LIST, TERMINAL_HELD_RUN_STATUSES, COLD_STORE_OWNED_TABLES;
|
|
14290
14306
|
var init_retention$1 = __esmMin((() => {
|
|
14291
14307
|
init_held_runs();
|
|
14292
14308
|
init_prometheus();
|
|
14293
|
-
logger$
|
|
14309
|
+
logger$142 = createLogger({ prefix: "retention" });
|
|
14294
14310
|
TERMINAL_RUN_STATUS_LIST = [...TERMINAL_RUN_STATES];
|
|
14295
14311
|
TERMINAL_HELD_RUN_STATUSES = [
|
|
14296
14312
|
"approved",
|
|
@@ -14328,37 +14344,37 @@ async function runCleanup(dedup, queue, extras) {
|
|
|
14328
14344
|
if (extras) {
|
|
14329
14345
|
try {
|
|
14330
14346
|
const pruned = await pruneRequestIdempotency(extras.db);
|
|
14331
|
-
if (pruned > 0) logger$
|
|
14347
|
+
if (pruned > 0) logger$141.info("Pruned stale request idempotency claims", { pruned });
|
|
14332
14348
|
} catch (err) {
|
|
14333
|
-
logger$
|
|
14349
|
+
logger$141.error("Failed to prune request idempotency claims", { error: toErrorMessage(err) });
|
|
14334
14350
|
}
|
|
14335
14351
|
const ttlDays = extras.dispatchQueueTtlDays ?? 30;
|
|
14336
14352
|
try {
|
|
14337
14353
|
dispatchRowsPruned = await queue.pruneTerminalDispatchRows(ttlDays);
|
|
14338
14354
|
if (dispatchRowsPruned > 0) {
|
|
14339
14355
|
dispatchQueueRowsPrunedTotal.add(dispatchRowsPruned);
|
|
14340
|
-
logger$
|
|
14356
|
+
logger$141.info("Pruned terminal dispatch_queue rows", {
|
|
14341
14357
|
dispatchRowsPruned,
|
|
14342
14358
|
ttlDays
|
|
14343
14359
|
});
|
|
14344
14360
|
}
|
|
14345
14361
|
} catch (err) {
|
|
14346
|
-
logger$
|
|
14362
|
+
logger$141.error("Failed to prune terminal dispatch_queue rows", { error: toErrorMessage(err) });
|
|
14347
14363
|
}
|
|
14348
14364
|
try {
|
|
14349
14365
|
const scrubbed = await queue.scrubTerminalSealedSecrets();
|
|
14350
|
-
if (scrubbed > 0) logger$
|
|
14366
|
+
if (scrubbed > 0) logger$141.info("Scrubbed sealed secrets of terminal dispatch_queue rows", { scrubbed });
|
|
14351
14367
|
} catch (err) {
|
|
14352
|
-
logger$
|
|
14368
|
+
logger$141.error("Failed to scrub sealed secrets of terminal dispatch_queue rows", { error: toErrorMessage(err) });
|
|
14353
14369
|
}
|
|
14354
14370
|
if (extras.logStorage && extras.logStorageIsS3) try {
|
|
14355
14371
|
logObjectsPruned = await pruneExpiredLogs(extras.logStorage, extras.stepLogTtlDays ?? 90);
|
|
14356
14372
|
if (logObjectsPruned > 0) {
|
|
14357
14373
|
stepLogsPrunedTotal.add(logObjectsPruned);
|
|
14358
|
-
logger$
|
|
14374
|
+
logger$141.info("Pruned expired step-log objects", { logObjectsPruned });
|
|
14359
14375
|
}
|
|
14360
14376
|
} catch (err) {
|
|
14361
|
-
logger$
|
|
14377
|
+
logger$141.error("Failed to prune expired step-log objects", { error: toErrorMessage(err) });
|
|
14362
14378
|
}
|
|
14363
14379
|
if (extras.checkRunTrackingStore) {
|
|
14364
14380
|
const checkRunTtlDefault = extras.checkRunTrackingTtlDays ?? 7;
|
|
@@ -14367,13 +14383,13 @@ async function runCleanup(dedup, queue, extras) {
|
|
|
14367
14383
|
checkRunRowsPruned = await extras.checkRunTrackingStore.pruneStale(ttlDays);
|
|
14368
14384
|
if (checkRunRowsPruned > 0) {
|
|
14369
14385
|
checkRunTrackingRowsPrunedTotal.add(checkRunRowsPruned);
|
|
14370
|
-
logger$
|
|
14386
|
+
logger$141.info("Pruned stale check-run-tracking rows", {
|
|
14371
14387
|
checkRunRowsPruned,
|
|
14372
14388
|
ttlDays
|
|
14373
14389
|
});
|
|
14374
14390
|
}
|
|
14375
14391
|
} catch (err) {
|
|
14376
|
-
logger$
|
|
14392
|
+
logger$141.error("Failed to prune stale check-run-tracking rows", { error: toErrorMessage(err) });
|
|
14377
14393
|
}
|
|
14378
14394
|
}
|
|
14379
14395
|
if (extras.retentionWindows) {
|
|
@@ -14401,20 +14417,20 @@ async function runCleanup(dedup, queue, extras) {
|
|
|
14401
14417
|
for (const runId of affectedRunIds) try {
|
|
14402
14418
|
await extras.executionTracker.completeRunIfAllJobsTerminal(runId);
|
|
14403
14419
|
} catch (err) {
|
|
14404
|
-
logger$
|
|
14420
|
+
logger$141.error("Error checking run completion after queue expiry", {
|
|
14405
14421
|
runId,
|
|
14406
14422
|
error: toErrorMessage(err)
|
|
14407
14423
|
});
|
|
14408
14424
|
}
|
|
14409
14425
|
}
|
|
14410
|
-
if (dedupDeleted > 0 || queueExpired > 0 || dispatchRowsPruned > 0 || logObjectsPruned > 0 || checkRunRowsPruned > 0) logger$
|
|
14426
|
+
if (dedupDeleted > 0 || queueExpired > 0 || dispatchRowsPruned > 0 || logObjectsPruned > 0 || checkRunRowsPruned > 0) logger$141.info("Cleanup completed", {
|
|
14411
14427
|
dedupDeleted,
|
|
14412
14428
|
queueExpired,
|
|
14413
14429
|
dispatchRowsPruned,
|
|
14414
14430
|
logObjectsPruned,
|
|
14415
14431
|
checkRunRowsPruned
|
|
14416
14432
|
});
|
|
14417
|
-
else logger$
|
|
14433
|
+
else logger$141.debug("Cleanup completed, nothing to clean");
|
|
14418
14434
|
return {
|
|
14419
14435
|
dedupDeleted,
|
|
14420
14436
|
queueExpired,
|
|
@@ -14433,14 +14449,14 @@ function createCleanupHandler(dedup, queue, extras) {
|
|
|
14433
14449
|
await runCleanup(dedup, queue, extras);
|
|
14434
14450
|
};
|
|
14435
14451
|
}
|
|
14436
|
-
var logger$
|
|
14452
|
+
var logger$141;
|
|
14437
14453
|
var init_cleanup$1 = __esmMin((() => {
|
|
14438
14454
|
init_log_retention();
|
|
14439
14455
|
init_request_idempotency();
|
|
14440
14456
|
init_prometheus();
|
|
14441
14457
|
init_terminalize_unroutable();
|
|
14442
14458
|
init_retention$1();
|
|
14443
|
-
logger$
|
|
14459
|
+
logger$141 = createLogger({ prefix: "cleanup" });
|
|
14444
14460
|
}));
|
|
14445
14461
|
//#endregion
|
|
14446
14462
|
//#region src/queue/unroutable-probe.ts
|
|
@@ -14482,7 +14498,7 @@ function createUnroutableProbeHandler(deps) {
|
|
|
14482
14498
|
if (job.unroutableSince !== null) {
|
|
14483
14499
|
await deps.queue.clearUnroutableState(job.id);
|
|
14484
14500
|
await deps.setRoutingReason(job.runId, job.jobName, null);
|
|
14485
|
-
logger$
|
|
14501
|
+
logger$140.info("Job became routable again; cleared unroutable state", {
|
|
14486
14502
|
runId: job.runId,
|
|
14487
14503
|
jobName: job.jobName
|
|
14488
14504
|
});
|
|
@@ -14492,7 +14508,7 @@ function createUnroutableProbeHandler(deps) {
|
|
|
14492
14508
|
if (job.unroutableSince === null) {
|
|
14493
14509
|
await deps.queue.markUnroutableSince(job.id, new Date(now));
|
|
14494
14510
|
await deps.setRoutingReason(job.runId, job.jobName, verdict.errorMessage);
|
|
14495
|
-
logger$
|
|
14511
|
+
logger$140.warn("Job is unroutable; starting grace window", {
|
|
14496
14512
|
runId: job.runId,
|
|
14497
14513
|
jobName: job.jobName,
|
|
14498
14514
|
graceMs,
|
|
@@ -14504,7 +14520,7 @@ function createUnroutableProbeHandler(deps) {
|
|
|
14504
14520
|
if (!await deps.queue.claimUnroutable(job.id)) continue;
|
|
14505
14521
|
await deps.terminalize(job);
|
|
14506
14522
|
deps.onFastFailed?.();
|
|
14507
|
-
logger$
|
|
14523
|
+
logger$140.warn("Job stayed unroutable for the whole grace window; failed it", {
|
|
14508
14524
|
runId: job.runId,
|
|
14509
14525
|
jobName: job.jobName,
|
|
14510
14526
|
graceMs
|
|
@@ -14513,10 +14529,10 @@ function createUnroutableProbeHandler(deps) {
|
|
|
14513
14529
|
}
|
|
14514
14530
|
};
|
|
14515
14531
|
}
|
|
14516
|
-
var logger$
|
|
14532
|
+
var logger$140, TICK_FLOOR_MS, TICK_CEILING_MS, DEFAULT_BATCH_LIMIT$1, DEFAULT_UNROUTABLE_GRACE_MS;
|
|
14517
14533
|
var init_unroutable_probe = __esmMin((() => {
|
|
14518
14534
|
init_terminalize_unroutable();
|
|
14519
|
-
logger$
|
|
14535
|
+
logger$140 = createLogger({ prefix: "unroutable-probe" });
|
|
14520
14536
|
TICK_FLOOR_MS = 5e3;
|
|
14521
14537
|
TICK_CEILING_MS = 3e4;
|
|
14522
14538
|
DEFAULT_BATCH_LIMIT$1 = 200;
|
|
@@ -14623,7 +14639,7 @@ function registerOrchestratorScheduledJob(opts) {
|
|
|
14623
14639
|
inFlight = (async () => {
|
|
14624
14640
|
const startedAt = Date.now();
|
|
14625
14641
|
try {
|
|
14626
|
-
logger$
|
|
14642
|
+
logger$139.info("Scheduled job starting", { job: opts.name });
|
|
14627
14643
|
await opts.handler();
|
|
14628
14644
|
const durationMs = Date.now() - startedAt;
|
|
14629
14645
|
const durationSeconds = durationMs / 1e3;
|
|
@@ -14637,7 +14653,7 @@ function registerOrchestratorScheduledJob(opts) {
|
|
|
14637
14653
|
const entry = jobMetricState.get(opts.name);
|
|
14638
14654
|
entry.consecutiveFailures = 0;
|
|
14639
14655
|
entry.lastSuccessAt = /* @__PURE__ */ new Date();
|
|
14640
|
-
logger$
|
|
14656
|
+
logger$139.info("Scheduled job completed", {
|
|
14641
14657
|
job: opts.name,
|
|
14642
14658
|
durationMs
|
|
14643
14659
|
});
|
|
@@ -14659,12 +14675,12 @@ function registerOrchestratorScheduledJob(opts) {
|
|
|
14659
14675
|
entry.consecutiveFailures += 1;
|
|
14660
14676
|
entry.lastFailureAt = /* @__PURE__ */ new Date();
|
|
14661
14677
|
jobConsecutiveFailures.set({ job: opts.name }, entry.consecutiveFailures);
|
|
14662
|
-
logger$
|
|
14678
|
+
logger$139.error("Scheduled job failed", {
|
|
14663
14679
|
job: opts.name,
|
|
14664
14680
|
durationMs,
|
|
14665
14681
|
error: errMsg
|
|
14666
14682
|
});
|
|
14667
|
-
if (opts.db) writeFailureAccessLog(opts, errMsg).catch((auditErr) => logger$
|
|
14683
|
+
if (opts.db) writeFailureAccessLog(opts, errMsg).catch((auditErr) => logger$139.error("Failed to write access_log for scheduled-job failure", {
|
|
14668
14684
|
job: opts.name,
|
|
14669
14685
|
error: toErrorMessage(auditErr)
|
|
14670
14686
|
}));
|
|
@@ -14681,10 +14697,10 @@ function registerOrchestratorScheduledJob(opts) {
|
|
|
14681
14697
|
}
|
|
14682
14698
|
const timer = setInterval(() => {
|
|
14683
14699
|
if (stopped) return;
|
|
14684
|
-
runDetached(logger$
|
|
14700
|
+
runDetached(logger$139, "Scheduled job run", () => runOnce(), { job: opts.name });
|
|
14685
14701
|
}, opts.intervalMs);
|
|
14686
14702
|
if (opts.unref !== false) timer.unref();
|
|
14687
|
-
if (opts.runOnStart === true) runDetached(logger$
|
|
14703
|
+
if (opts.runOnStart === true) runDetached(logger$139, "Scheduled job run", () => runOnce(), { job: opts.name });
|
|
14688
14704
|
const handle = {
|
|
14689
14705
|
name: opts.name,
|
|
14690
14706
|
stop() {
|
|
@@ -14698,7 +14714,7 @@ function registerOrchestratorScheduledJob(opts) {
|
|
|
14698
14714
|
}
|
|
14699
14715
|
};
|
|
14700
14716
|
handleRegistry.set(opts.name, handle);
|
|
14701
|
-
logger$
|
|
14717
|
+
logger$139.info("Scheduled job registered", {
|
|
14702
14718
|
job: opts.name,
|
|
14703
14719
|
intervalMs: opts.intervalMs,
|
|
14704
14720
|
runOnStart: opts.runOnStart === true
|
|
@@ -14722,11 +14738,11 @@ async function writeFailureAccessLog(opts, errorMessage) {
|
|
|
14722
14738
|
error_message: errorMessage
|
|
14723
14739
|
}).execute();
|
|
14724
14740
|
}
|
|
14725
|
-
var logger$
|
|
14741
|
+
var logger$139, OrchestratorScheduledJobName, ACCESS_LOG_ACTION_TICK_FAILURE, ACCESS_LOG_ACTION_TRIGGER, jobMetricState, handleRegistry;
|
|
14726
14742
|
var init_scheduled_job = __esmMin((() => {
|
|
14727
14743
|
init_scheduled_jobs();
|
|
14728
14744
|
init_run_detached();
|
|
14729
|
-
logger$
|
|
14745
|
+
logger$139 = createLogger({ prefix: "scheduled-job" });
|
|
14730
14746
|
OrchestratorScheduledJobName = z.enum([
|
|
14731
14747
|
"cleanup",
|
|
14732
14748
|
"orphan-secret-cleanup",
|
|
@@ -15338,13 +15354,13 @@ async function loadAccessLogRange(args) {
|
|
|
15338
15354
|
}
|
|
15339
15355
|
} catch (err) {
|
|
15340
15356
|
if (cursor?.source === "cold") {
|
|
15341
|
-
logger$
|
|
15357
|
+
logger$138.error("cold-store fetchRange failed during cold-resume; propagating", {
|
|
15342
15358
|
tenantId: tenantToScan,
|
|
15343
15359
|
error: toErrorMessage(err)
|
|
15344
15360
|
});
|
|
15345
15361
|
throw err;
|
|
15346
15362
|
}
|
|
15347
|
-
logger$
|
|
15363
|
+
logger$138.warn("cold-store fetchRange failed; returning hot rows only", {
|
|
15348
15364
|
tenantId: tenantToScan,
|
|
15349
15365
|
error: toErrorMessage(err)
|
|
15350
15366
|
});
|
|
@@ -15418,9 +15434,9 @@ function parseCursor$2(cursor) {
|
|
|
15418
15434
|
return null;
|
|
15419
15435
|
}
|
|
15420
15436
|
}
|
|
15421
|
-
var logger$
|
|
15437
|
+
var logger$138, SYNTHETIC_ORCH_TENANT;
|
|
15422
15438
|
var init_load_access_log_range = __esmMin((() => {
|
|
15423
|
-
logger$
|
|
15439
|
+
logger$138 = createLogger({ prefix: "load-access-log-range" });
|
|
15424
15440
|
SYNTHETIC_ORCH_TENANT = "__orchestrator__";
|
|
15425
15441
|
}));
|
|
15426
15442
|
//#endregion
|
|
@@ -16282,9 +16298,9 @@ var init_depth_refresher = __esmMin((() => {
|
|
|
16282
16298
|
function payloadFromObject(payload) {
|
|
16283
16299
|
return { raw: Buffer.from(JSON.stringify(payload), "utf-8") };
|
|
16284
16300
|
}
|
|
16285
|
-
var logger$
|
|
16301
|
+
var logger$137, EventLogWriter;
|
|
16286
16302
|
var init_event_log = __esmMin((() => {
|
|
16287
|
-
logger$
|
|
16303
|
+
logger$137 = createLogger({ prefix: "event-log" });
|
|
16288
16304
|
EventLogWriter = class EventLogWriter {
|
|
16289
16305
|
db;
|
|
16290
16306
|
logStorage;
|
|
@@ -16321,7 +16337,7 @@ var init_event_log = __esmMin((() => {
|
|
|
16321
16337
|
if (sizeBytes > maxBytes) {
|
|
16322
16338
|
payloadOmitted = true;
|
|
16323
16339
|
payloadOmittedReason = PayloadOmittedReason.enum.size_exceeded;
|
|
16324
|
-
logger$
|
|
16340
|
+
logger$137.warn("Webhook payload exceeds soft cap; recording metadata only", {
|
|
16325
16341
|
deliveryId: info.deliveryId,
|
|
16326
16342
|
sizeBytes,
|
|
16327
16343
|
maxBytes
|
|
@@ -16335,7 +16351,7 @@ var init_event_log = __esmMin((() => {
|
|
|
16335
16351
|
} catch (err) {
|
|
16336
16352
|
payloadOmitted = true;
|
|
16337
16353
|
payloadOmittedReason = PayloadOmittedReason.enum.storage_failed;
|
|
16338
|
-
logger$
|
|
16354
|
+
logger$137.error("Failed to upload event-log payload to object storage", {
|
|
16339
16355
|
deliveryId: info.deliveryId,
|
|
16340
16356
|
orgId: outcome.orgId,
|
|
16341
16357
|
error: toErrorMessage(err)
|
|
@@ -16376,7 +16392,7 @@ var init_event_log = __esmMin((() => {
|
|
|
16376
16392
|
});
|
|
16377
16393
|
}).execute();
|
|
16378
16394
|
} catch (err) {
|
|
16379
|
-
logger$
|
|
16395
|
+
logger$137.error("Failed to record event_log row", {
|
|
16380
16396
|
deliveryId: info.deliveryId,
|
|
16381
16397
|
orgId: outcome.orgId,
|
|
16382
16398
|
error: toErrorMessage(err)
|
|
@@ -16430,11 +16446,11 @@ var init_sampling_rate_limiter = __esmMin((() => {
|
|
|
16430
16446
|
}));
|
|
16431
16447
|
//#endregion
|
|
16432
16448
|
//#region src/audit/access-log.ts
|
|
16433
|
-
var logger$
|
|
16449
|
+
var logger$136, AccessLogWriter;
|
|
16434
16450
|
var init_access_log = __esmMin((() => {
|
|
16435
16451
|
init_load_access_log_range();
|
|
16436
16452
|
init_sampling_rate_limiter();
|
|
16437
|
-
logger$
|
|
16453
|
+
logger$136 = createLogger({ prefix: "access-log" });
|
|
16438
16454
|
AccessLogWriter = class {
|
|
16439
16455
|
db;
|
|
16440
16456
|
coldStore;
|
|
@@ -16455,7 +16471,7 @@ var init_access_log = __esmMin((() => {
|
|
|
16455
16471
|
try {
|
|
16456
16472
|
await this.insert(this.db, entry);
|
|
16457
16473
|
} catch (err) {
|
|
16458
|
-
logger$
|
|
16474
|
+
logger$136.error("access_log record failed", {
|
|
16459
16475
|
error: toErrorMessage(err),
|
|
16460
16476
|
action: entry.action,
|
|
16461
16477
|
source: entry.source
|
|
@@ -16552,7 +16568,7 @@ var init_access_log = __esmMin((() => {
|
|
|
16552
16568
|
fromTs: /* @__PURE__ */ new Date(0)
|
|
16553
16569
|
})) if (String(row.id) === id) return toAccessLogItem(row);
|
|
16554
16570
|
} catch (err) {
|
|
16555
|
-
logger$
|
|
16571
|
+
logger$136.warn("cold-store fetchRange failed for access-log getById", {
|
|
16556
16572
|
id,
|
|
16557
16573
|
tenantId,
|
|
16558
16574
|
error: toErrorMessage(err)
|
|
@@ -16629,14 +16645,14 @@ async function storeWebhookPayload(args) {
|
|
|
16629
16645
|
const backend = logStorage.constructor.name;
|
|
16630
16646
|
try {
|
|
16631
16647
|
await logStorage.append(payloadPath, payloadBytes);
|
|
16632
|
-
logger$
|
|
16648
|
+
logger$135.info("Stored webhook payload for run", {
|
|
16633
16649
|
runId,
|
|
16634
16650
|
payloadPath,
|
|
16635
16651
|
bytes: payloadBytes.length,
|
|
16636
16652
|
logStorageBackend: backend
|
|
16637
16653
|
});
|
|
16638
16654
|
} catch (err) {
|
|
16639
|
-
logger$
|
|
16655
|
+
logger$135.error("Failed to store webhook payload", {
|
|
16640
16656
|
runId,
|
|
16641
16657
|
payloadPath,
|
|
16642
16658
|
bytes: payloadBytes.length,
|
|
@@ -16645,9 +16661,9 @@ async function storeWebhookPayload(args) {
|
|
|
16645
16661
|
});
|
|
16646
16662
|
}
|
|
16647
16663
|
}
|
|
16648
|
-
var logger$
|
|
16664
|
+
var logger$135;
|
|
16649
16665
|
var init_webhook_payload_store = __esmMin((() => {
|
|
16650
|
-
logger$
|
|
16666
|
+
logger$135 = createLogger({ prefix: "pipeline" });
|
|
16651
16667
|
}));
|
|
16652
16668
|
//#endregion
|
|
16653
16669
|
//#region src/db/synthetic-job-ids.ts
|
|
@@ -16775,7 +16791,7 @@ async function runInvokeGate(deps, args) {
|
|
|
16775
16791
|
const reservedPrefix = reservedEventNamePrefix(event);
|
|
16776
16792
|
if (reservedPrefix) {
|
|
16777
16793
|
await deps.executionTracker.onJobStatus(runId, gateJobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, { error: reservedEventMessage(event, reservedPrefix) });
|
|
16778
|
-
logger$
|
|
16794
|
+
logger$134.warn("Invoke gate refused: reserved event-name prefix", {
|
|
16779
16795
|
runId,
|
|
16780
16796
|
gateJobName,
|
|
16781
16797
|
event,
|
|
@@ -16785,7 +16801,7 @@ async function runInvokeGate(deps, args) {
|
|
|
16785
16801
|
}
|
|
16786
16802
|
if (args.chainDepth >= deps.maxChainDepth) {
|
|
16787
16803
|
await deps.executionTracker.onJobStatus(runId, gateJobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, { error: `invoke chain depth ${args.chainDepth} reached the bound ${deps.maxChainDepth}; refusing to summon ${event}` });
|
|
16788
|
-
logger$
|
|
16804
|
+
logger$134.warn("Invoke gate refused: chain depth bound reached", {
|
|
16789
16805
|
runId,
|
|
16790
16806
|
gateJobName,
|
|
16791
16807
|
event,
|
|
@@ -16806,7 +16822,7 @@ async function runInvokeGate(deps, args) {
|
|
|
16806
16822
|
} catch (err) {
|
|
16807
16823
|
const reason = err instanceof SummonRefusedError ? err.message : `invoke gate summon failed: ${toErrorMessage(err)}`;
|
|
16808
16824
|
await deps.executionTracker.onJobStatus(runId, gateJobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, { error: reason });
|
|
16809
|
-
logger$
|
|
16825
|
+
logger$134.error("Invoke gate summon refused or failed", {
|
|
16810
16826
|
runId,
|
|
16811
16827
|
gateJobName,
|
|
16812
16828
|
event,
|
|
@@ -16820,7 +16836,7 @@ async function runInvokeGate(deps, args) {
|
|
|
16820
16836
|
return;
|
|
16821
16837
|
}
|
|
16822
16838
|
await createProxies(deps, args, runs);
|
|
16823
|
-
logger$
|
|
16839
|
+
logger$134.info("Invoke gate summoned subscribers", {
|
|
16824
16840
|
runId,
|
|
16825
16841
|
gateJobName,
|
|
16826
16842
|
event,
|
|
@@ -16837,7 +16853,7 @@ async function runInvokeGate(deps, args) {
|
|
|
16837
16853
|
async function releaseInvokeGate(deps, runId, gateJobName, params) {
|
|
16838
16854
|
const runRow = await deps.db.selectFrom("execution_runs").select(["repo_identifier", "chain_depth"]).where("run_id", "=", runId).executeTakeFirst();
|
|
16839
16855
|
if (!runRow) {
|
|
16840
|
-
logger$
|
|
16856
|
+
logger$134.warn("Invoke gate release: run not found; cannot summon", {
|
|
16841
16857
|
runId,
|
|
16842
16858
|
gateJobName
|
|
16843
16859
|
});
|
|
@@ -16858,7 +16874,7 @@ async function releaseInvokeGate(deps, runId, gateJobName, params) {
|
|
|
16858
16874
|
needs_satisfied: true,
|
|
16859
16875
|
ready_at: /* @__PURE__ */ new Date()
|
|
16860
16876
|
}).where("run_id", "=", runId).where("job_id", "=", gateJobId).execute();
|
|
16861
|
-
logger$
|
|
16877
|
+
logger$134.info("Invoke gate ready; summoning source-repo subscribers", {
|
|
16862
16878
|
runId,
|
|
16863
16879
|
gateJobName,
|
|
16864
16880
|
event: params.event,
|
|
@@ -16884,7 +16900,7 @@ async function handleZeroSubscribers(deps, args) {
|
|
|
16884
16900
|
const { runId, gateJobId, gateJobName, event, sourceRepo, optional } = args;
|
|
16885
16901
|
if (optional) {
|
|
16886
16902
|
await deps.executionTracker.onJobStatus(runId, gateJobId, ExecutionJobStatus.enum.success, Date.now());
|
|
16887
|
-
logger$
|
|
16903
|
+
logger$134.info("Invoke gate skipped: zero subscribers, optional", {
|
|
16888
16904
|
runId,
|
|
16889
16905
|
gateJobName,
|
|
16890
16906
|
event
|
|
@@ -16892,7 +16908,7 @@ async function handleZeroSubscribers(deps, args) {
|
|
|
16892
16908
|
return;
|
|
16893
16909
|
}
|
|
16894
16910
|
await deps.executionTracker.onJobStatus(runId, gateJobId, ExecutionJobStatus.enum.failed, Date.now(), void 0, { error: zeroSubscriberMessage(sourceRepo, event) });
|
|
16895
|
-
logger$
|
|
16911
|
+
logger$134.warn("Invoke gate failed: zero subscribers, required", {
|
|
16896
16912
|
runId,
|
|
16897
16913
|
gateJobName,
|
|
16898
16914
|
event
|
|
@@ -16925,10 +16941,10 @@ async function createProxies(deps, args, runs) {
|
|
|
16925
16941
|
await deps.executionTracker.reconcileSummonedRunIfTerminal(runs[i].runId);
|
|
16926
16942
|
}
|
|
16927
16943
|
}
|
|
16928
|
-
var logger$
|
|
16944
|
+
var logger$134, SummonRefusedError;
|
|
16929
16945
|
var init_invoke_gate = __esmMin((() => {
|
|
16930
16946
|
init_types$2();
|
|
16931
|
-
logger$
|
|
16947
|
+
logger$134 = createLogger({ prefix: "invoke-gate" });
|
|
16932
16948
|
SummonRefusedError = class extends Error {
|
|
16933
16949
|
constructor(reason) {
|
|
16934
16950
|
super(reason);
|
|
@@ -17087,7 +17103,7 @@ async function releaseQueuedHolds(args) {
|
|
|
17087
17103
|
await onJobRelease(await heldRunStore.release(orgId, hold.id));
|
|
17088
17104
|
resumed++;
|
|
17089
17105
|
} catch (err) {
|
|
17090
|
-
logger$
|
|
17106
|
+
logger$133.error("Failed to resume a released queued hold", {
|
|
17091
17107
|
holdId: hold.id,
|
|
17092
17108
|
runId: hold.run_id,
|
|
17093
17109
|
jobId: hold.job_id,
|
|
@@ -17095,7 +17111,7 @@ async function releaseQueuedHolds(args) {
|
|
|
17095
17111
|
error: toErrorMessage(err)
|
|
17096
17112
|
});
|
|
17097
17113
|
}
|
|
17098
|
-
if (resumed > 0) logger$
|
|
17114
|
+
if (resumed > 0) logger$133.info("Released queued concurrency holds", {
|
|
17099
17115
|
concurrencyGroup,
|
|
17100
17116
|
resumed,
|
|
17101
17117
|
queued: queued.length,
|
|
@@ -17103,10 +17119,10 @@ async function releaseQueuedHolds(args) {
|
|
|
17103
17119
|
});
|
|
17104
17120
|
return resumed;
|
|
17105
17121
|
}
|
|
17106
|
-
var logger$
|
|
17122
|
+
var logger$133;
|
|
17107
17123
|
var init_release_queued_holds = __esmMin((() => {
|
|
17108
17124
|
init_synthetic_job_ids();
|
|
17109
|
-
logger$
|
|
17125
|
+
logger$133 = createLogger({ prefix: "release-queued-holds" });
|
|
17110
17126
|
}));
|
|
17111
17127
|
//#endregion
|
|
17112
17128
|
//#region src/contexts/protection/branch-gate.ts
|
|
@@ -17845,7 +17861,7 @@ async function warnUnboundJobContexts(args) {
|
|
|
17845
17861
|
} catch {
|
|
17846
17862
|
continue;
|
|
17847
17863
|
}
|
|
17848
|
-
if (bindings === 0) logger$
|
|
17864
|
+
if (bindings === 0) logger$132.warn(UNBOUND_JOB_CONTEXT_MESSAGE, {
|
|
17849
17865
|
orgId,
|
|
17850
17866
|
context: name,
|
|
17851
17867
|
contextId: env.id,
|
|
@@ -17855,9 +17871,9 @@ async function warnUnboundJobContexts(args) {
|
|
|
17855
17871
|
});
|
|
17856
17872
|
}
|
|
17857
17873
|
}
|
|
17858
|
-
var logger$
|
|
17874
|
+
var logger$132, UNBOUND_JOB_CONTEXT_MESSAGE;
|
|
17859
17875
|
var init_unbound_job_context = __esmMin((() => {
|
|
17860
|
-
logger$
|
|
17876
|
+
logger$132 = createLogger({ prefix: "job-contexts" });
|
|
17861
17877
|
UNBOUND_JOB_CONTEXT_MESSAGE = "Job binds a context that has no scope binding; it receives no secrets from it";
|
|
17862
17878
|
}));
|
|
17863
17879
|
//#endregion
|
|
@@ -17917,7 +17933,7 @@ async function resolveRegistryAuthInto(args) {
|
|
|
17917
17933
|
const { container, dispatchCtx, trustTier, admittedContextIds } = registryAuth;
|
|
17918
17934
|
if (isUntrustedTier(trustTier)) {
|
|
17919
17935
|
containerRegistryAuthContributorStrippedTotal.add(1, { trust_tier: trustTier ?? "unknown" });
|
|
17920
|
-
logger$
|
|
17936
|
+
logger$131.info("Container registry auth withheld from an untrusted contributor", {
|
|
17921
17937
|
...log,
|
|
17922
17938
|
trustTier: trustTier ?? "unknown"
|
|
17923
17939
|
});
|
|
@@ -17948,7 +17964,7 @@ async function resolveRegistryAuthInto(args) {
|
|
|
17948
17964
|
} });
|
|
17949
17965
|
} catch (err) {
|
|
17950
17966
|
if ((registryAuth.onRefusal ?? "throw") === "throw" || lookupFailure !== void 0) throw err;
|
|
17951
|
-
logger$
|
|
17967
|
+
logger$131.warn("Container registry auth refused; dispatching without it", {
|
|
17952
17968
|
...log,
|
|
17953
17969
|
error: toErrorMessage(err)
|
|
17954
17970
|
});
|
|
@@ -18041,7 +18057,7 @@ async function resolveReleasedJobInput(args) {
|
|
|
18041
18057
|
});
|
|
18042
18058
|
const jobConfig = jobInput.jobConfig;
|
|
18043
18059
|
const fields = contextDataConfigFields(data, jobConfig.secrets);
|
|
18044
|
-
logger$
|
|
18060
|
+
logger$131.info("Resolved context data for a released job", {
|
|
18045
18061
|
...log,
|
|
18046
18062
|
contexts: entries.map((e) => e.name)
|
|
18047
18063
|
});
|
|
@@ -18056,7 +18072,7 @@ async function resolveReleasedJobInput(args) {
|
|
|
18056
18072
|
/** Log a release-time resolution failure; the caller fails the job. */
|
|
18057
18073
|
function logReleaseResolutionFailure(jobInput, err) {
|
|
18058
18074
|
const message = `context data could not be resolved on release: ${toErrorMessage(err)}`;
|
|
18059
|
-
logger$
|
|
18075
|
+
logger$131.error("Released job failed: context data resolution", {
|
|
18060
18076
|
runId: jobInput.runId,
|
|
18061
18077
|
workflow: jobInput.workflowName,
|
|
18062
18078
|
job: jobInput.jobName,
|
|
@@ -18064,7 +18080,7 @@ function logReleaseResolutionFailure(jobInput, err) {
|
|
|
18064
18080
|
});
|
|
18065
18081
|
return message;
|
|
18066
18082
|
}
|
|
18067
|
-
var logger$
|
|
18083
|
+
var logger$131, DeferredContextResolutionSchema, CONTEXT_DATA_CONFIG_KEYS;
|
|
18068
18084
|
var init_held_context_data = __esmMin((() => {
|
|
18069
18085
|
init_context_store();
|
|
18070
18086
|
init_prometheus();
|
|
@@ -18073,7 +18089,7 @@ var init_held_context_data = __esmMin((() => {
|
|
|
18073
18089
|
init_trust_tier();
|
|
18074
18090
|
init_job_contexts();
|
|
18075
18091
|
init_unbound_job_context();
|
|
18076
|
-
logger$
|
|
18092
|
+
logger$131 = createLogger({ prefix: "held-context-data" });
|
|
18077
18093
|
DeferredContextResolutionSchema = z.object({
|
|
18078
18094
|
/**
|
|
18079
18095
|
* The contexts that admitted the job's reject rules, in merge order. The id
|
|
@@ -19130,9 +19146,9 @@ var init_commit_message = __esmMin((() => {}));
|
|
|
19130
19146
|
* webhook normalization, lock file fetching, changed files retrieval,
|
|
19131
19147
|
* clone token creation, and URL building.
|
|
19132
19148
|
*/
|
|
19133
|
-
var logger$
|
|
19149
|
+
var logger$130, ProviderRegistry;
|
|
19134
19150
|
var init_provider_registry = __esmMin((() => {
|
|
19135
|
-
logger$
|
|
19151
|
+
logger$130 = createLogger({ prefix: "provider-registry" });
|
|
19136
19152
|
ProviderRegistry = class {
|
|
19137
19153
|
bundles = /* @__PURE__ */ new Map();
|
|
19138
19154
|
registerByRoutingKey(routingKey, bundle) {
|
|
@@ -19185,7 +19201,7 @@ var init_provider_registry = __esmMin((() => {
|
|
|
19185
19201
|
*/
|
|
19186
19202
|
unregister(routingKey) {
|
|
19187
19203
|
const existed = this.bundles.delete(routingKey);
|
|
19188
|
-
if (existed) logger$
|
|
19204
|
+
if (existed) logger$130.info("Provider bundle unregistered", { routingKey });
|
|
19189
19205
|
return existed;
|
|
19190
19206
|
}
|
|
19191
19207
|
/**
|
|
@@ -19200,7 +19216,7 @@ var init_provider_registry = __esmMin((() => {
|
|
|
19200
19216
|
clear() {
|
|
19201
19217
|
const count = this.bundles.size;
|
|
19202
19218
|
this.bundles.clear();
|
|
19203
|
-
if (count > 0) logger$
|
|
19219
|
+
if (count > 0) logger$130.info("Provider registry cleared", { previousCount: count });
|
|
19204
19220
|
}
|
|
19205
19221
|
};
|
|
19206
19222
|
}));
|
|
@@ -19459,7 +19475,7 @@ var init_resume_router = __esmMin((() => {}));
|
|
|
19459
19475
|
function findIdentityLink(identityLinks, provider, providerUsername, providerUserId) {
|
|
19460
19476
|
if (providerUserId === void 0 || providerUserId.length === 0) {
|
|
19461
19477
|
trustMatchRefusedNoIdTotal.add(1, { reason: "event_missing" });
|
|
19462
|
-
logger$
|
|
19478
|
+
logger$129.warn("Trust match refused: webhook event has no provider numeric id", {
|
|
19463
19479
|
provider,
|
|
19464
19480
|
providerUsername
|
|
19465
19481
|
});
|
|
@@ -19471,7 +19487,7 @@ function findIdentityLink(identityLinks, provider, providerUsername, providerUse
|
|
|
19471
19487
|
if (byUsername) {
|
|
19472
19488
|
const reason = byUsername.providerUserId === null || byUsername.providerUserId === void 0 ? "link_missing" : "id_mismatch";
|
|
19473
19489
|
trustMatchRefusedNoIdTotal.add(1, { reason });
|
|
19474
|
-
logger$
|
|
19490
|
+
logger$129.warn("Trust match refused: numeric id missing on link or did not match event", {
|
|
19475
19491
|
provider,
|
|
19476
19492
|
providerUsername,
|
|
19477
19493
|
providerUserId,
|
|
@@ -19481,10 +19497,10 @@ function findIdentityLink(identityLinks, provider, providerUsername, providerUse
|
|
|
19481
19497
|
}
|
|
19482
19498
|
return null;
|
|
19483
19499
|
}
|
|
19484
|
-
var logger$
|
|
19500
|
+
var logger$129;
|
|
19485
19501
|
var init_identity_link = __esmMin((() => {
|
|
19486
19502
|
init_prometheus();
|
|
19487
|
-
logger$
|
|
19503
|
+
logger$129 = createLogger({ prefix: "identity-link" });
|
|
19488
19504
|
}));
|
|
19489
19505
|
//#endregion
|
|
19490
19506
|
//#region src/pipeline/security-hold-check.ts
|
|
@@ -19672,7 +19688,7 @@ async function settleSecurityHoldCheck(args) {
|
|
|
19672
19688
|
excludeHoldIds: [hold.id, ...args.excludeHoldIds ?? []]
|
|
19673
19689
|
});
|
|
19674
19690
|
if (contender) {
|
|
19675
|
-
logger$
|
|
19691
|
+
logger$128.info("Left the security check pending: another hold on the commit still owns it", {
|
|
19676
19692
|
runId: hold.run_id,
|
|
19677
19693
|
holdId: hold.id,
|
|
19678
19694
|
contendingHoldId: contender
|
|
@@ -19696,7 +19712,7 @@ async function settleSecurityHoldCheck(args) {
|
|
|
19696
19712
|
commit
|
|
19697
19713
|
};
|
|
19698
19714
|
} catch (err) {
|
|
19699
|
-
logger$
|
|
19715
|
+
logger$128.warn("Failed to complete the security check run of an ended hold", {
|
|
19700
19716
|
runId: hold.run_id,
|
|
19701
19717
|
holdId: hold.id,
|
|
19702
19718
|
error: toErrorMessage(err)
|
|
@@ -19745,9 +19761,9 @@ async function findPendingCheckOwner(db, args) {
|
|
|
19745
19761
|
}
|
|
19746
19762
|
return null;
|
|
19747
19763
|
}
|
|
19748
|
-
var logger$
|
|
19764
|
+
var logger$128, HOLD_OUTCOME_CONCLUSION, HOLD_OUTCOME_TITLE, PUSH_AGAIN;
|
|
19749
19765
|
var init_security_hold_check = __esmMin((() => {
|
|
19750
|
-
logger$
|
|
19766
|
+
logger$128 = createLogger({ prefix: "security-hold-check" });
|
|
19751
19767
|
HOLD_OUTCOME_CONCLUSION = {
|
|
19752
19768
|
["approved"]: CheckRunConclusion.enum.success,
|
|
19753
19769
|
["rejected"]: CheckRunConclusion.enum.cancelled,
|
|
@@ -19818,14 +19834,14 @@ async function handleApprovalComment(params) {
|
|
|
19818
19834
|
const { commentBody, commenterUsername, commenterUserId, provider, repoIdentifier, prNumber, orgId, identityLinks, orgMemberPermissions, heldRunStore } = params;
|
|
19819
19835
|
const command = parseKiciCommand(commentBody);
|
|
19820
19836
|
if (!command) return { handled: false };
|
|
19821
|
-
logger$
|
|
19837
|
+
logger$127.info("Processing /kici command", {
|
|
19822
19838
|
action: command.action,
|
|
19823
19839
|
commenter: commenterUsername,
|
|
19824
19840
|
runId: command.runId
|
|
19825
19841
|
});
|
|
19826
19842
|
const identityLink = findIdentityLink(identityLinks, provider, commenterUsername, commenterUserId);
|
|
19827
19843
|
if (!identityLink) {
|
|
19828
|
-
logger$
|
|
19844
|
+
logger$127.info("No identity link for commenter", {
|
|
19829
19845
|
commenter: commenterUsername,
|
|
19830
19846
|
provider
|
|
19831
19847
|
});
|
|
@@ -19836,7 +19852,7 @@ async function handleApprovalComment(params) {
|
|
|
19836
19852
|
}
|
|
19837
19853
|
const ciTrustLevel = orgMemberPermissions.get(identityLink.userId) ?? "none";
|
|
19838
19854
|
if (ciTrustLevel !== "write" && ciTrustLevel !== "admin") {
|
|
19839
|
-
logger$
|
|
19855
|
+
logger$127.info("Insufficient ci_trust level", {
|
|
19840
19856
|
commenter: commenterUsername,
|
|
19841
19857
|
userId: identityLink.userId,
|
|
19842
19858
|
ciTrustLevel
|
|
@@ -19848,7 +19864,7 @@ async function handleApprovalComment(params) {
|
|
|
19848
19864
|
}
|
|
19849
19865
|
const pendingHolds = await heldRunStore.listPendingSecurityHoldsForPr(orgId, repoIdentifier, prNumber);
|
|
19850
19866
|
if (pendingHolds.length === 0) {
|
|
19851
|
-
logger$
|
|
19867
|
+
logger$127.info("No pending security holds found for PR", {
|
|
19852
19868
|
orgId,
|
|
19853
19869
|
repoIdentifier,
|
|
19854
19870
|
prNumber
|
|
@@ -19857,7 +19873,7 @@ async function handleApprovalComment(params) {
|
|
|
19857
19873
|
}
|
|
19858
19874
|
const targetHolds = command.runId ? pendingHolds.filter((h) => h.run_id === command.runId) : pendingHolds;
|
|
19859
19875
|
if (targetHolds.length === 0) {
|
|
19860
|
-
logger$
|
|
19876
|
+
logger$127.info("No matching security holds found", {
|
|
19861
19877
|
orgId,
|
|
19862
19878
|
runId: command.runId
|
|
19863
19879
|
});
|
|
@@ -19867,7 +19883,7 @@ async function handleApprovalComment(params) {
|
|
|
19867
19883
|
for (const hold of targetHolds) try {
|
|
19868
19884
|
if (approved) {
|
|
19869
19885
|
await heldRunStore.approveByQueueType(orgId, hold.id, identityLink.userId, "security");
|
|
19870
|
-
logger$
|
|
19886
|
+
logger$127.info("Security hold approved", {
|
|
19871
19887
|
heldRunId: hold.id,
|
|
19872
19888
|
runId: hold.run_id,
|
|
19873
19889
|
approvedBy: identityLink.userId
|
|
@@ -19884,7 +19900,7 @@ async function handleApprovalComment(params) {
|
|
|
19884
19900
|
onJobRelease: params.onJobRelease,
|
|
19885
19901
|
onWorkflowRelease: params.onWorkflowRelease
|
|
19886
19902
|
}).catch((err) => {
|
|
19887
|
-
logger$
|
|
19903
|
+
logger$127.error("Failed to resume a held run after its security hold was approved", {
|
|
19888
19904
|
heldRunId: hold.id,
|
|
19889
19905
|
runId: hold.run_id,
|
|
19890
19906
|
jobId: hold.job_id,
|
|
@@ -19895,7 +19911,7 @@ async function handleApprovalComment(params) {
|
|
|
19895
19911
|
} else {
|
|
19896
19912
|
const rejectReason = `Rejected by ${commenterUsername} via ${REJECT_COMMAND}`;
|
|
19897
19913
|
await heldRunStore.reject(orgId, hold.id, rejectReason);
|
|
19898
|
-
logger$
|
|
19914
|
+
logger$127.info("Security hold rejected", {
|
|
19899
19915
|
heldRunId: hold.id,
|
|
19900
19916
|
runId: hold.run_id,
|
|
19901
19917
|
rejectedBy: commenterUsername
|
|
@@ -19904,7 +19920,7 @@ async function handleApprovalComment(params) {
|
|
|
19904
19920
|
if (hold.hold_scope === HoldScope.enum.workflow && params.onWorkflowReject) await params.onWorkflowReject(hold, rejectReason).then((posted) => {
|
|
19905
19921
|
securityCheckWritten = posted;
|
|
19906
19922
|
}).catch((err) => {
|
|
19907
|
-
logger$
|
|
19923
|
+
logger$127.error("Failed to cancel a held run after its security hold was rejected", {
|
|
19908
19924
|
heldRunId: hold.id,
|
|
19909
19925
|
runId: hold.run_id,
|
|
19910
19926
|
error: toErrorMessage(err)
|
|
@@ -19919,7 +19935,7 @@ async function handleApprovalComment(params) {
|
|
|
19919
19935
|
});
|
|
19920
19936
|
}
|
|
19921
19937
|
} catch (err) {
|
|
19922
|
-
logger$
|
|
19938
|
+
logger$127.error("Failed to process security hold", {
|
|
19923
19939
|
heldRunId: hold.id,
|
|
19924
19940
|
action: command.action,
|
|
19925
19941
|
error: toErrorMessage(err)
|
|
@@ -19927,12 +19943,12 @@ async function handleApprovalComment(params) {
|
|
|
19927
19943
|
}
|
|
19928
19944
|
return { handled: true };
|
|
19929
19945
|
}
|
|
19930
|
-
var logger$
|
|
19946
|
+
var logger$127, APPROVE_COMMAND, REJECT_COMMAND;
|
|
19931
19947
|
var init_comment_handler = __esmMin((() => {
|
|
19932
19948
|
init_resume_router();
|
|
19933
19949
|
init_identity_link();
|
|
19934
19950
|
init_security_hold_check();
|
|
19935
|
-
logger$
|
|
19951
|
+
logger$127 = createLogger({ prefix: "comment-handler" });
|
|
19936
19952
|
APPROVE_COMMAND = "/kici approve";
|
|
19937
19953
|
REJECT_COMMAND = "/kici reject";
|
|
19938
19954
|
}));
|
|
@@ -20052,12 +20068,12 @@ async function dispatchGlobalCandidateViaPipeline(args) {
|
|
|
20052
20068
|
sourceRepo: args.sourceRepoIdentifier
|
|
20053
20069
|
};
|
|
20054
20070
|
if (args.resolved.jobs.length === 0) {
|
|
20055
|
-
logger$
|
|
20071
|
+
logger$126.warn("Organization-wide workflow cleared with no jobs to dispatch", logFields);
|
|
20056
20072
|
return;
|
|
20057
20073
|
}
|
|
20058
20074
|
const workflowBundle = args.deps.providerRegistry.getByRoutingKey(reg.routingKey);
|
|
20059
20075
|
if (!workflowBundle) {
|
|
20060
|
-
logger$
|
|
20076
|
+
logger$126.warn("Global workflow skipped: no provider bundle for its workflow repository", {
|
|
20061
20077
|
...logFields,
|
|
20062
20078
|
workflowRoutingKey: reg.routingKey
|
|
20063
20079
|
});
|
|
@@ -20067,7 +20083,7 @@ async function dispatchGlobalCandidateViaPipeline(args) {
|
|
|
20067
20083
|
try {
|
|
20068
20084
|
workflowCredentials = await mintWorkflowRepoCredentials(workflowBundle, reg);
|
|
20069
20085
|
} catch (err) {
|
|
20070
|
-
logger$
|
|
20086
|
+
logger$126.warn("Global workflow skipped: cannot mint credentials for its workflow repository", {
|
|
20071
20087
|
...logFields,
|
|
20072
20088
|
error: toErrorMessage(err)
|
|
20073
20089
|
});
|
|
@@ -20113,10 +20129,10 @@ async function dispatchGlobalCandidateViaPipeline(args) {
|
|
|
20113
20129
|
});
|
|
20114
20130
|
return runId;
|
|
20115
20131
|
}
|
|
20116
|
-
var logger$
|
|
20132
|
+
var logger$126, UNKNOWN_SOURCE_FILE;
|
|
20117
20133
|
var init_global_dispatch = __esmMin((() => {
|
|
20118
20134
|
init_dispatch_matched_workflow();
|
|
20119
|
-
logger$
|
|
20135
|
+
logger$126 = createLogger({ prefix: "global-dispatch" });
|
|
20120
20136
|
UNKNOWN_SOURCE_FILE = ".kici/workflows/unknown.ts";
|
|
20121
20137
|
}));
|
|
20122
20138
|
//#endregion
|
|
@@ -20152,7 +20168,7 @@ async function completeUndispatchedHoldChecks(args) {
|
|
|
20152
20168
|
if (!pending) return;
|
|
20153
20169
|
const [owner, repo] = pending.repoIdentifier.split("/");
|
|
20154
20170
|
if (!owner || !repo) {
|
|
20155
|
-
logger$
|
|
20171
|
+
logger$125.warn("Cannot complete hold check runs: repo identifier is not owner/repo", {
|
|
20156
20172
|
runId,
|
|
20157
20173
|
repoIdentifier: pending.repoIdentifier
|
|
20158
20174
|
});
|
|
@@ -20172,16 +20188,16 @@ async function completeUndispatchedHoldChecks(args) {
|
|
|
20172
20188
|
summary: args.summary
|
|
20173
20189
|
});
|
|
20174
20190
|
} catch (err) {
|
|
20175
|
-
logger$
|
|
20191
|
+
logger$125.warn("Failed to complete check runs for an undispatched hold", {
|
|
20176
20192
|
runId,
|
|
20177
20193
|
error: toErrorMessage(err)
|
|
20178
20194
|
});
|
|
20179
20195
|
}
|
|
20180
20196
|
}
|
|
20181
|
-
var logger$
|
|
20197
|
+
var logger$125;
|
|
20182
20198
|
var init_undispatched_hold_checks = __esmMin((() => {
|
|
20183
20199
|
init_pending_workflow_context();
|
|
20184
|
-
logger$
|
|
20200
|
+
logger$125 = createLogger({ prefix: "undispatched-hold-checks" });
|
|
20185
20201
|
}));
|
|
20186
20202
|
//#endregion
|
|
20187
20203
|
//#region src/cache/global-eval-round-cache.ts
|
|
@@ -20295,13 +20311,17 @@ var KEY_SEPARATOR$1, GlobalEvalRoundCache, init_global_eval_round_cache = __esmM
|
|
|
20295
20311
|
}
|
|
20296
20312
|
};
|
|
20297
20313
|
}));
|
|
20298
|
-
var init_replay_chunker = __esmMin((() => {}))
|
|
20314
|
+
var init_replay_chunker = __esmMin((() => {}));
|
|
20315
|
+
var init_outbound_frame_guard = __esmMin((() => {
|
|
20316
|
+
createLogger({ prefix: "platform-client" });
|
|
20317
|
+
})), init_event_buffer = __esmMin((() => {})), init_relay_buffer = __esmMin((() => {
|
|
20299
20318
|
createLogger({ prefix: "relay-buffer" });
|
|
20300
20319
|
})), ADMITTED_PIPELINE_LIFETIME_MS, DASHBOARD_FRAME_TYPES;
|
|
20301
20320
|
var init_platform_client = __esmMin((() => {
|
|
20302
20321
|
init_replay_chunker();
|
|
20303
20322
|
init_prometheus();
|
|
20304
20323
|
init_run_detached();
|
|
20324
|
+
init_outbound_frame_guard();
|
|
20305
20325
|
init_event_buffer();
|
|
20306
20326
|
init_relay_buffer();
|
|
20307
20327
|
createLogger({ prefix: "platform-client" });
|
|
@@ -20726,22 +20746,6 @@ function roundDecidedNothing(verdicts) {
|
|
|
20726
20746
|
const collected = [...verdicts.values()];
|
|
20727
20747
|
return collected.length > 0 && collected.every((verdict) => verdict.indeterminate === true);
|
|
20728
20748
|
}
|
|
20729
|
-
/**
|
|
20730
|
-
* Truncate `text` to at most `max` UTF-16 code units, marking that it was cut.
|
|
20731
|
-
*
|
|
20732
|
-
* - A `max` no larger than the marker returns the truncated marker, never a
|
|
20733
|
-
* string longer than `max`.
|
|
20734
|
-
* - The cut backs off one unit rather than split a surrogate pair.
|
|
20735
|
-
*/
|
|
20736
|
-
function truncateReasonText(text, max) {
|
|
20737
|
-
if (max <= 0) return "";
|
|
20738
|
-
if (text.length <= max) return text;
|
|
20739
|
-
if (max <= TRUNCATION_MARKER.length) return TRUNCATION_MARKER.slice(0, max);
|
|
20740
|
-
let cut = max - TRUNCATION_MARKER.length;
|
|
20741
|
-
const last = text.charCodeAt(cut - 1);
|
|
20742
|
-
if (last >= 55296 && last <= 56319) cut -= 1;
|
|
20743
|
-
return text.slice(0, cut) + TRUNCATION_MARKER;
|
|
20744
|
-
}
|
|
20745
20749
|
/** The distinct reasons a round gave, for the error a failed or partial round reports. */
|
|
20746
20750
|
function distinctReasons(verdicts) {
|
|
20747
20751
|
const reasons = /* @__PURE__ */ new Set();
|
|
@@ -21002,7 +21006,7 @@ function recordVerdictOutcomes(verdicts) {
|
|
|
21002
21006
|
globalEvalVerdictsTotal.add(1, { outcome });
|
|
21003
21007
|
}
|
|
21004
21008
|
}
|
|
21005
|
-
var logger$121, ROUND_JOB_PREFIX, FALLBACK_TARGET, ACCEPTED_DISPATCH_STATUSES, KEY_SEPARATOR, GlobalEvalVerdictOutcome, GlobalEvalRoundResultLabel, GlobalEvalCacheLookupResult, MIN_GLOBAL_EVAL_AGENT_VERSION, WAIT_CEILING_MULTIPLIER, MAX_RAISED_WAIT_CEILING_MS, MIN_WAIT_CEILING_RATIO, MAX_ROUND_REASON_CHARS,
|
|
21009
|
+
var logger$121, ROUND_JOB_PREFIX, FALLBACK_TARGET, ACCEPTED_DISPATCH_STATUSES, KEY_SEPARATOR, GlobalEvalVerdictOutcome, GlobalEvalRoundResultLabel, GlobalEvalCacheLookupResult, MIN_GLOBAL_EVAL_AGENT_VERSION, WAIT_CEILING_MULTIPLIER, MAX_RAISED_WAIT_CEILING_MS, MIN_WAIT_CEILING_RATIO, MAX_ROUND_REASON_CHARS, ROUND_MAX_ATTEMPTS;
|
|
21006
21010
|
var init_global_eval_round = __esmMin((() => {
|
|
21007
21011
|
init_prometheus();
|
|
21008
21012
|
init_global_eval_round_cache();
|
|
@@ -21039,7 +21043,6 @@ var init_global_eval_round = __esmMin((() => {
|
|
|
21039
21043
|
MAX_RAISED_WAIT_CEILING_MS = ADMITTED_PIPELINE_LIFETIME_MS;
|
|
21040
21044
|
MIN_WAIT_CEILING_RATIO = 1.25;
|
|
21041
21045
|
MAX_ROUND_REASON_CHARS = 4e3;
|
|
21042
|
-
TRUNCATION_MARKER = "… (truncated)";
|
|
21043
21046
|
ROUND_MAX_ATTEMPTS = 2;
|
|
21044
21047
|
}));
|
|
21045
21048
|
//#endregion
|
|
@@ -47183,7 +47186,8 @@ function createAgentWsHandler(deps) {
|
|
|
47183
47186
|
activeJobs: msg.activeJobs
|
|
47184
47187
|
});
|
|
47185
47188
|
if (msg.activeJobs === 0) registry.clearBusyHeld(agentId);
|
|
47186
|
-
if (
|
|
47189
|
+
if (msg.draining === true) registry.markDrainReported(agentId);
|
|
47190
|
+
if (!entry.draining && entry.activeJobs < entry.maxConcurrency) await dispatcher.onAgentAvailable(agentId);
|
|
47187
47191
|
}
|
|
47188
47192
|
break;
|
|
47189
47193
|
}
|
|
@@ -63420,6 +63424,35 @@ async function provisionRemoteSource(db, params) {
|
|
|
63420
63424
|
}
|
|
63421
63425
|
var init_remote_source_store = __esmMin((() => {}));
|
|
63422
63426
|
//#endregion
|
|
63427
|
+
//#region src/ws/test-run-log-cursor.ts
|
|
63428
|
+
/**
|
|
63429
|
+
* The opaque per-file cursor of a `kici run remote` log stream.
|
|
63430
|
+
*
|
|
63431
|
+
* It records, for each of a run's log files, how many lines the caller already
|
|
63432
|
+
* received: `{ "<file>": <lines> }`, JSON-encoded, then base64url so it fits a
|
|
63433
|
+
* query string. The file key is the path below `executions/<runId>/`. A file
|
|
63434
|
+
* that appears later starts at line 0 wherever it sorts, so no line is lost or
|
|
63435
|
+
* sent twice when a step's file sorts before a file that was already read.
|
|
63436
|
+
*/
|
|
63437
|
+
function encodeRunLogCursor(offsets) {
|
|
63438
|
+
return Buffer.from(JSON.stringify(offsets), "utf8").toString("base64url");
|
|
63439
|
+
}
|
|
63440
|
+
/** Decode a cursor. The start token decodes to no offsets; malformed input to null. */
|
|
63441
|
+
function decodeRunLogCursor(cursor) {
|
|
63442
|
+
if (cursor === TEST_RUN_LOG_CURSOR_START) return {};
|
|
63443
|
+
try {
|
|
63444
|
+
const decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
|
63445
|
+
const parsed = offsetsSchema.safeParse(decoded);
|
|
63446
|
+
return parsed.success ? parsed.data : null;
|
|
63447
|
+
} catch {
|
|
63448
|
+
return null;
|
|
63449
|
+
}
|
|
63450
|
+
}
|
|
63451
|
+
var offsetsSchema;
|
|
63452
|
+
var init_test_run_log_cursor = __esmMin((() => {
|
|
63453
|
+
offsetsSchema = z.record(z.string(), z.number().int().nonnegative());
|
|
63454
|
+
}));
|
|
63455
|
+
//#endregion
|
|
63423
63456
|
//#region src/routes/uploads.ts
|
|
63424
63457
|
/**
|
|
63425
63458
|
* Test-run upload provisioning. The developer encrypts the overlay tarball with
|
|
@@ -63597,7 +63630,10 @@ async function resolveLockFileForTest(input, deps, runId) {
|
|
|
63597
63630
|
function selectMatchedDecisions(input, fullLockFile, simulatedEvent) {
|
|
63598
63631
|
let decisions;
|
|
63599
63632
|
if (input.workflowName) {
|
|
63600
|
-
if (!fullLockFile.workflows.find((w) => w.name === input.workflowName))
|
|
63633
|
+
if (!fullLockFile.workflows.find((w) => w.name === input.workflowName)) {
|
|
63634
|
+
const available = fullLockFile.workflows.map((w) => w.name).join(", ");
|
|
63635
|
+
return { rejected: `Workflow '${input.workflowName}' not found in lock file. Available workflows: ${available || "(none)"}` };
|
|
63636
|
+
}
|
|
63601
63637
|
decisions = [createWorkflowDecision(input.workflowName, true, [], void 0, "Direct test run")];
|
|
63602
63638
|
} else decisions = matchAllWorkflows(fullLockFile.workflows, simulatedEvent);
|
|
63603
63639
|
const matched = decisions.filter((d) => d.matched);
|
|
@@ -64000,11 +64036,18 @@ async function handleTestRunStatus(msg, deps) {
|
|
|
64000
64036
|
};
|
|
64001
64037
|
}
|
|
64002
64038
|
/**
|
|
64003
|
-
* Return the next chunk of a run's logs
|
|
64004
|
-
*
|
|
64005
|
-
*
|
|
64006
|
-
*
|
|
64007
|
-
*
|
|
64039
|
+
* Return the next chunk of a run's logs.
|
|
64040
|
+
*
|
|
64041
|
+
* With `logCursor` (the per-file cursor), each log file is read from its own
|
|
64042
|
+
* offset, so a file that appears later is read from its first line wherever it
|
|
64043
|
+
* sorts. Without it, the stream is the flat `(jobName, stepIndex, lineIndex)`
|
|
64044
|
+
* concatenation and `cursor` counts the lines already delivered, which an
|
|
64045
|
+
* older CLI sends.
|
|
64046
|
+
*
|
|
64047
|
+
* `done` is true only once the run is terminal and the stream is delivered to
|
|
64048
|
+
* its end. For a terminal run, pending log appends are drained first, because
|
|
64049
|
+
* the agent's chunks are persisted fire-and-forget and the status can turn
|
|
64050
|
+
* terminal while the last write is still in flight.
|
|
64008
64051
|
*/
|
|
64009
64052
|
async function handleTestRunLogs(msg, deps) {
|
|
64010
64053
|
const run = await deps.db.selectFrom("execution_runs").select([
|
|
@@ -64015,6 +64058,11 @@ async function handleTestRunLogs(msg, deps) {
|
|
|
64015
64058
|
if (!run || !run.is_test_run) return { error: "Run not found" };
|
|
64016
64059
|
const terminal = TERMINAL_RUN_STATES.has(run.status);
|
|
64017
64060
|
if (terminal) await deps.logWriter?.drain(msg.runId);
|
|
64061
|
+
if (msg.logCursor !== void 0) {
|
|
64062
|
+
const offsets = decodeRunLogCursor(msg.logCursor);
|
|
64063
|
+
if (!offsets) return { error: "Invalid log cursor" };
|
|
64064
|
+
return readFromFileOffsets(msg.runId, deps.logStorage, offsets, terminal);
|
|
64065
|
+
}
|
|
64018
64066
|
const allLines = await collectRunLogLines(msg.runId, deps.logStorage);
|
|
64019
64067
|
const cursor = Math.max(0, msg.cursor);
|
|
64020
64068
|
const lines = allLines.slice(cursor);
|
|
@@ -64025,26 +64073,55 @@ async function handleTestRunLogs(msg, deps) {
|
|
|
64025
64073
|
done: terminal && nextCursor >= allLines.length
|
|
64026
64074
|
};
|
|
64027
64075
|
}
|
|
64028
|
-
/**
|
|
64029
|
-
async function
|
|
64030
|
-
|
|
64031
|
-
|
|
64032
|
-
|
|
64033
|
-
|
|
64034
|
-
|
|
64035
|
-
path
|
|
64076
|
+
/** A run's stored log files, ordered by `(jobName, stepIndex)`. */
|
|
64077
|
+
async function listRunLogFiles(runId, logStorage) {
|
|
64078
|
+
return (await logStorage.list(`executions/${runId}`)).map((path) => {
|
|
64079
|
+
const m = path.match(LOG_PATH_RE);
|
|
64080
|
+
if (!m) return null;
|
|
64081
|
+
return {
|
|
64082
|
+
key: `job-${m[1]}/step-${m[2]}.log`,
|
|
64083
|
+
path,
|
|
64036
64084
|
jobName: m[1],
|
|
64037
64085
|
stepIndex: parseInt(m[2], 10)
|
|
64038
|
-
}
|
|
64086
|
+
};
|
|
64039
64087
|
}).filter((f) => f !== null).sort((a, b) => a.jobName.localeCompare(b.jobName) || a.stepIndex - b.stepIndex);
|
|
64088
|
+
}
|
|
64089
|
+
/** A log file's non-empty lines; its unterminated last line only when `withTail`. */
|
|
64090
|
+
function splitLogLines(data, withTail) {
|
|
64091
|
+
const parts = data.split("\n");
|
|
64092
|
+
const tail = parts.pop() ?? "";
|
|
64093
|
+
const lines = parts.filter((line) => line.length > 0);
|
|
64094
|
+
if (withTail && tail.length > 0) lines.push(tail);
|
|
64095
|
+
return lines;
|
|
64096
|
+
}
|
|
64097
|
+
/** Reads every step log under `executions/<runId>`; empty with no log storage. */
|
|
64098
|
+
async function collectRunLogLines(runId, logStorage) {
|
|
64099
|
+
if (!logStorage) return [];
|
|
64040
64100
|
const lines = [];
|
|
64041
|
-
for (const
|
|
64042
|
-
const result = await logStorage.read(entry.path);
|
|
64043
|
-
for (const line of result.data.split("\n")) if (line.length > 0) lines.push(line);
|
|
64044
|
-
}
|
|
64101
|
+
for (const file of await listRunLogFiles(runId, logStorage)) lines.push(...splitLogLines((await logStorage.read(file.path)).data, true));
|
|
64045
64102
|
return lines;
|
|
64046
64103
|
}
|
|
64047
64104
|
/**
|
|
64105
|
+
* Read each log file from its own offset. A live file's last line can still
|
|
64106
|
+
* be mid-write, so it is read once it ends, or once the run is terminal.
|
|
64107
|
+
*/
|
|
64108
|
+
async function readFromFileOffsets(runId, logStorage, offsets, terminal) {
|
|
64109
|
+
const next = { ...offsets };
|
|
64110
|
+
const lines = [];
|
|
64111
|
+
if (logStorage) for (const file of await listRunLogFiles(runId, logStorage)) {
|
|
64112
|
+
const fileLines = splitLogLines((await logStorage.read(file.path)).data, terminal);
|
|
64113
|
+
const from = Math.min(offsets[file.key] ?? 0, fileLines.length);
|
|
64114
|
+
lines.push(...fileLines.slice(from));
|
|
64115
|
+
next[file.key] = Math.max(offsets[file.key] ?? 0, fileLines.length);
|
|
64116
|
+
}
|
|
64117
|
+
return {
|
|
64118
|
+
lines,
|
|
64119
|
+
nextCursor: Object.values(next).reduce((sum, n) => sum + n, 0),
|
|
64120
|
+
done: terminal,
|
|
64121
|
+
nextLogCursor: encodeRunLogCursor(next)
|
|
64122
|
+
};
|
|
64123
|
+
}
|
|
64124
|
+
/**
|
|
64048
64125
|
* Cancel a running test run: signal cancellation to every dispatched agent,
|
|
64049
64126
|
* mark still-pending dispatch queue entries `expired`, and mark the
|
|
64050
64127
|
* `execution_runs` row cancelled. Returns `{ cancelled }` — true when the run
|
|
@@ -64098,6 +64175,7 @@ async function handleTestCancel(msg, deps) {
|
|
|
64098
64175
|
var logger$59, LOG_PATH_RE;
|
|
64099
64176
|
var init_test_relay_handlers = __esmMin((() => {
|
|
64100
64177
|
init_job_queue();
|
|
64178
|
+
init_test_run_log_cursor();
|
|
64101
64179
|
init_uploads();
|
|
64102
64180
|
init_test_pipeline();
|
|
64103
64181
|
init_run_detached();
|
|
@@ -64359,7 +64437,8 @@ function createTestDirectRoutes(deps) {
|
|
|
64359
64437
|
app.get("/api/v1/test/runs/:runId/logs", (c) => runRoute(c, deps, "test_run.read", (msg, d) => handleTestRunLogs({
|
|
64360
64438
|
type: "test.relay.run.logs",
|
|
64361
64439
|
...msg,
|
|
64362
|
-
cursor: parseCursor$1(c.req.query("cursor"))
|
|
64440
|
+
cursor: parseCursor$1(c.req.query("cursor")),
|
|
64441
|
+
logCursor: c.req.query("logCursor")
|
|
64363
64442
|
}, d)));
|
|
64364
64443
|
app.post("/api/v1/test/runs/:runId/cancel", (c) => runRoute(c, deps, "test_run.trigger", (msg, d) => handleTestCancel({
|
|
64365
64444
|
type: "test.relay.cancel",
|
|
@@ -68447,14 +68526,14 @@ function createHealthRoutes$1(deps = {}) {
|
|
|
68447
68526
|
return createHealthRoutes({
|
|
68448
68527
|
livenessInfo: () => {
|
|
68449
68528
|
return {
|
|
68450
|
-
version: "0.
|
|
68451
|
-
buildDate: "2026-10-
|
|
68452
|
-
sdkVersion: "0.
|
|
68529
|
+
version: "0.20.0",
|
|
68530
|
+
buildDate: "2026-10-17T08:12:16.425Z",
|
|
68531
|
+
sdkVersion: "0.20.0",
|
|
68453
68532
|
sdkBundleHash: "b012bd8bace3df9e574252bf290c652bc39766da617acf51729a08586756266d",
|
|
68454
|
-
sharedVersion: "0.
|
|
68533
|
+
sharedVersion: "0.20.0",
|
|
68455
68534
|
sharedBundleHash: "23378ace59973f47203be3ff171bb7d5e66702eaa335927b390f94b9771a1f8b",
|
|
68456
|
-
engineVersion: "0.
|
|
68457
|
-
engineBundleHash: "
|
|
68535
|
+
engineVersion: "0.20.0",
|
|
68536
|
+
engineBundleHash: "406208ee50766cd0d4ce3ab1fd5dec4501d5117a74acd294b25736d9bd6e89bd"
|
|
68458
68537
|
};
|
|
68459
68538
|
},
|
|
68460
68539
|
readinessCheck: deps.db ? async () => {
|
|
@@ -68486,7 +68565,7 @@ function createCapabilitiesRoutes() {
|
|
|
68486
68565
|
const app = new Hono();
|
|
68487
68566
|
app.get("/api/v1/capabilities", (c) => {
|
|
68488
68567
|
const manifest = {
|
|
68489
|
-
orchestratorVersion: "0.
|
|
68568
|
+
orchestratorVersion: "0.20.0",
|
|
68490
68569
|
protocolVersion: PROTOCOL_VERSION,
|
|
68491
68570
|
minProtocolVersion: MIN_PROTOCOL_VERSION
|
|
68492
68571
|
};
|
|
@@ -71720,7 +71799,7 @@ var init_app = __esmMin((() => {
|
|
|
71720
71799
|
init_agent_metrics_aggregator();
|
|
71721
71800
|
init_run_detached();
|
|
71722
71801
|
logger$38 = createLogger({ prefix: "app" });
|
|
71723
|
-
ORCHESTRATOR_VERSION$2 = "0.
|
|
71802
|
+
ORCHESTRATOR_VERSION$2 = "0.20.0";
|
|
71724
71803
|
SourceLocationStore = class {
|
|
71725
71804
|
cache = /* @__PURE__ */ new Map();
|
|
71726
71805
|
key(workflowName, jobName) {
|
|
@@ -72256,6 +72335,7 @@ function buildAgentContainerHostConfig(input) {
|
|
|
72256
72335
|
...typeof cpus === "number" && cpus > 0 ? { NanoCpus: Math.round(cpus * 1e9) } : {},
|
|
72257
72336
|
...input.binds && input.binds.length > 0 ? { Binds: input.binds } : {},
|
|
72258
72337
|
...input.extraHosts && input.extraHosts.length > 0 ? { ExtraHosts: input.extraHosts } : {},
|
|
72338
|
+
...input.supplementaryGroups && input.supplementaryGroups.length > 0 ? { GroupAdd: input.supplementaryGroups.map(String) } : {},
|
|
72259
72339
|
AutoRemove: false,
|
|
72260
72340
|
PidsLimit: AGENT_CONTAINER_PIDS_LIMIT
|
|
72261
72341
|
};
|
|
@@ -72582,6 +72662,19 @@ async function detectRuntime(runtimeHint) {
|
|
|
72582
72662
|
* Create a LogCapture from a running container's stdout/stderr streams.
|
|
72583
72663
|
* Uses dockerode's demuxStream to strip Docker's 8-byte multiplexed headers.
|
|
72584
72664
|
*/
|
|
72665
|
+
/**
|
|
72666
|
+
* The cpu / memory ceiling for one spawn. `effectiveLimits` from ScalerManager
|
|
72667
|
+
* wins: it already folds in job overrides, label-set and scaler defaults.
|
|
72668
|
+
* Otherwise the label set's (or the scaler's default) `resources.limits` apply.
|
|
72669
|
+
*/
|
|
72670
|
+
function resolveSpawnLimits(effectiveLimits, resources) {
|
|
72671
|
+
if (effectiveLimits && (effectiveLimits.cpus || effectiveLimits.memBytes)) return effectiveLimits;
|
|
72672
|
+
const limits = resources?.limits;
|
|
72673
|
+
return {
|
|
72674
|
+
...limits?.memory ? { memBytes: parseMemoryString(limits.memory) } : {},
|
|
72675
|
+
...limits?.cpus ? { cpus: limits.cpus } : {}
|
|
72676
|
+
};
|
|
72677
|
+
}
|
|
72585
72678
|
async function createContainerLogCapture(docker, containerId) {
|
|
72586
72679
|
const stream = await docker.getContainer(containerId).logs({
|
|
72587
72680
|
follow: true,
|
|
@@ -72898,15 +72991,7 @@ var init_container_backend = __esmMin((() => {
|
|
|
72898
72991
|
];
|
|
72899
72992
|
const binds = [...matchedLabelSet.volumes ?? []];
|
|
72900
72993
|
if (matchedLabelSet.containerSocket && this.resolvedSocketPath) binds.push(`${this.resolvedSocketPath}:${this.resolvedSocketPath}`);
|
|
72901
|
-
|
|
72902
|
-
if (effectiveLimits && (effectiveLimits.cpus || effectiveLimits.memBytes)) resolvedLimits = effectiveLimits;
|
|
72903
|
-
else {
|
|
72904
|
-
const limits = (matchedLabelSet.resources ?? this.defaultResources)?.limits;
|
|
72905
|
-
resolvedLimits = {
|
|
72906
|
-
...limits?.memory ? { memBytes: parseMemoryString(limits.memory) } : {},
|
|
72907
|
-
...limits?.cpus ? { cpus: limits.cpus } : {}
|
|
72908
|
-
};
|
|
72909
|
-
}
|
|
72994
|
+
const resolvedLimits = resolveSpawnLimits(effectiveLimits, matchedLabelSet.resources ?? this.defaultResources);
|
|
72910
72995
|
const jobContainer = spawnContext?.container;
|
|
72911
72996
|
const spawnImage = jobContainer?.image ?? matchedLabelSet.image;
|
|
72912
72997
|
await pullImageIfMissing({
|
|
@@ -72929,6 +73014,7 @@ var init_container_backend = __esmMin((() => {
|
|
|
72929
73014
|
binds.push(runtimeInjectBind(runtimeVolume));
|
|
72930
73015
|
agentInit = runtimeInitCommand(await agentImageLabels(this.docker, matchedLabelSet.image));
|
|
72931
73016
|
}
|
|
73017
|
+
if (!jobContainer) env.push("KICI_SCALER_SINGLE_JOB=1");
|
|
72932
73018
|
emit(ScalerEventType$1.enum["scaler.provisioning"], "creating container");
|
|
72933
73019
|
const normalizedLabelSetStr = normalizeLabelSet(labelSet);
|
|
72934
73020
|
const container = await this.docker.createContainer({
|
|
@@ -72947,7 +73033,8 @@ var init_container_backend = __esmMin((() => {
|
|
|
72947
73033
|
HostConfig: buildAgentContainerHostConfig({
|
|
72948
73034
|
limits: resolvedLimits,
|
|
72949
73035
|
binds,
|
|
72950
|
-
...this.extraHosts ? { extraHosts: this.extraHosts } : {}
|
|
73036
|
+
...this.extraHosts ? { extraHosts: this.extraHosts } : {},
|
|
73037
|
+
...!jobContainer && { supplementaryGroups: [1990] }
|
|
72951
73038
|
}),
|
|
72952
73039
|
...this.networkIsolation && { NetworkingConfig: { EndpointsConfig: { ["kici-agent-net"]: {} } } }
|
|
72953
73040
|
});
|
|
@@ -91327,13 +91414,13 @@ var init_worker_core = __esmMin((() => {
|
|
|
91327
91414
|
init_app_on_error();
|
|
91328
91415
|
init_run_detached();
|
|
91329
91416
|
init_peer_job_cancel();
|
|
91330
|
-
ORCHESTRATOR_VERSION$1 = "0.
|
|
91331
|
-
WORKER_SDK_VERSION = "0.
|
|
91417
|
+
ORCHESTRATOR_VERSION$1 = "0.20.0";
|
|
91418
|
+
WORKER_SDK_VERSION = "0.20.0";
|
|
91332
91419
|
WORKER_SDK_BUNDLE_HASH = "b012bd8bace3df9e574252bf290c652bc39766da617acf51729a08586756266d";
|
|
91333
|
-
WORKER_SHARED_VERSION = "0.
|
|
91420
|
+
WORKER_SHARED_VERSION = "0.20.0";
|
|
91334
91421
|
WORKER_SHARED_BUNDLE_HASH = "23378ace59973f47203be3ff171bb7d5e66702eaa335927b390f94b9771a1f8b";
|
|
91335
|
-
WORKER_ENGINE_VERSION = "0.
|
|
91336
|
-
WORKER_ENGINE_BUNDLE_HASH = "
|
|
91422
|
+
WORKER_ENGINE_VERSION = "0.20.0";
|
|
91423
|
+
WORKER_ENGINE_BUNDLE_HASH = "406208ee50766cd0d4ce3ab1fd5dec4501d5117a74acd294b25736d9bd6e89bd";
|
|
91337
91424
|
logger$2 = createLogger({ prefix: "worker" });
|
|
91338
91425
|
DRAIN_TIMEOUT_MS = 3e5;
|
|
91339
91426
|
}));
|
|
@@ -91362,13 +91449,13 @@ init_webhook_url_resolvers();
|
|
|
91362
91449
|
init_run_detached();
|
|
91363
91450
|
init_peer_job_cancel();
|
|
91364
91451
|
init_orphan_requests();
|
|
91365
|
-
const ORCHESTRATOR_VERSION = "0.
|
|
91366
|
-
const SDK_VERSION = "0.
|
|
91452
|
+
const ORCHESTRATOR_VERSION = "0.20.0";
|
|
91453
|
+
const SDK_VERSION = "0.20.0";
|
|
91367
91454
|
const SDK_BUNDLE_HASH = "b012bd8bace3df9e574252bf290c652bc39766da617acf51729a08586756266d";
|
|
91368
|
-
const SHARED_VERSION = "0.
|
|
91455
|
+
const SHARED_VERSION = "0.20.0";
|
|
91369
91456
|
const SHARED_BUNDLE_HASH = "23378ace59973f47203be3ff171bb7d5e66702eaa335927b390f94b9771a1f8b";
|
|
91370
|
-
const ENGINE_VERSION = "0.
|
|
91371
|
-
const ENGINE_BUNDLE_HASH = "
|
|
91457
|
+
const ENGINE_VERSION = "0.20.0";
|
|
91458
|
+
const ENGINE_BUNDLE_HASH = "406208ee50766cd0d4ce3ab1fd5dec4501d5117a74acd294b25736d9bd6e89bd";
|
|
91372
91459
|
const otelSdk = initTelemetry({
|
|
91373
91460
|
serviceName: "kici-orchestrator",
|
|
91374
91461
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|