@kici-dev/orchestrator 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/dist/__test-helpers__/test-db.d.ts +14 -1
  2. package/dist/agent/dispatcher.d.ts +137 -41
  3. package/dist/agent/registry.d.ts +79 -2
  4. package/dist/app.d.ts +2 -0
  5. package/dist/cancel/cancel-held-run.d.ts +68 -0
  6. package/dist/cancel/cancel-route-answer.d.ts +21 -0
  7. package/dist/cancel/cancel-run.d.ts +32 -0
  8. package/dist/cli/api-client.d.ts +3 -0
  9. package/dist/cli/commands/context.d.ts +3 -1
  10. package/dist/cli/commands/runs.d.ts +7 -0
  11. package/dist/cli/commands/shared/unbound-context-warning.d.ts +43 -0
  12. package/dist/cli/commands/signing-key.d.ts +22 -1
  13. package/dist/cli.js +1306 -739
  14. package/dist/cluster/cluster-settings-reader.d.ts +1 -1
  15. package/dist/cluster/coordinator.d.ts +2 -0
  16. package/dist/cluster/live-coordinator-peer.d.ts +13 -0
  17. package/dist/config.d.ts +9 -0
  18. package/dist/contexts/context-store.d.ts +3 -2
  19. package/dist/contexts/protection/satisfiability.d.ts +6 -3
  20. package/dist/db/migrations/145_execution_runs_workflow_provenance.d.ts +4 -0
  21. package/dist/db/migrations/146_pending_job_contexts_context_resolution.d.ts +4 -0
  22. package/dist/db/migrations/147_sealed_job_secrets.d.ts +19 -0
  23. package/dist/db/migrations/148_cluster_settings_sealed_secrets_retry_backoff_ms.d.ts +21 -0
  24. package/dist/db/migrations/149_workflow_registrations_dep_cache_key.d.ts +4 -0
  25. package/dist/db/migrations/150_execution_runs_status_epoch.d.ts +18 -0
  26. package/dist/db/migrations/151_workflow_registrations_dep_cache_key_sha.d.ts +4 -0
  27. package/dist/db/repos/signing-keys-repo.d.ts +48 -3
  28. package/dist/db/types.d.ts +81 -1
  29. package/dist/git/clone-token-auth.d.ts +7 -0
  30. package/dist/git/dispatch-git-auth.d.ts +63 -0
  31. package/dist/lockfile-validate.d.ts +4 -2
  32. package/dist/metrics/prometheus.d.ts +1 -1
  33. package/dist/oidc/id-token-claims.d.ts +10 -0
  34. package/dist/oidc/oidc-mint-registration.d.ts +2 -0
  35. package/dist/oidc/orchestrator-mint.d.ts +25 -4
  36. package/dist/oidc/reconcile-signing-key.d.ts +31 -6
  37. package/dist/oidc/resolve-signer.d.ts +12 -6
  38. package/dist/pipeline/decorating-secret-resolver.d.ts +5 -3
  39. package/dist/pipeline/dispatch-matched-workflow.d.ts +105 -13
  40. package/dist/pipeline/dispatch-matched-workflow.test-helpers.d.ts +121 -0
  41. package/dist/pipeline/global-dispatch-identity.d.ts +76 -0
  42. package/dist/pipeline/global-dispatch.d.ts +91 -0
  43. package/dist/pipeline/global-eval-round.d.ts +56 -4
  44. package/dist/pipeline/global-round-hold.d.ts +41 -0
  45. package/dist/pipeline/held-context-data.d.ts +143 -0
  46. package/dist/pipeline/held-round-registrations.d.ts +40 -0
  47. package/dist/pipeline/install-gate.d.ts +93 -0
  48. package/dist/pipeline/install-secrets-resolver.d.ts +28 -13
  49. package/dist/pipeline/job-contexts.d.ts +21 -6
  50. package/dist/pipeline/pending-workflow-context.d.ts +15 -3
  51. package/dist/pipeline/process-webhook.d.ts +38 -1
  52. package/dist/pipeline/processor.d.ts +20 -0
  53. package/dist/pipeline/rerun-global.d.ts +28 -0
  54. package/dist/pipeline/rerun-inputs.d.ts +88 -0
  55. package/dist/pipeline/rerun.d.ts +25 -2
  56. package/dist/pipeline/resume-workflow.d.ts +36 -2
  57. package/dist/pipeline/unbound-job-context.d.ts +34 -0
  58. package/dist/pipeline/workflow-lock-at-sha.d.ts +67 -0
  59. package/dist/provenance/backfill-run.d.ts +5 -0
  60. package/dist/providers/github/check-status-poster.d.ts +4 -16
  61. package/dist/queue/job-queue.d.ts +127 -10
  62. package/dist/queue/terminalize-unroutable.d.ts +37 -4
  63. package/dist/registration/registration-index.d.ts +13 -1
  64. package/dist/registration/registration-store.d.ts +10 -1
  65. package/dist/reporting/execution-tracker.d.ts +149 -8
  66. package/dist/reporting/log-chunk-sink.d.ts +1 -1
  67. package/dist/reporting/log-writer.d.ts +14 -0
  68. package/dist/reporting/step-log-path.d.ts +8 -0
  69. package/dist/reporting/step-log-reader.d.ts +21 -3
  70. package/dist/routes/admin-signing-keys.d.ts +33 -0
  71. package/dist/scaler/agent-fit.d.ts +159 -0
  72. package/dist/scaler/container-routing.d.ts +16 -16
  73. package/dist/scaler/manager.d.ts +23 -17
  74. package/dist/secrets/context-secret-resolver.d.ts +1 -1
  75. package/dist/secrets/index.d.ts +1 -1
  76. package/dist/secrets/job-secret-gate.d.ts +25 -3
  77. package/dist/secrets/job-secret-seal.d.ts +36 -0
  78. package/dist/secrets/master-key-rotation.d.ts +7 -3
  79. package/dist/secrets/secret-resolver.d.ts +42 -23
  80. package/dist/server.js +27732 -23477
  81. package/dist/standalone.js +58270 -53199
  82. package/dist/worker/in-memory-job-queue.d.ts +12 -3
  83. package/dist/worker/reroute-dispatch.d.ts +35 -0
  84. package/dist/ws/agent-handler.d.ts +23 -10
  85. package/dist/ws/git-credential-relay.d.ts +16 -3
  86. package/installer-image-digests.json +3 -3
  87. package/package.json +4 -4
  88. package/sbom.spdx.json +53 -53
@@ -4,10 +4,17 @@ import type { Database } from '../db/types.js';
4
4
  import type { ExecutionTracker } from '../reporting/execution-tracker.js';
5
5
  import type { CheckRunReporter } from '../reporting/check-run-reporter.js';
6
6
  import type { PendingGlobalEvalTracker } from '../cache/pending-global-evals.js';
7
+ import type { AgentRegistry } from '../agent/registry.js';
8
+ import { type AgentFitJob, type ScalerAgentView } from '../scaler/agent-fit.js';
7
9
  import type { ExpiredJobInfo } from './job-queue.js';
8
10
  /**
9
11
  * Whether ANYTHING could ever run a job with these selectors — a registered
10
- * agent (regardless of capacity) or a scaler backend able to spawn one.
12
+ * agent (regardless of capacity) that can run it, or a scaler backend able to
13
+ * spawn one.
14
+ *
15
+ * `job`, when given, is what the agent must be able to do beyond its labels:
16
+ * above all start the job's container (`canAgentRunJob`). Absent, any
17
+ * label-matching agent counts.
11
18
  *
12
19
  * The predicate is allowed to answer "routable" conservatively: the scaler half
13
20
  * matches exact labels only, so a pattern-only `runsOn` reads routable on a
@@ -15,9 +22,20 @@ import type { ExpiredJobInfo } from './job-queue.js';
15
22
  * how quickly the job settles, never safety — a job that reads routable
16
23
  * falls through to the queue-timeout backstop.
17
24
  */
18
- export type CanRouteLabels = (requiredLabels: string[], requiredPatterns: LabelMatcher[], excludeLabels: string[], excludePatterns: LabelMatcher[]) => boolean;
25
+ export type CanRouteLabels = (requiredLabels: string[], requiredPatterns: LabelMatcher[], excludeLabels: string[], excludePatterns: LabelMatcher[], job?: AgentFitJob) => boolean;
26
+ /**
27
+ * The one routability predicate, built on the registry and the scaler: the
28
+ * unroutable probe and the queue-expiry sweep both ask it.
29
+ */
30
+ export declare function makeCanRouteLabels(deps: {
31
+ registry: Pick<AgentRegistry, 'hasMatchingAgent'>;
32
+ scaler?: {
33
+ hasBackendForLabels(labels: string[], excludeLabels: string[]): boolean;
34
+ agentView(agentId: string): ScalerAgentView | undefined;
35
+ };
36
+ }): CanRouteLabels;
19
37
  /** The routing facts a verdict is computed from. */
20
- export type JobRoutingFacts = Pick<ExpiredJobInfo, 'lastProvisioningError' | 'runsOnLabels' | 'runsOnPatterns' | 'excludeLabels' | 'excludePatterns'>;
38
+ export type JobRoutingFacts = Pick<ExpiredJobInfo, 'lastProvisioningError' | 'runsOnLabels' | 'runsOnPatterns' | 'excludeLabels' | 'excludePatterns'> & Partial<Pick<ExpiredJobInfo, 'id' | 'container'>>;
21
39
  /** Everything {@link terminalizeUnroutableJob} needs to settle a job. */
22
40
  export interface TerminalizeDeps {
23
41
  db: Kysely<Database>;
@@ -37,7 +55,22 @@ export interface TerminalizeDeps {
37
55
  */
38
56
  pendingGlobalEvals?: Pick<PendingGlobalEvalTracker, 'reject'>;
39
57
  }
40
- export declare function unroutableMessage(job: JobRoutingFacts): string;
58
+ /** Why nothing can run a job. */
59
+ export declare enum UnroutableCause {
60
+ /** No agent or scaler backend matches its labels. */
61
+ NoMatch = "no-match",
62
+ /**
63
+ * Agents match its labels, but it is a container job and none of them can
64
+ * start a container.
65
+ */
66
+ NoContainerRuntime = "no-container-runtime",
67
+ /**
68
+ * Agents match its labels, but each was started inside another job's image
69
+ * and runs only that job.
70
+ */
71
+ OnlyOtherJobImageAgents = "only-other-job-image-agents"
72
+ }
73
+ export declare function unroutableMessage(job: JobRoutingFacts, cause?: UnroutableCause): string;
41
74
  /**
42
75
  * Split the two reasons a queued job never ran: nothing in the fleet matched
43
76
  * its `runsOn` (`unroutable` — a label/fleet problem an operator has to fix)
@@ -1,9 +1,16 @@
1
1
  import type { LockWorkflow } from '@kici-dev/engine';
2
+ import type { DepCacheKey } from '@kici-dev/shared';
2
3
  import type { RegistrationStore } from './registration-store.js';
3
4
  /**
4
5
  * A registered workflow entry with typed fields for fast in-memory lookup.
6
+ *
7
+ * Its `lockfileHash` / `siblingsDigest` are the dependency-cache key of the
8
+ * lock file that registered it, so a run dispatched from the registration uses
9
+ * the dependency cache the way a run dispatched from a fetched lock file does.
10
+ * Both are null unless the key was written for the registration's current
11
+ * commit, so a run never restores an older lock file's dependencies.
5
12
  */
6
- export interface RegisteredWorkflow {
13
+ export interface RegisteredWorkflow extends DepCacheKey {
7
14
  id: string;
8
15
  repoIdentifier: string;
9
16
  workflowName: string;
@@ -136,6 +143,11 @@ export declare class RegistrationIndex {
136
143
  * Disabled rows are filtered out, mirroring the other typed lookups.
137
144
  */
138
145
  getByOrgAndRepo(customerId: string, repoIdentifier: string): RegisteredWorkflow[];
146
+ /**
147
+ * {@link getByOrgAndRepo} with disabled entries included, for a caller that
148
+ * must tell a disabled workflow from one that is no longer registered.
149
+ */
150
+ getAllByOrgAndRepo(customerId: string, repoIdentifier: string): RegisteredWorkflow[];
139
151
  /**
140
152
  * Get global workflows matching a trigger type within a routing key.
141
153
  * Used by the webhook processor for dual-query (per-repo + global).
@@ -1,12 +1,13 @@
1
1
  import type { Kysely } from 'kysely';
2
2
  import type { LockWorkflow } from '@kici-dev/engine';
3
+ import { type DepCacheKey } from '@kici-dev/shared';
3
4
  import type { Database } from '../db/types.js';
4
5
  /**
5
6
  * Parsed registration row with typed fields.
6
7
  * DB stores lock_entry as JSON string and trigger_types as TEXT[],
7
8
  * but this interface exposes them as typed objects.
8
9
  */
9
- export interface RegistrationRow {
10
+ export interface RegistrationRow extends DepCacheKey {
10
11
  id: string;
11
12
  repo_identifier: string;
12
13
  workflow_name: string;
@@ -63,6 +64,14 @@ interface ReplaceAllOptions {
63
64
  */
64
65
  defaultBranch?: string | null;
65
66
  sourceFile?: string;
67
+ /**
68
+ * The dependency-cache key of the lock file the workflows come from. Written
69
+ * on every row, like the lock entry it belongs to: an omitted key, or a lock
70
+ * file that records none, stores no key, so a row never keeps the key of a
71
+ * lock file it no longer carries the entry of. `commitSha` is written next to
72
+ * it as the commit the key describes.
73
+ */
74
+ depCacheKey?: DepCacheKey;
66
75
  /** Set of workflow names that should be marked as global */
67
76
  globalWorkflowNames?: Set<string>;
68
77
  }
@@ -17,7 +17,27 @@ import { JobKind } from '../db/types.js';
17
17
  import { ExecutionRunStatus, type InitFailure, RunFailureClass, ScalerEventType, OrchLogPhase } from '@kici-dev/engine';
18
18
  import type { ObserverRegistry } from '../ws/observer-registry.js';
19
19
  import type { LogStorage } from './log-storage.js';
20
- import type { JobQueue } from '../queue/job-queue.js';
20
+ import { type JobQueue } from '../queue/job-queue.js';
21
+ /**
22
+ * The repository that defines a run's workflow, with the commit the run
23
+ * dispatched from it and that repository's registered branch.
24
+ */
25
+ export interface WorkflowRepoProvenance {
26
+ identifier: string;
27
+ sha: string | null;
28
+ branch: string | null;
29
+ }
30
+ /**
31
+ * The run a re-run re-executes and the root of its re-run chain. Stamped by the
32
+ * pre-dispatch writers too, so a re-run that is held or fails before its first
33
+ * job still records what it re-ran.
34
+ */
35
+ export interface RerunLineage {
36
+ parentRunId: string;
37
+ originalRunId: string;
38
+ }
39
+ /** The `trigger_decision` key a held evaluation round records its covered workflows under. */
40
+ export declare const HELD_ROUND_WORKFLOWS_KEY = "heldRoundWorkflows";
21
41
  /** Context passed to onExecutionComplete for commit status updates. */
22
42
  export interface ExecutionContext {
23
43
  workflowName: string;
@@ -39,8 +59,7 @@ export interface ExecutionContext {
39
59
  workflowRepoIdentifier?: string;
40
60
  /**
41
61
  * True when this run records a global evaluation round rather than a
42
- * workflow. Forwarded to the Platform so its own re-run refusal can admit the
43
- * round's re-evaluation.
62
+ * workflow. Forwarded to the Platform, which mirrors it on the run row.
44
63
  */
45
64
  isGlobalEvalRound?: boolean;
46
65
  /** Git branch or tag (e.g. "main", "feature/foo"). */
@@ -71,6 +90,13 @@ export interface ExecutionContext {
71
90
  * subscriptions can match on it. Null/undefined for success or non-terminal.
72
91
  */
73
92
  failureClass?: RunFailureClass | null;
93
+ /**
94
+ * The run's status generation (`execution_runs.status_epoch`), forwarded on
95
+ * `execution.status` as `statusEpoch`. Absent reads as 0. It rises only when
96
+ * a run leaves a terminal status to continue, so the Platform can keep a
97
+ * finished run finished against a non-terminal frame that arrives late.
98
+ */
99
+ statusEpoch?: number;
74
100
  /**
75
101
  * Resolved trust tier of the run's ref, and which branch's lock file it was
76
102
  * evaluated against. Mirror the `execution_runs.trust_tier` /
@@ -396,12 +422,13 @@ export declare class ExecutionTracker {
396
422
  /** Pull-request number for PR-triggered runs; null/omitted for non-PR runs. */
397
423
  prNumber?: number | null,
398
424
  /**
399
- * The repository that DEFINES the workflow, when that is not
400
- * `repoIdentifier` — an organization-wide workflow authored in one
401
- * repository and dispatched against another. Omitted/null for every
402
- * per-repository run, where the two are the same repository.
425
+ * The repository that DEFINES the workflow, with the commit dispatched from
426
+ * it and its registered branch, when that is not `repoIdentifier` — an
427
+ * organization-wide workflow authored in one repository and dispatched
428
+ * against another. Omitted/null for every per-repository run, where the two
429
+ * are the same repository.
403
430
  */
404
- workflowRepoIdentifier?: string | null,
431
+ workflowRepo?: WorkflowRepoProvenance | null,
405
432
  /**
406
433
  * Pull-request head context, resolved from the normalized event.
407
434
  *
@@ -436,6 +463,16 @@ export declare class ExecutionTracker {
436
463
  * partitions by it without joining execution_runs.
437
464
  */
438
465
  private insertTrackedJobRows;
466
+ /**
467
+ * Cancel job rows just registered under a run that has already stopped.
468
+ *
469
+ * A job registers after its dispatch, so a cancel landing in between leaves
470
+ * the run terminal while the job's row is written `pending`. The queue
471
+ * refuses that job's dispatch (`JobQueue.insertDispatched`), so no agent
472
+ * will ever report on it: its row is cancelled here, as the cancel cancels
473
+ * the queued jobs it can see.
474
+ */
475
+ private cancelJobsOfStoppedRun;
439
476
  /**
440
477
  * Find the synthetic needs-pending job ID for a given job name in a run.
441
478
  * Used by dispatchReadyJob to locate the placeholder entry before replacing it.
@@ -580,6 +617,29 @@ export declare class ExecutionTracker {
580
617
  * (in which case the caller skips this status update entirely).
581
618
  */
582
619
  private recoverRunFromDb;
620
+ /**
621
+ * Reopen a run rehydrated from a prematurely `failed` row (e.g. "No agents
622
+ * available to dispatch jobs", but a queued job is now running): reset the
623
+ * row to `running`, clear the failure it recorded, and raise its status
624
+ * generation. The run completion logic then sets the correct final status.
625
+ *
626
+ * Called only once the job report that rehydrated the run has been applied to
627
+ * its job row. A replay of a frame the row already records must not reopen a
628
+ * finished run: it would roll the run up a second time. A run that failed on
629
+ * a build failure is never reopened — that failure is intentionally terminal.
630
+ */
631
+ private reopenRecoveredRun;
632
+ /**
633
+ * Read the run row's status generation into the in-memory run after this
634
+ * coordinator's reopen matched nothing.
635
+ *
636
+ * The reopen loses when another coordinator already reopened the run, or when
637
+ * this view is a generation behind the row. Either way the row's generation
638
+ * is the current one: a coordinator that kept its own older value would send
639
+ * every later frame of the run at a generation the Platform keeps as older, so
640
+ * none of them would apply.
641
+ */
642
+ private adoptStoredStatusEpoch;
583
643
  /**
584
644
  * Phase 1b: recover job state from the dispatch_queue and add it to the
585
645
  * run's in-memory map. Returns the freshly-inserted job entry.
@@ -746,6 +806,7 @@ export declare class ExecutionTracker {
746
806
  provenance?: {
747
807
  chainDepth?: number;
748
808
  dispatchedByFailureLifecycle?: boolean;
809
+ rerunLineage?: RerunLineage;
749
810
  }): Promise<void>;
750
811
  /**
751
812
  * Write a `failed` execution_runs row directly for an init failure, and drop
@@ -813,6 +874,8 @@ export declare class ExecutionTracker {
813
874
  * loop.
814
875
  */
815
876
  dispatchedByFailureLifecycle?: boolean;
877
+ /** Re-run lineage, when this run re-runs another. */
878
+ rerunLineage?: RerunLineage;
816
879
  }): Promise<void>;
817
880
  /**
818
881
  * Write one `failed` execution_runs row for a global eval round that never
@@ -903,6 +966,16 @@ export declare class ExecutionTracker {
903
966
  * whole time it sits in the queue and for the run it resumes into.
904
967
  */
905
968
  workflowRepoIdentifier: string;
969
+ /**
970
+ * Commit of the workflow repository the held run dispatches, recorded only
971
+ * when the workflow is defined in another repository.
972
+ */
973
+ workflowSha?: string | null;
974
+ /**
975
+ * The workflow repository's registered branch, recorded only when the
976
+ * workflow is defined in another repository.
977
+ */
978
+ workflowBranch?: string | null;
906
979
  /**
907
980
  * Inherited invoke-chain depth for a run summoned by an invoke gate.
908
981
  * REQUIRED to be threaded by any caller that has one: a hold is resumable,
@@ -917,6 +990,60 @@ export declare class ExecutionTracker {
917
990
  * excluded from batch accumulation (`EventRouter.isFailureLifecycleRun`).
918
991
  */
919
992
  dispatchedByFailureLifecycle?: boolean;
993
+ /** Re-run lineage, when this run re-runs another. */
994
+ rerunLineage?: RerunLineage;
995
+ /**
996
+ * True when the held run is a pre-run global evaluation round rather than a
997
+ * workflow: the structural marker the release and re-run paths branch on,
998
+ * written here because a held round is recorded through this method.
999
+ */
1000
+ isGlobalEvalRound?: boolean;
1001
+ /**
1002
+ * For a held evaluation round: the workflows the round covers, recorded so
1003
+ * its release can refuse when one of them lost its registration while the
1004
+ * round waited. Stored in the row's `trigger_decision` blob.
1005
+ */
1006
+ heldRoundWorkflows?: readonly string[];
1007
+ /**
1008
+ * The source `providerContext` was taken from, when it is not
1009
+ * `routingKey`. Recorded for a held evaluation round, whose release re-drives
1010
+ * the organization-wide pass with that same pair — see
1011
+ * {@link ExecutionTracker.recordGlobalEvalRoundFailureRun}.
1012
+ */
1013
+ dispatchRoutingKey?: string;
1014
+ }): Promise<void>;
1015
+ /**
1016
+ * Claim a held global evaluation round for its release: move it from `held`
1017
+ * to `pending`. Returns false when the row is not a held round — another
1018
+ * release already claimed it — so a re-fired release signal dispatches the
1019
+ * round at most once.
1020
+ *
1021
+ * `pending`, as a resumed per-repository run (`resumeHeldRun`), and not
1022
+ * `running`: orphan recovery fails a `running` row older than its stale
1023
+ * threshold that has no job rows, and a round row keeps the `started_at` of
1024
+ * its hold and never has job rows. No sweeper selects a `pending` row with no
1025
+ * job rows and no workflow timeout.
1026
+ */
1027
+ claimHeldGlobalEvalRound(runId: string): Promise<boolean>;
1028
+ /**
1029
+ * Terminalize a held global evaluation round whose hold was released.
1030
+ *
1031
+ * The released round runs as a job of its own and creates no run row when it
1032
+ * decides, so nothing else ever moves this row out of `held`. `success`
1033
+ * records that the round reached a verdict for its workflow repository;
1034
+ * `failed` that it did not, with the reason. Guarded on `pending`, the
1035
+ * state {@link ExecutionTracker.claimHeldGlobalEvalRound} moved the row to,
1036
+ * so only the release that claimed it settles it.
1037
+ *
1038
+ * A settled round is not an execution: no `executions_total` sample is
1039
+ * counted for a success, and the forwarded frame carries `isGlobalEvalRound`
1040
+ * so the Platform raises no run notification or outbound webhook for it.
1041
+ */
1042
+ completeReleasedGlobalEvalRound(runId: string, outcome: {
1043
+ status: typeof ExecutionRunStatus.enum.success;
1044
+ } | {
1045
+ status: typeof ExecutionRunStatus.enum.failed;
1046
+ reason: string;
920
1047
  }): Promise<void>;
921
1048
  /**
922
1049
  * Flip a `held` run back to `pending` so the resumed dispatch can proceed
@@ -928,6 +1055,19 @@ export declare class ExecutionTracker {
928
1055
  * to `cancelled` and fires the status-change forward so Platform projects it.
929
1056
  */
930
1057
  cancelHeldRun(runId: string, reason: string): Promise<void>;
1058
+ /**
1059
+ * Cancel a run that has no job rows: a run cancelled before it dispatched
1060
+ * any job, a pre-run evaluation round, or a `pending` row a crash left
1061
+ * behind. No job will ever report a terminal status for it, so the cancel
1062
+ * itself writes the terminal row and forwards it, as `cancelHeldRun` does.
1063
+ *
1064
+ * Guarded on the row being non-terminal, not `held`, and having no job rows,
1065
+ * so a run a job registered on in the meantime is left to the job-driven
1066
+ * completion and a held run to the hold-rejection path.
1067
+ * The cancellation reason already stamped on the row wins over `reason`.
1068
+ * Returns whether a row was cancelled.
1069
+ */
1070
+ cancelJoblessRun(runId: string, reason: string): Promise<boolean>;
931
1071
  /**
932
1072
  * Mark a run as failed immediately with a reason message.
933
1073
  *
@@ -1159,6 +1299,7 @@ export declare class ExecutionTracker {
1159
1299
  startedAt: number;
1160
1300
  completedAt?: number;
1161
1301
  durationMs?: number;
1302
+ statusEpoch?: number;
1162
1303
  jobs: Array<{
1163
1304
  jobId: string;
1164
1305
  jobName: string;
@@ -46,5 +46,5 @@ export interface LogChunkSinkDeps {
46
46
  */
47
47
  forwardToPlatform?: (chunk: NormalizedLogChunk) => void;
48
48
  }
49
- export declare function createLogChunkSink(deps: LogChunkSinkDeps): (chunk: NormalizedLogChunk) => void;
49
+ export declare function createLogChunkSink(deps: LogChunkSinkDeps): (chunk: NormalizedLogChunk) => Promise<void>;
50
50
  //# sourceMappingURL=log-chunk-sink.d.ts.map
@@ -74,6 +74,20 @@ export declare class LogWriter {
74
74
  * the agent did not report one, which is recorded as `stdout`.
75
75
  */
76
76
  appendChunk(runId: string, jobName: string, stepIndex: number, lines: string[], timestamp: number, jobId?: string, stepName?: string, stream?: LogStream): Promise<void>;
77
+ /**
78
+ * Make {@link drain} for `runId` wait for `pending` as it waits for an
79
+ * append.
80
+ *
81
+ * For a caller that still has work to do before it can call
82
+ * {@link appendChunk} — resolving the job name that names the storage key.
83
+ * Registering only inside `appendChunk` leaves that work outside the
84
+ * snapshot: a run that completes while the name is being resolved is drained
85
+ * first, the chunk's append lands afterwards, and on object storage its
86
+ * segment is never sealed, so the chunk is never readable. A job whose setup
87
+ * fails sends its last setup lines immediately before its terminal status,
88
+ * which is exactly that window.
89
+ */
90
+ trackPending(runId: string, pending: Promise<unknown>): void;
77
91
  }
78
92
  export {};
79
93
  //# sourceMappingURL=log-writer.d.ts.map
@@ -0,0 +1,8 @@
1
+ /**
2
+ * The storage key of one step's log. The log writer, the `execution_steps`
3
+ * row's `log_path`, and the step-log readers all build it here, so they agree
4
+ * on the layout. A job's workflow-level log is step `-1`, stored as
5
+ * `step--1.log`.
6
+ */
7
+ export declare function stepLogPath(runId: string, jobName: string, stepIndex: number): string;
8
+ //# sourceMappingURL=step-log-path.d.ts.map
@@ -3,8 +3,11 @@
3
3
  *
4
4
  * Resolves a step's stored log file via the same `execution_steps.log_path` +
5
5
  * `LogStorage` access the dashboard step.logs path uses, then returns the lines
6
- * with line-based cursor pagination. `toAgentStepLogs` wraps every line in an
7
- * untrusted envelope (log content is process/user output — never trusted).
6
+ * with line-based cursor pagination. A job's workflow-level log (step `-1`:
7
+ * setup narration such as the clone and the dependency install) has no step
8
+ * row, so its key is derived from the job name the writer used.
9
+ * `toAgentStepLogs` wraps every line in an untrusted envelope (log content is
10
+ * process/user output — never trusted).
8
11
  */
9
12
  import type { Kysely } from 'kysely';
10
13
  import { type AgentStepLogs } from '@kici-dev/engine';
@@ -18,6 +21,11 @@ export interface RawStepLogs {
18
21
  lines: string[];
19
22
  totalLines: number;
20
23
  nextCursor: string | null;
24
+ /**
25
+ * Whether a log is stored for the step. `false` means none was ever written;
26
+ * `true` with no lines means the stored log is empty.
27
+ */
28
+ recorded: boolean;
21
29
  }
22
30
  export interface ReadStepLogArgs {
23
31
  runId: string;
@@ -31,9 +39,19 @@ export interface ReadStepLogArgs {
31
39
  /**
32
40
  * Read a page of a step's log lines. Returns an empty page (not an error) when
33
41
  * the step row or its log file is absent, so a never-emitted-logs step reads as
34
- * `{ lines: [], totalLines: 0, nextCursor: null }`.
42
+ * `{ lines: [], totalLines: 0, nextCursor: null, recorded: false }`.
35
43
  */
36
44
  export declare function readStepLogLines(deps: StepLogReaderDeps, args: ReadStepLogArgs): Promise<RawStepLogs>;
45
+ /**
46
+ * The stored log key of the requested step, or `null` when it has none.
47
+ *
48
+ * A step's row names its log. The workflow-level log is written under a
49
+ * negative index that no `step.status` carries (the wire schema is
50
+ * non-negative), so no row ever names it: its key is built from the job name
51
+ * (see {@link resolveJobName}, which reads the job's row where the log writer
52
+ * reads its in-memory name) and used only when the file exists.
53
+ */
54
+ export declare function resolveStepLogPath(deps: StepLogReaderDeps, args: Pick<ReadStepLogArgs, 'runId' | 'jobId' | 'stepIndex'>): Promise<string | null>;
37
55
  /** Wrap raw step logs into the untrusted-tagged `AgentStepLogs` shape. */
38
56
  export declare function toAgentStepLogs(runId: string, jobId: string, stepIndex: number, raw: RawStepLogs): AgentStepLogs;
39
57
  //# sourceMappingURL=step-log-reader.d.ts.map
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Admin API route for reading the orchestrator's provenance signing keys.
3
+ *
4
+ * GET /api/v1/admin/signing-keys — the trusted keys (every status except
5
+ * `revoked`), oldest first.
6
+ *
7
+ * Returns `{ keys: [...] }` carrying only `SIGNING_KEY_METADATA_COLUMNS`: never
8
+ * the public JWK (served by `/.well-known/jwks.json`) and never the wrapped
9
+ * private key. Backs `kici-admin signing-key list` when the CLI reads over the
10
+ * admin API instead of the database.
11
+ *
12
+ * The keys belong to the whole orchestrator, not to one tenant, so the route
13
+ * requires an unscoped token, and `secret.read`, the permission the other
14
+ * read-only admin inspection routes use.
15
+ */
16
+ import { Hono } from 'hono';
17
+ import type { Kysely } from 'kysely';
18
+ import type { Database } from '../db/types.js';
19
+ import type { RbacEnforcer, Role } from '../secrets/rbac.js';
20
+ export interface AdminSigningKeyRoutesDeps {
21
+ db: Kysely<Database>;
22
+ rbac: RbacEnforcer;
23
+ }
24
+ type AdminEnv = {
25
+ Variables: {
26
+ role: Role;
27
+ userId: string;
28
+ routingKey: string | null;
29
+ };
30
+ };
31
+ export declare function createAdminSigningKeyRoutes(deps: AdminSigningKeyRoutesDeps): Hono<AdminEnv>;
32
+ export {};
33
+ //# sourceMappingURL=admin-signing-keys.d.ts.map
@@ -0,0 +1,159 @@
1
+ /**
2
+ * Which agents may run a job, beyond what its labels say — above all, which
3
+ * agents may run a job that declares `container:`.
4
+ *
5
+ * A container job runs in one of two places. Either the scaler starts an agent
6
+ * inside the job's image (the agent then runs the steps directly), or an
7
+ * ordinary agent nests the job's container through a container runtime on its
8
+ * host. So an agent not started for the job can run it only when the agent can
9
+ * start containers, and an agent started inside one job's image can run nothing
10
+ * else: any other job would run in an image it never declared.
11
+ *
12
+ * {@link canAgentRunJob} is the one predicate; every dispatch path that hands a
13
+ * queued job to an agent asks it.
14
+ */
15
+ import { ScalerBackendType } from '@kici-dev/engine';
16
+ import type { ResourceRequest } from '@kici-dev/engine';
17
+ import type { LabelSetConfig } from './types.js';
18
+ /** What a job asks of the container runtime on the agent that runs it. */
19
+ export declare enum JobContainerNeed {
20
+ /** The job declares no `container:`, so any agent can run it. */
21
+ None = "none",
22
+ /** The job names an image: the agent pulls it and runs the job inside it. */
23
+ Image = "image",
24
+ /** The job names a Dockerfile: the agent builds the image, then runs it. */
25
+ Dockerfile = "dockerfile"
26
+ }
27
+ /**
28
+ * Classify a job's `container:` declaration.
29
+ *
30
+ * Mirrors the agent, which runs a job in container mode whenever its
31
+ * `container` field is set at all. So any non-empty declaration counts, not
32
+ * only a well-formed `image`: routing a job to an agent that would then nest
33
+ * a container it cannot start is the failure this classification prevents.
34
+ */
35
+ export declare function jobContainerNeed(container: unknown): JobContainerNeed;
36
+ /** The self-reported labels that prove an agent's host can start a container. */
37
+ export declare const CONTAINER_RUNTIME_LABELS: readonly string[];
38
+ /**
39
+ * Whether an agent's registered labels prove it can start a container.
40
+ *
41
+ * The agent reports `kici:runtime:docker` / `kici:runtime:podman` when it finds
42
+ * that runtime's socket at startup. `kici:runtime:container-build` does not
43
+ * count: it says a build CLI is on PATH, and a host can have the CLI and no
44
+ * daemon to run the result on.
45
+ */
46
+ export declare function hasContainerRuntime(labels: Iterable<string>): boolean;
47
+ /**
48
+ * Whether a spawn this backend makes for a job that names its own image
49
+ * starts the agent INSIDE that image.
50
+ *
51
+ * - The container backend always does: a spawn carrying the job's image runs
52
+ * that image with the KiCI runtime injected.
53
+ * - The bare-metal backend does only for a label set that declares an `image`
54
+ * and no `binaryPath`. Such a set has no local binary to start, so it can
55
+ * only mean job-image mode; every other bare-metal set starts a local agent
56
+ * that nests the job's container. The backend decides its launch with this
57
+ * same function, so the two cannot disagree.
58
+ * - Every other backend starts an ordinary agent.
59
+ */
60
+ export declare function spawnsInJobImage(backendType: ScalerBackendType, labelSet: Pick<LabelSetConfig, 'image' | 'binaryPath'>): boolean;
61
+ /** The facts about one job that decide which scaler agents may run it. */
62
+ export interface AgentFitJob {
63
+ /**
64
+ * The queued job's id. An agent the scaler started for a job recognizes
65
+ * that job by it. Absent for a job not yet in the queue, which no agent can
66
+ * have been started for.
67
+ */
68
+ jobId?: string;
69
+ /** The cpu / memory shape the job declares, if any. */
70
+ resources?: ResourceRequest;
71
+ /** What the job's `container:` asks of the agent's host. */
72
+ container: JobContainerNeed;
73
+ }
74
+ /**
75
+ * The first agent release whose `kici:runtime:docker` / `kici:runtime:podman`
76
+ * labels describe the runtime its job containers actually start on. From it
77
+ * on, an agent with neither label has told the orchestrator it cannot start a
78
+ * container.
79
+ *
80
+ * Earlier releases (0.6.0 to 0.9.x) report runtime labels too, but read
81
+ * `DOCKER_HOST` only as a socket file path, while their container client
82
+ * honours any `DOCKER_HOST`. An agent pointed at `tcp://…` — a Docker-in-Docker
83
+ * sidecar, say — reports no Docker label and runs container jobs fine, so its
84
+ * missing label proves nothing.
85
+ */
86
+ export declare const MIN_RUNTIME_FACTS_AGENT_VERSION = "0.10.0";
87
+ /**
88
+ * Whether an agent's runtime labels are proof, so that a missing
89
+ * `kici:runtime:docker` / `kici:runtime:podman` label means no runtime: its
90
+ * version is at least {@link MIN_RUNTIME_FACTS_AGENT_VERSION}, or it reports
91
+ * `kici:runtime:job-image`, which only such a release sends.
92
+ *
93
+ * Any other runtime label is not proof: `kici:runtime:container-build` says a
94
+ * build CLI is on PATH, and an older agent reports it beside a remote
95
+ * `DOCKER_HOST` it does not report.
96
+ */
97
+ export declare function reportsRuntimeFacts(agent: {
98
+ labels: Iterable<string>;
99
+ version: string | null | undefined;
100
+ }): boolean;
101
+ /** What the scaler knows about an agent it started. */
102
+ export interface ScalerAgentView {
103
+ /** The job the scaler started the agent for, and whether it runs in that job's image. */
104
+ binding?: {
105
+ jobId: string;
106
+ jobImage: boolean;
107
+ };
108
+ /** The scaler pre-spawned the agent for its warm pool. */
109
+ prespawned: boolean;
110
+ /**
111
+ * Whether the agent's fixed shape leaves a job's declared shape unchanged.
112
+ * Always true for an agent the scaler did not pre-spawn.
113
+ */
114
+ shapeFits(resources: ResourceRequest | undefined): boolean;
115
+ }
116
+ /** The facts about one registered agent that {@link canAgentRunJob} reads. */
117
+ export interface FitAgent {
118
+ /** The labels the agent registered with, its self-reported facts included. */
119
+ labels: ReadonlySet<string>;
120
+ /** The agent's self-reported version, or null when it reported none. */
121
+ version: string | null;
122
+ /** The registry records the agent as scaler-started. */
123
+ scalerManaged: boolean;
124
+ /** Present when this coordinator's scaler started the agent. */
125
+ scaler?: ScalerAgentView | undefined;
126
+ }
127
+ /**
128
+ * The facts `canAgentRunJob` judges a registered agent on: what it reported at
129
+ * registration, and what the scaler that started it (if one did) recorded.
130
+ */
131
+ export declare function fitAgentFor(agent: {
132
+ agentId: string;
133
+ labels: ReadonlySet<string>;
134
+ version: string | null;
135
+ scalerManaged: boolean;
136
+ }, scalerAgentView?: (agentId: string) => ScalerAgentView | undefined): FitAgent;
137
+ /**
138
+ * Whether this agent may run this job. Every dispatch path that hands a
139
+ * queued job to an agent asks this one predicate; labels have already matched.
140
+ *
141
+ * In order:
142
+ *
143
+ * - **The agent started for this job** always runs it: the scaler chose that
144
+ * spawn for this job, image and shape included.
145
+ * - **An agent inside another job's image** runs nothing else. The scaler's
146
+ * record says so, or the agent itself does (`kici:runtime:job-image`).
147
+ * - **A job that declares `container:`** needs an agent that reports
148
+ * `kici:runtime:docker` or `kici:runtime:podman`, whenever the missing label
149
+ * is proof ({@link runtimeLabelIsProof}).
150
+ * - **A pre-spawned (warm) agent** must also match any shape the job declares.
151
+ */
152
+ export declare function canAgentRunJob(agent: FitAgent, job: AgentFitJob): boolean;
153
+ /**
154
+ * Whether {@link canAgentRunJob} can ever answer false for this agent. The
155
+ * queue drain carries the predicate into its claim only when it can: for an
156
+ * agent that runs any job, the drain keeps its single-statement fast path.
157
+ */
158
+ export declare function agentMayRefuse(agent: FitAgent): boolean;
159
+ //# sourceMappingURL=agent-fit.d.ts.map