@kici-dev/orchestrator 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/dist/agent/dispatcher.d.ts +39 -0
  2. package/dist/agent/registry.d.ts +31 -0
  3. package/dist/app.d.ts +6 -0
  4. package/dist/cache/cache-upload-settle.d.ts +75 -0
  5. package/dist/cache/pending-inits.d.ts +9 -0
  6. package/dist/cancel/cancel-run.d.ts +22 -4
  7. package/dist/cancel/peer-job-cancel.d.ts +62 -0
  8. package/dist/cancel/sweep-stuck-cancelling.d.ts +3 -1
  9. package/dist/cli/api-client.d.ts +73 -0
  10. package/dist/cli/commands/cluster-settings.d.ts +31 -1
  11. package/dist/cli/commands/local-hook.d.ts +3 -2
  12. package/dist/cli/commands/local-trigger.d.ts +7 -0
  13. package/dist/cli/commands/peer.d.ts +27 -1
  14. package/dist/cli/commands/scaler.d.ts +40 -4
  15. package/dist/cli.js +2549 -972
  16. package/dist/cluster/cluster-settings-reader.d.ts +8 -2
  17. package/dist/cluster/coordinator-credential.d.ts +45 -0
  18. package/dist/cluster/coordinator.d.ts +52 -10
  19. package/dist/cluster/index.d.ts +1 -0
  20. package/dist/cluster/instance-heartbeat.d.ts +6 -1
  21. package/dist/cluster/join-token.d.ts +2 -0
  22. package/dist/cluster/peer-auth-coordinator.d.ts +13 -0
  23. package/dist/cluster/peer-client.d.ts +41 -4
  24. package/dist/cluster/peer-credentials.d.ts +31 -0
  25. package/dist/cluster/peer-forget.d.ts +138 -0
  26. package/dist/cluster/peer-handler.d.ts +18 -1
  27. package/dist/cluster/scaler-orphans-peer.d.ts +40 -0
  28. package/dist/cluster/settings-propagation.d.ts +85 -0
  29. package/dist/cluster/worker-cluster-settings.d.ts +29 -0
  30. package/dist/config.d.ts +12 -0
  31. package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts +15 -0
  32. package/dist/db/migrations/153_execution_runs_cancel_force.d.ts +16 -0
  33. package/dist/db/migrations/154_cluster_settings_firecracker_api_socket_wait.d.ts +4 -0
  34. package/dist/db/migrations/155_org_settings_reroute_spawn_retry.d.ts +17 -0
  35. package/dist/db/migrations/156_kici_events_match_outcome.d.ts +17 -0
  36. package/dist/db/types.d.ts +31 -0
  37. package/dist/events/event-router.d.ts +7 -1
  38. package/dist/events/event-store.d.ts +20 -3
  39. package/dist/events/types.d.ts +45 -0
  40. package/dist/helpers/windows-batch.d.ts +59 -0
  41. package/dist/index.js +252 -53
  42. package/dist/metrics/prometheus.d.ts +2 -0
  43. package/dist/orchestrator-core.d.ts +7 -0
  44. package/dist/pipeline/changed-files.d.ts +51 -0
  45. package/dist/pipeline/cross-source-tally.d.ts +21 -0
  46. package/dist/pipeline/degraded-reason.d.ts +8 -0
  47. package/dist/pipeline/dispatch-matched-workflow.d.ts +10 -0
  48. package/dist/pipeline/global-eval-round.d.ts +3 -2
  49. package/dist/pipeline/processor.d.ts +9 -1
  50. package/dist/pipeline/rerun-inputs.d.ts +16 -7
  51. package/dist/providers/github/changed-files.d.ts +14 -4
  52. package/dist/providers/universal-git/changed-files.d.ts +6 -0
  53. package/dist/providers/universal-git/config.d.ts +4 -0
  54. package/dist/registration/registration-store.d.ts +10 -0
  55. package/dist/routes/admin-cluster-settings.d.ts +13 -1
  56. package/dist/routes/admin-event-inspect.d.ts +36 -0
  57. package/dist/routes/admin-peer-forget.d.ts +43 -0
  58. package/dist/routes/admin-scaler-orphans.d.ts +64 -0
  59. package/dist/routes/admin.d.ts +23 -0
  60. package/dist/scaler/backend-factory.d.ts +6 -1
  61. package/dist/scaler/bare-metal-backend.d.ts +17 -15
  62. package/dist/scaler/bare-metal-launch.d.ts +90 -0
  63. package/dist/scaler/container-backend.d.ts +2 -0
  64. package/dist/scaler/container-runtime.d.ts +9 -0
  65. package/dist/scaler/firecracker-backend.d.ts +181 -7
  66. package/dist/scaler/live-vms.d.ts +57 -0
  67. package/dist/scaler/manager.d.ts +138 -5
  68. package/dist/scaler/orphan-requests.d.ts +23 -0
  69. package/dist/scaler/overlay-template.d.ts +38 -0
  70. package/dist/scaler/scaler-state-store.d.ts +7 -0
  71. package/dist/scaler/spawn-errors.d.ts +17 -0
  72. package/dist/scaler/startup-orphan-sweep.d.ts +15 -0
  73. package/dist/scaler/types.d.ts +21 -2
  74. package/dist/scaler/vm-job-bindings.d.ts +17 -0
  75. package/dist/secrets/rbac.d.ts +7 -1
  76. package/dist/server.js +7086 -2557
  77. package/dist/stale-detector/workflow-deadline-detector.d.ts +7 -1
  78. package/dist/standalone.js +16473 -11945
  79. package/dist/storage/s3.d.ts +36 -0
  80. package/dist/worker/in-memory-execution-tracker.d.ts +5 -0
  81. package/dist/worker/in-memory-job-queue.d.ts +21 -1
  82. package/dist/worker/queued-reroute-release.d.ts +39 -0
  83. package/dist/worker/reroute-spawn-control.d.ts +58 -0
  84. package/dist/worker/spawn-retry-budget.d.ts +87 -0
  85. package/dist/worker-core.d.ts +11 -2
  86. package/dist/ws/agent-handler.d.ts +24 -3
  87. package/dist/ws/agent-heartbeat.d.ts +2 -2
  88. package/installer-image-digests.json +3 -3
  89. package/package.json +5 -5
  90. package/sbom.spdx.json +53 -53
@@ -0,0 +1,51 @@
1
+ /**
2
+ * The one changed-files resolver every dispatch path calls: same-source, the
3
+ * no-lock-file organization-wide pass, cross-source repo mode, and rerun.
4
+ *
5
+ * `unavailable` means the orchestrator holds no authoritative list; whether the
6
+ * agent can compute one is the range's question, answered by `resolveDiffRange`.
7
+ */
8
+ import { type ChangedFilesStatus, type DiffRange, type LockWorkflow, type SimulatedEvent, type WebhookNormalizer, type WorkflowDecision } from '@kici-dev/engine';
9
+ import type { ProviderBundle } from '../provider-registry.js';
10
+ export interface ResolvedChangedFiles {
11
+ files: string[];
12
+ status: ChangedFilesStatus;
13
+ range: DiffRange;
14
+ }
15
+ /**
16
+ * Resolve the files an event changed through a bundle's fetcher.
17
+ *
18
+ * - no workflow in `workflows` has a `paths` trigger → `skipped`;
19
+ * - a deleted branch → `fetched` + `[]`, whichever provider delivered it;
20
+ * - no fetcher, or a fetcher that throws → `unavailable`, never a failed delivery.
21
+ */
22
+ export declare function resolveEventChangedFiles(args: {
23
+ bundle: Pick<ProviderBundle, 'changedFilesFetcher'>;
24
+ credentials: unknown;
25
+ repoIdentifier: string;
26
+ eventName: string;
27
+ payload: unknown;
28
+ event: SimulatedEvent;
29
+ /** Omitted: always resolve — a re-evaluation has no lock file to consult. */
30
+ workflows?: readonly LockWorkflow[];
31
+ }): Promise<ResolvedChangedFiles>;
32
+ /**
33
+ * Stamp the repository's default branch on a normalized event, so the
34
+ * new-branch range (`resolveDiffRange`) and the agent that receives this event
35
+ * read the same value. `extractDefaultBranch` prefers the normalizer's hook,
36
+ * which covers GitLab's `project.default_branch`.
37
+ */
38
+ export declare function withDefaultBranch(event: SimulatedEvent, payload: unknown, normalizer: WebhookNormalizer): SimulatedEvent;
39
+ /** Copy a resolved list and its status onto the event the matcher reads. */
40
+ export declare function stampChangedFiles(event: SimulatedEvent, resolved: ResolvedChangedFiles): SimulatedEvent;
41
+ /**
42
+ * Downgrade a cross-source pull-request deferral to a conservative match.
43
+ *
44
+ * A cross-source job checks out the registration's commit, never the pull
45
+ * request's head, so no agent can diff the pull request for it. A path
46
+ * decision the orchestrator could not make therefore runs the workflow, as a
47
+ * range-less push does, instead of handing the agent a diff of the wrong
48
+ * commits. Push ranges diff explicit commits and keep their deferral.
49
+ */
50
+ export declare function withoutCrossSourcePrDeferral(decision: WorkflowDecision, event: SimulatedEvent): WorkflowDecision;
51
+ //# sourceMappingURL=changed-files.d.ts.map
@@ -0,0 +1,21 @@
1
+ /**
2
+ * A generic delivery can run both halves — cross-source for other sources'
3
+ * registrations, and the inbound source's own lock when its normalizer reads a
4
+ * repository. The delivery still gets one event-log row: the same-source path
5
+ * writes it, and this wrapper folds the cross-source half into it.
6
+ */
7
+ import type { ProcessingDeps } from './processor.js';
8
+ export interface CrossSourceTally {
9
+ candidatesConsidered: number;
10
+ jobsDispatched: number;
11
+ /** A cross-source candidate evaluated `paths` against an unavailable diff. */
12
+ degraded: boolean;
13
+ }
14
+ /**
15
+ * Wrap `deps.eventLog` so whichever row the same-source path records — corrupt
16
+ * lock, missing lock, skip or processed — also counts the cross-source half's
17
+ * dispatched jobs and carries its degraded reason. A reason the same-source
18
+ * path already set wins.
19
+ */
20
+ export declare function withCrossSourceTally(deps: ProcessingDeps, tally: CrossSourceTally): ProcessingDeps;
21
+ //# sourceMappingURL=cross-source-tally.d.ts.map
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Annotation written to a `processed` event-log row when path filters were
3
+ * evaluated against an unavailable changed-files diff. Reuses the existing
4
+ * free-text field so the outcome is never a silent `processed / matched 0` —
5
+ * no new event-log status enum value is added.
6
+ */
7
+ export declare const DEGRADED_CHANGED_FILES_REASON = "trigger evaluation degraded: changed files unavailable \u2014 path filters matched conservatively";
8
+ //# sourceMappingURL=degraded-reason.d.ts.map
@@ -704,6 +704,8 @@ export declare function materializeStaticJobsSafe(staticJobs: readonly LockJob[]
704
704
  expansionMap: Map<string, readonly string[]>;
705
705
  matrixFailures: RejectedJob[];
706
706
  }>;
707
+ /** The `paths` lists this workflow's matching left to the agent, or undefined. */
708
+ export declare function deferredPathsOf(ctx: Pick<WorkflowDispatchContext, 'decision'>): string[][] | undefined;
707
709
  /**
708
710
  * Scan a lock workflow's always-mode approval timeouts (workflow-level +
709
711
  * each static job) for a non-positive / non-finite `timeoutSeconds`. Returns the
@@ -879,6 +881,8 @@ export declare function catchUpNeedsGatedJobs(args: {
879
881
  }): Promise<void>;
880
882
  /** Why an init result must not lead to a dispatch. */
881
883
  export declare enum InitDispatchSuppression {
884
+ /** The agent's clone diff matched none of the deferred `paths`. */
885
+ Paths = "paths",
882
886
  /** The workflow's own `filter` decided the workflow does not apply. */
883
887
  Filter = "filter",
884
888
  /** The job is already rejected by a context rule, or held for approval. */
@@ -887,6 +891,11 @@ export declare enum InitDispatchSuppression {
887
891
  /**
888
892
  * Decide whether an arrived init result may dispatch its job.
889
893
  *
894
+ * `Paths` is checked first: a workflow whose deferred `paths` did not match
895
+ * never triggered, whatever its filter says. Only an explicit `false`
896
+ * suppresses — an agent that predates deferred paths reports no verdict, and
897
+ * that absence dispatches.
898
+ *
890
899
  * `Filter` requires the workflow to actually declare a filter as well as the
891
900
  * agent to have reported `false`: a buggy or rogue agent must not be able to
892
901
  * suppress a filter-less workflow by inventing the field, and an agent that
@@ -917,6 +926,7 @@ export declare enum InitDispatchSuppression {
917
926
  */
918
927
  export declare function initDispatchSuppression(workflow: Pick<LockWorkflow, 'hasFilter'>, initResult: {
919
928
  filterPassed?: boolean;
929
+ pathsPassed?: boolean;
920
930
  }, jobEnvData: Pick<JobEnvData, 'rejected' | 'held'>): InitDispatchSuppression | null;
921
931
  export interface GeneratedJobConfig {
922
932
  /**
@@ -87,8 +87,9 @@ export declare function hasDeferredGenerator(lockEntry: LockWorkflow): boolean;
87
87
  * those that must go through an eval round first.
88
88
  *
89
89
  * A candidate needs the round when it declares a `filter` (only the agent may
90
- * run the predicate) or carries a needs-free `DynamicJobFn` (only the agent may
91
- * run the generator, and it depends on nothing the run produces). Everything
90
+ * run the predicate), carries a needs-free `DynamicJobFn` (only the agent may
91
+ * run the generator, and it depends on nothing the run produces), or matched
92
+ * with deferred `paths` (only the agent's clone holds the diff). Everything
92
93
  * else — static jobs, and result-aware generators the run evaluates later — is
93
94
  * handed to the dispatch pipeline as declared, so routing it through a round
94
95
  * would add a job dispatch and an agent round trip for nothing.
@@ -49,6 +49,8 @@ import type { RegistrationIndex } from '../registration/registration-index.js';
49
49
  import type { CronScheduler } from '../cron/cron-scheduler.js';
50
50
  import type { GlobalWorkflowPolicy } from '../security/global-workflow-policy.js';
51
51
  import type { EventLogWriter } from '../webhook/event-log.js';
52
+ /** The one event-log method the pipeline calls; a per-delivery wrapper can stand in for the writer. */
53
+ export type EventLogRecorder = Pick<EventLogWriter, 'record'>;
52
54
  import { EventLogSource } from '@kici-dev/engine';
53
55
  import type { LockJob } from '@kici-dev/engine';
54
56
  import type { ContextStore } from '../contexts/context-store.js';
@@ -203,6 +205,12 @@ export declare function resolveLockFileWithFallback(args: {
203
205
  registrationIndex: RegistrationIndex | undefined;
204
206
  lockFileCache: LockFileCache;
205
207
  deliveryId: string;
208
+ /**
209
+ * False when cross-source repo mode already evaluated this delivery's
210
+ * other-source registrations for the same repository: the fallback would
211
+ * re-read exactly those registrations. Defaults to true.
212
+ */
213
+ allowRegistrationFallback?: boolean;
206
214
  }): Promise<{
207
215
  lockFile: FullLockFile | null;
208
216
  resolvedVia: 'inbound' | 'fallback' | 'miss' | 'corrupt';
@@ -476,7 +484,7 @@ export interface ProcessingDeps {
476
484
  /** Global workflow policy for org-level permission enforcement. Optional -- if not set, global workflows are unrestricted. */
477
485
  globalWorkflowPolicy?: GlobalWorkflowPolicy;
478
486
  /** Inbound webhook delivery log writer. Optional -- if not set, deliveries are not persisted to event_log. */
479
- eventLog?: EventLogWriter;
487
+ eventLog?: EventLogRecorder;
480
488
  /** Where this delivery arrived: 'relay' (Platform WS) or 'direct' (HTTP).
481
489
  * Used by the eventLog writer to populate the source column. Defaults to
482
490
  * 'direct' when omitted (independent / direct paths). */
@@ -1,3 +1,10 @@
1
+ /**
2
+ * Inputs every re-run path rebuilds from an original run: its stored webhook
3
+ * payload, the provider binding its row records, the delivery's event name, the
4
+ * normalized event, and its changed files. Shared by the per-repository re-run,
5
+ * the evaluation-round re-run and release, and the cross-repository global
6
+ * re-run, so the paths cannot disagree about how a run's inputs are read back.
7
+ */
1
8
  import type { SimulatedEvent } from '@kici-dev/engine';
2
9
  import type { ProviderRegistry } from '../provider-registry.js';
3
10
  import type { TrustPolicyOutcome } from '../security/trust-policy-gate.js';
@@ -59,7 +66,9 @@ export declare function loadDeliveryEventName(originalRun: OriginalRunRow, deps:
59
66
  /** The verdict an approved security hold stands for. */
60
67
  export declare const RELEASED_DECISION: TrustPolicyOutcome;
61
68
  /**
62
- * The normalized form of the delivery the round was deciding.
69
+ * The normalized form of the delivery the round was deciding, stamped with the
70
+ * repository's default branch as the delivery stamped it: the jobs a re-run
71
+ * dispatches carry this event, and an agent derives a new-branch diff from it.
63
72
  *
64
73
  * Read-only, and it throws — so it runs before the requestId claim, never
65
74
  * inside the re-evaluation.
@@ -69,13 +78,13 @@ export declare function normalizeRoundEvent(originalRun: OriginalRunRow, provide
69
78
  action: string | null;
70
79
  }, payload: Record<string, unknown>, refusal?: RerunRefusal): SimulatedEvent;
71
80
  /**
72
- * Stamp the re-evaluated event with the source repository's changed files.
81
+ * Stamp the re-evaluated event with the source repository's default branch and
82
+ * changed files.
73
83
  *
74
- * Unconditional, unlike the delivery path's fetch: that path skips the fetch
75
- * when no trigger in the source repo's lock file uses path patterns, and a
76
- * scoped re-evaluation has no such lock file to read. An error carries
77
- * `unavailable`, which every path filter downstream already treats
78
- * conservatively.
84
+ * The resolve is unconditional: the delivery path skips it when no trigger in
85
+ * the source repo's lock file uses path patterns, and a scoped re-evaluation
86
+ * has no such lock file to bound it. A bundle with no fetcher, or a fetch
87
+ * error, carries `unavailable`.
79
88
  */
80
89
  export declare function withChangedFiles(opts: {
81
90
  event: SimulatedEvent;
@@ -28,10 +28,15 @@ export declare function withRateLimitRetry<T>(fn: () => Promise<T>, context: str
28
28
  /**
29
29
  * GitHub-specific implementation of ChangedFilesFetcher.
30
30
  *
31
- * For PRs: uses paginated pulls.listFiles API (handles up to 3000 files).
32
- * For pushes: uses repos.compareCommits API (logs warning at 300+ files).
33
- * For initial pushes (zero SHA): returns empty array.
34
- * For unknown events: returns empty array.
31
+ * For PRs: uses the paginated pulls.listFiles API; a result at GitHub's
32
+ * 3000-file cap is truncated, so it reports `unavailable`.
33
+ * For pushes: uses the repos.compareCommits API; a result at GitHub's 300-file
34
+ * cap is truncated, so it reports `unavailable`.
35
+ * For a new branch (zero `before`): compares the default branch against
36
+ * `after` (three-dot), the files the branch adds. With no known default
37
+ * branch, or the default branch itself being created, it reports `unavailable`.
38
+ * For a deleted branch (zero `after`): `fetched` + [].
39
+ * For unknown events: reports `unavailable`.
35
40
  */
36
41
  export declare class GitHubChangedFilesFetcher implements ChangedFilesFetcher {
37
42
  private readonly config;
@@ -55,5 +60,10 @@ export declare class GitHubChangedFilesFetcher implements ChangedFilesFetcher {
55
60
  * Get changed files for a push event using compareCommits API.
56
61
  */
57
62
  private getPushChangedFiles;
63
+ /**
64
+ * Compare two refs through the compareCommits API. A result at GitHub's
65
+ * 300-file cap is truncated, so it reports `unavailable`.
66
+ */
67
+ private compare;
58
68
  }
59
69
  //# sourceMappingURL=changed-files.d.ts.map
@@ -8,6 +8,10 @@
8
8
  * `commits[].{added,modified,removed}` arrays in the push payload, so we
9
9
  * extract them via JSONPath from the already-delivered webhook body.
10
10
  *
11
+ * Forges cap `commits[]`. A preset whose forge sends a total commit count
12
+ * (`commitsTotal`) or has a documented list cap (`commitsCap`) reports a
13
+ * truncated push as `unavailable` rather than a partial list.
14
+ *
11
15
  * For PR events we report `unavailable`: no per-commit diff is present in the
12
16
  * webhook body, so the upstream trigger matcher matches path filters
13
17
  * conservatively (the workflow runs) instead of silently never matching. A
@@ -38,6 +42,8 @@ export declare class UniversalGitChangedFilesFetcher implements ChangedFilesFetc
38
42
  * @param _credentials Unused — no REST call is made.
39
43
  */
40
44
  getChangedFiles(_repoIdentifier: string, eventType: string, payload: unknown, _credentials: unknown): Promise<ChangedFilesResult>;
45
+ /** Whether the payload lists fewer commits than the push carried. */
46
+ private isTruncated;
41
47
  private classifyEvent;
42
48
  }
43
49
  //# sourceMappingURL=changed-files.d.ts.map
@@ -73,6 +73,8 @@ export declare const UniversalGitPayloadPathsSchema: z.ZodObject<{
73
73
  commitsModified: z.ZodString;
74
74
  commitsRemoved: z.ZodString;
75
75
  commitMessage: z.ZodOptional<z.ZodString>;
76
+ commitsTotal: z.ZodOptional<z.ZodString>;
77
+ commitsCap: z.ZodOptional<z.ZodNumber>;
76
78
  }, z.core.$strip>;
77
79
  export type UniversalGitPayloadPaths = z.infer<typeof UniversalGitPayloadPathsSchema>;
78
80
  /**
@@ -131,6 +133,8 @@ export declare const UniversalGitConfigSchema: z.ZodObject<{
131
133
  commitsModified: z.ZodString;
132
134
  commitsRemoved: z.ZodString;
133
135
  commitMessage: z.ZodOptional<z.ZodString>;
136
+ commitsTotal: z.ZodOptional<z.ZodString>;
137
+ commitsCap: z.ZodOptional<z.ZodNumber>;
134
138
  }, z.core.$strip>>;
135
139
  eventMapping: z.ZodOptional<z.ZodObject<{
136
140
  push: z.ZodArray<z.ZodString>;
@@ -74,6 +74,16 @@ interface ReplaceAllOptions {
74
74
  depCacheKey?: DepCacheKey;
75
75
  /** Set of workflow names that should be marked as global */
76
76
  globalWorkflowNames?: Set<string>;
77
+ /**
78
+ * Delete this routing key's rows for the repo whose workflow the incoming set
79
+ * does not name. `true` (the default) is the default-branch push sync: the
80
+ * lock file is the whole truth for the repo. The manual registration route
81
+ * passes `false`: it upserts the workflows it names and leaves every other
82
+ * registration of the repo alone, which is what
83
+ * `kici-admin workflow register-manual` documents and what its direct-DB
84
+ * mode does.
85
+ */
86
+ prune?: boolean;
77
87
  }
78
88
  /**
79
89
  * DB CRUD for workflow_registrations table with atomic upsert-based replace-all.
@@ -4,7 +4,9 @@
4
4
  * Exposes GET / PATCH `/api/v1/admin/cluster-settings` so `kici-admin
5
5
  * cluster-settings` can read and write the single `cluster_settings` row
6
6
  * (id='default') without going through the Platform dashboard proxy — the CLI
7
- * stays operable even when Platform is unavailable.
7
+ * stays operable even when Platform is unavailable. `GET
8
+ * /api/v1/admin/cluster-settings/propagation` reports which settings version
9
+ * each orchestrator this coordinator knows has applied.
8
10
  *
9
11
  * These knobs are fleet-wide, not per-tenant, so the route is deliberately NOT
10
12
  * under `/orgs/:customerId/*`. Mutations require the `secret.write` RBAC
@@ -15,9 +17,19 @@ import { Hono } from 'hono';
15
17
  import { type Kysely } from 'kysely';
16
18
  import type { Database } from '../db/types.js';
17
19
  import type { RbacEnforcer, Role } from '../secrets/rbac.js';
20
+ import type { PeerRegistry } from '../cluster/peer-registry.js';
21
+ /** The live view behind `GET /cluster-settings/propagation`. */
22
+ export interface ClusterSettingsPropagationSource {
23
+ instanceId: string;
24
+ peerRegistry: Pick<PeerRegistry, 'getAllPeers'>;
25
+ /** The cluster-settings version this coordinator's reads currently serve. */
26
+ localVersion: () => number;
27
+ }
18
28
  interface ClusterSettingsRouteDeps {
19
29
  db: Kysely<Database>;
20
30
  rbac: RbacEnforcer;
31
+ /** Mounts `GET /cluster-settings/propagation` when set. */
32
+ propagation?: ClusterSettingsPropagationSource;
21
33
  }
22
34
  type AdminEnv = {
23
35
  Variables: {
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Admin API routes for inspecting internal events (`kici_events`).
3
+ *
4
+ * GET /api/v1/admin/events — list events, newest first (name, outcome, time filters)
5
+ * GET /api/v1/admin/events/:id — one event: its payload and the runs it dispatched
6
+ *
7
+ * Each row carries its dispatch state (`pending`, `leased`, `retrying`,
8
+ * `processed`, `dlq`) and, once processed, how the router resolved it
9
+ * (`matched`, `no-target-repo`, …) with the number of workflows it reached.
10
+ * Both routes need `event_dlq.read`, the permission that already gates reading
11
+ * `kici_events` rows; the payload on `show` also needs
12
+ * `event_log.read_payload`. A routing-key-scoped token sees only events
13
+ * emitted under its own key.
14
+ */
15
+ import { Hono } from 'hono';
16
+ import type { Kysely } from 'kysely';
17
+ import type { Database } from '../db/types.js';
18
+ import type { EventStore } from '../events/event-store.js';
19
+ import type { TokenManager } from '../secrets/token-manager.js';
20
+ import type { RbacEnforcer, Role } from '../secrets/rbac.js';
21
+ export interface AdminEventInspectRoutesDeps {
22
+ db: Kysely<Database>;
23
+ eventStore: EventStore;
24
+ tokenManager: TokenManager;
25
+ rbac: RbacEnforcer;
26
+ }
27
+ type AdminEnv = {
28
+ Variables: {
29
+ role: Role;
30
+ userId: string;
31
+ routingKey: string | null;
32
+ };
33
+ };
34
+ export declare function createAdminEventInspectRoutes(deps: AdminEventInspectRoutesDeps): Hono<AdminEnv>;
35
+ export {};
36
+ //# sourceMappingURL=admin-event-inspect.d.ts.map
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Forget a departed peer, behind `kici-admin peer forget <instance-id>`.
3
+ *
4
+ * Mounted inside `createAdminRoutes` at `/api/v1/admin`, so the Bearer-token
5
+ * auth middleware has already resolved the caller's role. Needs an unscoped
6
+ * token and the `peer.manage` permission (owner, admin).
7
+ *
8
+ * - `POST /api/v1/admin/peers/forget { instanceId, timeoutMs? }` — drop the
9
+ * peer from this coordinator's live peer registry, then from every connected
10
+ * sibling coordinator's, and answer with one result per coordinator.
11
+ *
12
+ * Forgetting never touches the peer's credential (`kici-admin peer revoke`
13
+ * refuses its next connection). A peer that connects again later is
14
+ * registered again as usual.
15
+ */
16
+ import { Hono } from 'hono';
17
+ import type { AccessLogWriter } from '../audit/access-log.js';
18
+ import { type PeerForgetResult } from '../cluster/peer-forget.js';
19
+ import type { RbacEnforcer, Role } from '../secrets/rbac.js';
20
+ type AdminEnv = {
21
+ Variables: {
22
+ role: Role;
23
+ userId: string;
24
+ routingKey: string | null;
25
+ };
26
+ };
27
+ /** What the route needs from the coordinator it runs on. */
28
+ export interface PeerForgetRouteDeps {
29
+ /**
30
+ * Forget `instanceId` here, then on every connected sibling coordinator
31
+ * (skipped when this coordinator keeps the peer). The first result is this
32
+ * coordinator's. `acknowledgeBackstop` lets a forget that switches an
33
+ * event-provision backstop back on go ahead.
34
+ */
35
+ forget: (instanceId: string, timeoutMs: number, acknowledgeBackstop: boolean) => Promise<PeerForgetResult[]>;
36
+ }
37
+ export declare function createPeerForgetRoutes(deps: {
38
+ peerForget: PeerForgetRouteDeps;
39
+ rbac: RbacEnforcer;
40
+ accessLog?: AccessLogWriter;
41
+ }): Hono<AdminEnv>;
42
+ export {};
43
+ //# sourceMappingURL=admin-peer-forget.d.ts.map
@@ -0,0 +1,64 @@
1
+ /**
2
+ * Live orphaned Firecracker VM admin routes, behind `kici-admin scaler orphans`.
3
+ *
4
+ * Mounted inside `createAdminRoutes` at `/api/v1/admin`, so the Bearer-token
5
+ * auth middleware has already resolved the caller's role. Both routes need an
6
+ * unscoped token: a VM belongs to a host, not to one routing key.
7
+ *
8
+ * - `GET /api/v1/admin/scaler/orphans?target=<instance-id>&timeoutMs=<n>` —
9
+ * every live Firecracker VM on the node, with how it is tracked
10
+ * (`scaler.read`).
11
+ * - `POST /api/v1/admin/scaler/orphans/stop { target?, vmIds, timeoutMs? }` —
12
+ * stop the approved VMs nothing tracks (`scaler.manage`).
13
+ *
14
+ * The node is this coordinator, or the peer `target` names, reached over the
15
+ * authenticated peer channel. The coordinator adds the one tracker only it
16
+ * can answer — a non-terminal job in its dispatch queue bound to the VM's
17
+ * agent — and never forwards a bound VM to the node's stop.
18
+ */
19
+ import { Hono } from 'hono';
20
+ import { type PeerScalerOrphansResponse } from '@kici-dev/engine';
21
+ import type { AccessLogWriter } from '../audit/access-log.js';
22
+ import type { RbacEnforcer, Role } from '../secrets/rbac.js';
23
+ import { type ScalerOrphansAnswer, type ScalerOrphansRequest } from '../scaler/orphan-requests.js';
24
+ import { ScalerOrphansForwardFailure } from '../cluster/scaler-orphans-peer.js';
25
+ type AdminEnv = {
26
+ Variables: {
27
+ role: Role;
28
+ userId: string;
29
+ routingKey: string | null;
30
+ };
31
+ };
32
+ /** What the routes need from the coordinator they run on. */
33
+ export interface ScalerOrphansRouteDeps {
34
+ /** This coordinator's instance id. */
35
+ instanceId: string;
36
+ role: 'coordinator' | 'worker';
37
+ /** The cluster role a connected peer registered with, if known. */
38
+ peerRole: (instanceId: string) => 'coordinator' | 'worker' | undefined;
39
+ /** Answer from this coordinator's own host and tracking. */
40
+ answerLocal: (req: ScalerOrphansRequest) => Promise<ScalerOrphansAnswer>;
41
+ /** Forward to a peer over the peer channel. */
42
+ forward: (target: string, req: ScalerOrphansRequest, timeoutMs: number) => Promise<PeerScalerOrphansResponse | ScalerOrphansForwardFailure>;
43
+ /** Non-terminal dispatch-queue jobs bound to these agent ids, keyed by agent id. */
44
+ findBindings: (agentIds: string[]) => Promise<Map<string, string>>;
45
+ /**
46
+ * The cluster instance each of these agent ids is registered with — this
47
+ * coordinator or a peer whose heartbeat lists it — keyed by agent id. An
48
+ * agent can register with an HA sibling of the node that runs its VM.
49
+ */
50
+ findRegistrations: (agentIds: string[]) => Map<string, string>;
51
+ /**
52
+ * The coordinator peers this cluster expects but has no live link to. Their
53
+ * registered agents are unknown, so while any is missing no VM can be shown
54
+ * to be untracked.
55
+ */
56
+ disconnectedCoordinators: () => string[];
57
+ }
58
+ export declare function createScalerOrphansRoutes(deps: {
59
+ orphans: ScalerOrphansRouteDeps;
60
+ rbac: RbacEnforcer;
61
+ accessLog?: AccessLogWriter;
62
+ }): Hono<AdminEnv>;
63
+ export {};
64
+ //# sourceMappingURL=admin-scaler-orphans.d.ts.map
@@ -18,6 +18,9 @@ import type { AgentRegistry } from '../agent/registry.js';
18
18
  import type { SharedConfigStore } from '../config/shared-store.js';
19
19
  import type { ResolvedMasterKeys } from '../secrets/config.js';
20
20
  import { type HeldRunReleaseWiring } from './admin-held-runs.js';
21
+ import { type ClusterSettingsPropagationSource } from './admin-cluster-settings.js';
22
+ import { type ScalerOrphansRouteDeps } from './admin-scaler-orphans.js';
23
+ import { type PeerForgetRouteDeps } from './admin-peer-forget.js';
21
24
  import type { DrainController } from '../drain/drain-controller.js';
22
25
  import type { SourceStore } from '../sources/source-store.js';
23
26
  import type { JoinTokenManager } from '../cluster/join-token.js';
@@ -159,6 +162,26 @@ export interface AdminRouteDeps {
159
162
  * write is best-effort, never gating.
160
163
  */
161
164
  accessLog?: AccessLogWriter;
165
+ /**
166
+ * Optional -- the coordinator's live-VM listing and stop, for this node or a
167
+ * `--target` peer. Backs `GET /api/v1/admin/scaler/orphans` and
168
+ * `POST /api/v1/admin/scaler/orphans/stop` (`kici-admin scaler orphans`).
169
+ * Bound by orchestrator-core once the cluster is up.
170
+ */
171
+ scalerOrphans?: ScalerOrphansRouteDeps;
172
+ /**
173
+ * Optional -- forget a departed peer here and on every connected sibling
174
+ * coordinator. Backs `POST /api/v1/admin/peers/forget` (`kici-admin peer
175
+ * forget`). Bound by orchestrator-core once the cluster is up.
176
+ */
177
+ peerForget?: PeerForgetRouteDeps;
178
+ /**
179
+ * Optional -- the peer view behind
180
+ * `GET /api/v1/admin/cluster-settings/propagation` (the propagation section of
181
+ * `kici-admin cluster-settings show`). Bound by orchestrator-core once the
182
+ * cluster is up; the route is not mounted without it.
183
+ */
184
+ clusterSettingsPropagation?: ClusterSettingsPropagationSource;
162
185
  /**
163
186
  * Optional -- fulfil deferred attestations on demand (mints in the running
164
187
  * orchestrator process, which owns the Platform WS). Backs
@@ -38,7 +38,12 @@ export interface BackendFactoryContext {
38
38
  tokenTtlMs?: number;
39
39
  /** Live per-spawn resolver for the fleet-wide agent-token TTL. */
40
40
  tokenTtlProvider: () => Promise<number>;
41
- /** DB-backed on the leader, in-memory on the worker. */
41
+ /**
42
+ * Live per-spawn resolver for the fleet-wide Firecracker API-socket wait.
43
+ * A coordinator reads `cluster_settings`; a worker reads its pulled snapshot.
44
+ */
45
+ firecrackerApiSocketWaitMsProvider: () => Promise<number>;
46
+ /** DB-backed on a coordinator, in-memory on a worker. */
42
47
  ipAllocator: (params: IpAllocatorParams) => IpAllocator;
43
48
  /**
44
49
  * The already-constructed scaler state store. The event backend's claim
@@ -2,7 +2,8 @@
2
2
  * Bare-metal scaler backend implementation.
3
3
  *
4
4
  * Manages ephemeral agent processes using child_process.spawn.
5
- * Uses detached process groups for clean killing of entire process trees.
5
+ * On Linux and macOS each agent leads its own process group, which teardown
6
+ * signals; on Windows teardown ends the agent's process tree with taskkill.
6
7
  * All agents are single-use: destroyed after job completion.
7
8
  */
8
9
  import { type ToolRequirement } from '@kici-dev/shared';
@@ -98,6 +99,11 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
98
99
  get labelSets(): LabelSetConfig[];
99
100
  getActiveCount(): number;
100
101
  spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
102
+ /**
103
+ * Record that the agent registered. From then on its exit is a disconnect
104
+ * the manager handles, not a failed spawn.
105
+ */
106
+ markRegistered(managedId: string): void;
101
107
  /**
102
108
  * Launch the agent inside the job's own container image.
103
109
  *
@@ -145,23 +151,19 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
145
151
  entry?: ScalerEntry;
146
152
  }): ValidationResult;
147
153
  /**
148
- * Build the (command, args) tuple used to spawn an agent process.
154
+ * End a Windows agent's whole process tree.
149
155
  *
150
- * Two modes:
151
- * - Direct: returns (binaryPath, []) — the historical bare-metal flow.
152
- * Used when enforceCgroups is false, the host is non-Linux, or the
153
- * resolved limits carry no positive cpus/memBytes.
154
- * - systemd-run scope: returns ('systemd-run', [...]) wrapping the
155
- * binary in a transient `--user --scope --slice=kici-scaler` cgroup
156
- * with `CPUQuota=` / `MemoryMax=` properties translated from the
157
- * effective limits. The scope name embeds `agentId` so each agent is
158
- * a separately-named transient unit (and `systemctl --user status
159
- * kici-agent-<agentId>.scope` works).
156
+ * Windows has no process groups to signal and no SIGTERM, and Node.js cannot
157
+ * send a ctrl-C there (`process.kill` terminates the target unconditionally),
158
+ * so the tree is ended at once with `taskkill /T /F`. The pid is cmd.exe's, which runs the agent's `node` and
159
+ * everything it started.
160
160
  *
161
- * `CPUQuota` is expressed as a percent (1.0 cpus = 100%). `MemoryMax`
162
- * accepts a raw byte count.
161
+ * Nothing runs once Node.js has seen the child exit: libuv then releases its
162
+ * handle on the process, Windows may give the pid to an unrelated process,
163
+ * and `/T` would end that process's tree. While the child runs, the open
164
+ * handle keeps the pid ours.
163
165
  */
164
- private buildSpawnInvocation;
166
+ private endWindowsTree;
165
167
  /**
166
168
  * Wait for a process to exit within a timeout.
167
169
  * Returns true if the process exited, false if timed out.