@kici-dev/orchestrator 0.1.14 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +13 -1
  2. package/dist/__test-helpers__/mock-db.d.ts +2 -0
  3. package/dist/agent/dispatcher.d.ts +110 -6
  4. package/dist/agent/registry.d.ts +14 -0
  5. package/dist/app.d.ts +36 -0
  6. package/dist/approvals/apply-decision.d.ts +60 -0
  7. package/dist/approvals/approval-resolver.d.ts +66 -0
  8. package/dist/approvals/step-approval-bridge.d.ts +72 -0
  9. package/dist/approvals/team-membership-lookup.d.ts +13 -0
  10. package/dist/cache/agent-job-failed-error.d.ts +13 -0
  11. package/dist/cache/dispatch-cache-ref-tracker.d.ts +45 -0
  12. package/dist/cache/index.d.ts +2 -0
  13. package/dist/cache/user-cache.d.ts +116 -0
  14. package/dist/cancel/cancel-run.d.ts +56 -0
  15. package/dist/cli/api-client.d.ts +24 -0
  16. package/dist/cli/commands/environment.d.ts +1 -0
  17. package/dist/cli/commands/firecracker/index.d.ts +11 -0
  18. package/dist/cli/commands/firecracker/provision.d.ts +13 -0
  19. package/dist/cli/commands/firecracker/teardown.d.ts +3 -0
  20. package/dist/cli/commands/firecracker/verify.d.ts +3 -0
  21. package/dist/cli/commands/scaler.d.ts +18 -0
  22. package/dist/cli/kici-admin.d.ts +10 -1
  23. package/dist/cli/service/image-digests.d.ts +21 -0
  24. package/dist/cli.js +4148 -455
  25. package/dist/cluster/peer-client.d.ts +23 -1
  26. package/dist/cluster/peer-handler.d.ts +9 -1
  27. package/dist/cluster/peer-registry.d.ts +6 -0
  28. package/dist/config/schema.d.ts +4 -0
  29. package/dist/config.d.ts +13 -0
  30. package/dist/dashboard/handler.d.ts +92 -1
  31. package/dist/db/migrations/026_event_log_lockfile_corrupt.d.ts +11 -0
  32. package/dist/db/migrations/027_workflow_timeout.d.ts +20 -0
  33. package/dist/db/migrations/028_org_settings_user_cache.d.ts +4 -0
  34. package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +16 -0
  35. package/dist/db/migrations/030_held_runs_env_set_null.d.ts +13 -0
  36. package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +19 -0
  37. package/dist/db/migrations/032_org_settings_dispatch_ack_timeout.d.ts +14 -0
  38. package/dist/db/migrations/033_org_settings_approval.d.ts +19 -0
  39. package/dist/db/migrations/034_held_runs_generalize.d.ts +24 -0
  40. package/dist/db/types.d.ts +89 -3
  41. package/dist/diagnostics/bundle-writer.d.ts +1 -17
  42. package/dist/diagnostics/checks/firecracker-network.d.ts +13 -0
  43. package/dist/diagnostics/checks/index.d.ts +2 -1
  44. package/dist/diagnostics/fleet-collector.d.ts +52 -0
  45. package/dist/diagnostics/fleet-constants.d.ts +8 -0
  46. package/dist/diagnostics/fleet-selection.d.ts +15 -0
  47. package/dist/diagnostics/fleet-topology.d.ts +47 -0
  48. package/dist/diagnostics/fleet-wiring.d.ts +60 -0
  49. package/dist/environments/environment-store.d.ts +14 -1
  50. package/dist/environments/held-runs.d.ts +69 -1
  51. package/dist/firecracker/host-network.d.ts +83 -0
  52. package/dist/firecracker/persist.d.ts +14 -0
  53. package/dist/index.d.ts +1 -0
  54. package/dist/index.js +380 -40
  55. package/dist/lockfile-cache.d.ts +1 -1
  56. package/dist/metrics/prometheus.d.ts +8 -0
  57. package/dist/orchestrator-core.d.ts +13 -2
  58. package/dist/pipeline/dispatch-matched-workflow.d.ts +9 -0
  59. package/dist/pipeline/inline-eval.d.ts +17 -2
  60. package/dist/pipeline/process-webhook.d.ts +19 -0
  61. package/dist/pipeline/processor.d.ts +28 -1
  62. package/dist/pipeline/test-pipeline.d.ts +10 -0
  63. package/dist/providers/github/lock-file.d.ts +1 -1
  64. package/dist/providers/internal/lock-file-fetcher.d.ts +3 -2
  65. package/dist/queue/job-queue.d.ts +53 -1
  66. package/dist/reporting/execution-tracker.d.ts +3 -1
  67. package/dist/routes/admin-environments.d.ts +1 -0
  68. package/dist/routes/fleet.d.ts +20 -0
  69. package/dist/scaler/bare-metal-backend.d.ts +1 -0
  70. package/dist/scaler/config.d.ts +4 -2
  71. package/dist/scaler/container-backend.d.ts +3 -2
  72. package/dist/scaler/disk-guard.d.ts +27 -0
  73. package/dist/scaler/firecracker-backend.d.ts +35 -0
  74. package/dist/scaler/manager.d.ts +8 -0
  75. package/dist/scaler/nftables.d.ts +29 -3
  76. package/dist/scaler/reap-orphans.d.ts +27 -0
  77. package/dist/scaler/types.d.ts +28 -1
  78. package/dist/server.js +6138 -1587
  79. package/dist/stale-detector/stale-run-detector.d.ts +18 -0
  80. package/dist/stale-detector/workflow-deadline-detector.d.ts +49 -0
  81. package/dist/standalone.js +18499 -14925
  82. package/dist/storage/filesystem.d.ts +14 -4
  83. package/dist/storage/s3.d.ts +19 -5
  84. package/dist/storage/types.d.ts +32 -5
  85. package/dist/worker/in-memory-job-queue.d.ts +40 -7
  86. package/dist/ws/agent-handler.d.ts +45 -0
  87. package/dist/ws/dashboard-env-handler.d.ts +36 -0
  88. package/dist/ws/fleet-agent-collector.d.ts +23 -0
  89. package/dist/ws/platform-client.d.ts +10 -1
  90. package/installer-image-digests.json +7 -0
  91. package/package.json +15 -11
  92. package/sbom.spdx.json +62 -57
@@ -13,11 +13,20 @@
13
13
  * exported function is a narrative orchestrator that threads the typed
14
14
  * results through the pipeline.
15
15
  */
16
+ import { CacheRefScope } from '@kici-dev/engine';
16
17
  import type { LockWorkflow, SimulatedEvent, WorkflowDecision } from '@kici-dev/engine';
17
18
  import type { WebhookInfo } from '../webhook/handler.js';
18
19
  import type { ProviderBundle } from '../provider-registry.js';
19
20
  import type { TrustResolution } from '../security/trust-resolver.js';
20
21
  import { type ProcessingDeps } from './processor.js';
22
+ /**
23
+ * Trusted refs (write+ contributor, default-branch) get the org-shared cache
24
+ * write scope; everyone else (fork PR, unknown/known-but-not-trusted) is
25
+ * confined to a per-run isolated write scope. Absent trust resolution =>
26
+ * isolated (fail-closed), so an unresolved trust state can never poison the
27
+ * org-shared cache.
28
+ */
29
+ export declare function deriveCacheRefScope(trust: TrustResolution | undefined): CacheRefScope;
21
30
  /**
22
31
  * Context for dispatching a single matched workflow.
23
32
  *
@@ -5,11 +5,12 @@
5
5
  * in a sandboxed VM context at dispatch time, eliminating the init-job
6
6
  * round-trip for pure functions.
7
7
  */
8
+ import type { LockJob } from '@kici-dev/engine';
8
9
  /**
9
10
  * Evaluate an inline expression that returns a string.
10
11
  *
11
12
  * @param expression - Serialized arrow function, e.g. '(event) => event.ref.split("/").pop()'
12
- * @param event - Normalized webhook event payload
13
+ * @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
13
14
  * @returns The string result
14
15
  * @throws TypeError if result is not a string
15
16
  * @throws Error on timeout or sandbox violation
@@ -19,10 +20,24 @@ export declare function evaluateInlineString(expression: string, event: object):
19
20
  * Evaluate an inline expression that returns a Record<string, string>.
20
21
  *
21
22
  * @param expression - Serialized arrow function, e.g. '(event) => ({ NODE_ENV: event.env })'
22
- * @param event - Normalized webhook event payload
23
+ * @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
23
24
  * @returns The record result
24
25
  * @throws TypeError if result is not a plain object
25
26
  * @throws Error on timeout or sandbox violation
26
27
  */
27
28
  export declare function evaluateInlineRecord(expression: string, event: object): Record<string, string>;
29
+ /**
30
+ * Evaluate a lock job's inline (pure-function) dynamic fields against the
31
+ * normalized event envelope (SimulatedEvent shape: { type, action,
32
+ * targetBranch, sourceBranch, payload, … }). The raw provider webhook body is
33
+ * nested at `event.payload` — the same shape rules and step contexts see.
34
+ *
35
+ * Throws a job-attributed Error when any expression throws; inline evaluation
36
+ * failures are immediate dispatch failures (no init-job fallback).
37
+ */
38
+ export declare function evaluateInlineFields(lockJob: LockJob, event: object): {
39
+ inlineEnvironmentName: string | undefined;
40
+ inlineEnv: Record<string, string> | undefined;
41
+ inlineConcurrencyGroup: string | undefined;
42
+ };
28
43
  //# sourceMappingURL=inline-eval.d.ts.map
@@ -16,8 +16,26 @@
16
16
  * Internal helpers are pure phase functions returning typed results; the only
17
17
  * top-level export is `processWebhook`, callable from server.ts / app.ts.
18
18
  */
19
+ import type { SimulatedEvent } from '@kici-dev/engine';
19
20
  import type { WebhookInfo } from '../webhook/handler.js';
21
+ import type { ProviderBundle } from '../provider-registry.js';
22
+ import type { TrustResolution } from '../security/trust-resolver.js';
20
23
  import { type ProcessingDeps } from './processor.js';
24
+ interface TrustOutcome {
25
+ trustResolution: TrustResolution | undefined;
26
+ /** Default 'base' for PR events; trust resolution may override to 'head'. */
27
+ lockFileSource: 'head' | 'base';
28
+ }
29
+ export declare function resolveTrustForPR(args: {
30
+ info: WebhookInfo;
31
+ deps: ProcessingDeps;
32
+ bundle: ProviderBundle;
33
+ event: SimulatedEvent;
34
+ payload: Record<string, unknown>;
35
+ resolvedOrgId: string;
36
+ repoIdentifier: string;
37
+ credentials: Record<string, unknown>;
38
+ }): Promise<TrustOutcome>;
21
39
  /**
22
40
  * Process a webhook through the complete pipeline.
23
41
  *
@@ -35,4 +53,5 @@ import { type ProcessingDeps } from './processor.js';
35
53
  * 10. Forward Platform trace + record event log
36
54
  */
37
55
  export declare function processWebhook(info: WebhookInfo, deps: ProcessingDeps): Promise<void>;
56
+ export {};
38
57
  //# sourceMappingURL=process-webhook.d.ts.map
@@ -30,10 +30,13 @@ import type { CheckRunReporter } from '../reporting/check-run-reporter.js';
30
30
  import type { ExecutionTracker } from '../reporting/execution-tracker.js';
31
31
  import type { AgentRegistry } from '../agent/registry.js';
32
32
  import type { RunCoordinator } from '../cluster/coordinator.js';
33
+ import type { TeamMembershipLookup } from '../approvals/team-membership-lookup.js';
33
34
  import type { LogStorage } from '../reporting/log-storage.js';
34
35
  import type { SecretResolver } from '../secrets/secret-resolver.js';
35
36
  import type { ContributorCache } from '../security/contributor-cache.js';
37
+ import type { AccessLogWriter } from '../audit/access-log.js';
36
38
  import type { LockFile as FullLockFile, LockWorkflow, SimulatedEvent, WebhookNormalizer } from '@kici-dev/engine';
39
+ import { LockFileParseError } from '@kici-dev/engine';
37
40
  import type { EventRouter } from '../events/event-router.js';
38
41
  import type { RegistrationStore } from '../registration/registration-store.js';
39
42
  import type { RegistrationIndex } from '../registration/registration-index.js';
@@ -158,7 +161,7 @@ export declare function resolveLockFileWithFallback(args: {
158
161
  deliveryId: string;
159
162
  }): Promise<{
160
163
  lockFile: FullLockFile | null;
161
- resolvedVia: 'inbound' | 'fallback' | 'miss';
164
+ resolvedVia: 'inbound' | 'fallback' | 'miss' | 'corrupt';
162
165
  fallbackRoutingKey?: string;
163
166
  /** The winning provider bundle when resolvedVia='fallback'. Used by the dispatch
164
167
  * site to swap repoUrlBuilder and cloneTokenProvider (Layer 4 cross-provider fix). */
@@ -166,6 +169,10 @@ export declare function resolveLockFileWithFallback(args: {
166
169
  /** The winning registration's providerContext when resolvedVia='fallback'.
167
170
  * Carries installationId etc. for clone token issuance. */
168
171
  fallbackCredentials?: Record<string, unknown>;
172
+ /** Set when resolvedVia='corrupt': the parse error seen while attempting to
173
+ * resolve a lock file. A valid fallback always wins over a corrupt inbound,
174
+ * so this is only surfaced when NOTHING resolved. */
175
+ corruptError?: LockFileParseError;
169
176
  }>;
170
177
  /**
171
178
  * Resolve the customer/org ID for a routing key.
@@ -208,6 +215,17 @@ export declare function extractCommitMessage(event: string, payload: unknown): s
208
215
  * Build a human-readable summary for a security hold check.
209
216
  */
210
217
  export declare function buildSecurityHoldSummary(reason: string, tier: string, contributorUsername?: string): string;
218
+ /**
219
+ * Build the pending check-run description for a job/workflow approval hold,
220
+ * naming the clauses an approver must satisfy. `{team:X}` renders as
221
+ * "team X", `{user:Y}` as "user Y". An empty clause list (any eligible
222
+ * reviewer satisfies the hold) falls back to a generic message.
223
+ */
224
+ export declare function summarizeApprovalClauses(clauses: ReadonlyArray<{
225
+ team: string;
226
+ } | {
227
+ user: string;
228
+ }>): string;
211
229
  /**
212
230
  * Dependencies for the processing pipeline.
213
231
  * All injected for testability. Fully provider-agnostic.
@@ -278,6 +296,12 @@ export interface ProcessingDeps {
278
296
  identityLinks?: IdentityLink[];
279
297
  /** ci_trust permission levels per user ID from Platform push. Optional -- defaults to empty. */
280
298
  orgMemberPermissions?: Map<string, PermissionLevel>;
299
+ /**
300
+ * Team-membership lookup pushed from the Platform (team name → member set).
301
+ * Consumed by the approval resolver to satisfy `{team}` approval clauses.
302
+ * Optional -- defaults to "no teams".
303
+ */
304
+ teamMembershipLookup?: TeamMembershipLookup;
281
305
  /** Global workflow policy for org-level permission enforcement. Optional -- if not set, global workflows are unrestricted. */
282
306
  globalWorkflowPolicy?: GlobalWorkflowPolicy;
283
307
  /** Inbound webhook delivery log writer. Optional -- if not set, deliveries are not persisted to event_log. */
@@ -291,6 +315,9 @@ export interface ProcessingDeps {
291
315
  * created in server.ts and threaded through both the Platform-relay WS
292
316
  * path and the generic webhook HTTP path. */
293
317
  contributorCache?: ContributorCache;
318
+ /** Access-log writer for the orchestrator audit stream. Optional -- if not
319
+ * set, hold-creation audit rows (`held_run.request`) are skipped. */
320
+ accessLogWriter?: AccessLogWriter;
294
321
  }
295
322
  /**
296
323
  * Check if any trigger in the lock file workflows uses path filters.
@@ -25,6 +25,8 @@ import type { BuildCoordinator } from '../cache/index.js';
25
25
  import type { DepCache } from '../cache/index.js';
26
26
  import type { PendingBuildTracker } from '../cache/index.js';
27
27
  import type { SecretResolver } from '../secrets/secret-resolver.js';
28
+ import type { EnvironmentStore } from '../environments/environment-store.js';
29
+ import type { VariableStore } from '../environments/variable-store.js';
28
30
  import type { LogStorage } from '../reporting/log-storage.js';
29
31
  import type { Kysely } from 'kysely';
30
32
  import type { Database } from '../db/types.js';
@@ -49,6 +51,10 @@ export interface TestTriggerInput {
49
51
  uploadId?: string;
50
52
  /** Fixture secret context mappings (optional). */
51
53
  secrets?: Record<string, string>;
54
+ /** Base64 X25519+AES-GCM blob of the developer's local secrets ({flat, contexts}). */
55
+ encryptedSecrets?: string;
56
+ /** Base64 ephemeral CLI public key that encrypted `encryptedSecrets`. */
57
+ encryptedSecretsKey?: string;
52
58
  /** Direct workflow run -- bypass triggers (optional). */
53
59
  workflowName?: string;
54
60
  /** Resolved overlay metadata for tarball download + decryption. */
@@ -97,6 +103,10 @@ export interface TestPipelineDeps {
97
103
  logStorage?: LogStorage;
98
104
  /** Database connection for environment protection checks. Optional. */
99
105
  db?: Kysely<Database>;
106
+ /** Environment store for resolving environment ids in test dispatch parity. Optional. */
107
+ environmentStore?: EnvironmentStore;
108
+ /** Variable store for resolving environment variables in test dispatch parity. Optional. */
109
+ variableStore?: VariableStore;
100
110
  }
101
111
  /**
102
112
  * Process a test trigger through the existing pipeline.
@@ -7,7 +7,7 @@
7
7
  * NOTE: This is the raw fetcher without caching. The orchestrator wraps
8
8
  * this with an LRU cache (LockFileCache) for production use.
9
9
  */
10
- import type { LockFileFetcher, LockFile } from '@kici-dev/engine';
10
+ import { type LockFileFetcher, type LockFile } from '@kici-dev/engine';
11
11
  import { type GitHubAppConfig } from './auth.js';
12
12
  /**
13
13
  * GitHub-specific implementation of LockFileFetcher.
@@ -25,10 +25,11 @@ export declare class InternalLockFileFetcher implements LockFileFetcher {
25
25
  * Fetch the lock file from the local filesystem.
26
26
  *
27
27
  * @param repoIdentifier - Either a file:// URL or a relative path under repoBasePath
28
- * @param _ref - Git ref (ignored for filesystem access -- always reads current state)
28
+ * @param ref - Git ref (ignored for filesystem access -- always reads current state)
29
29
  * @param _credentials - Not used (file:// access needs no auth)
30
30
  * @returns Parsed LockFile, or null if not found
31
+ * @throws LockFileParseError when the file is present but unparseable/invalid
31
32
  */
32
- fetchLockFile(repoIdentifier: string, _ref: string, _credentials: unknown): Promise<LockFile | null>;
33
+ fetchLockFile(repoIdentifier: string, ref: string, _credentials: unknown): Promise<LockFile | null>;
33
34
  }
34
35
  //# sourceMappingURL=lock-file-fetcher.d.ts.map
@@ -43,6 +43,13 @@ export declare enum DispatchQueueStatus {
43
43
  Expired = "expired",
44
44
  Recovering = "recovering"
45
45
  }
46
+ /**
47
+ * Maximum delivery attempts for a single dispatch_queue job. A job whose
48
+ * `dispatch_attempts` reaches this value is failed permanently instead of
49
+ * being requeued again. Bounds requeue loops from repeated job.reject /
50
+ * pre-start agent loss; `expires_at` is the time-based backstop.
51
+ */
52
+ export declare const MAX_DISPATCH_ATTEMPTS = 5;
46
53
  /**
47
54
  * Input for enqueuing a job. Callers provide these fields;
48
55
  * the queue generates id, status, created_at, and expires_at.
@@ -307,6 +314,34 @@ export declare class JobQueue {
307
314
  * the leader-gated sweep (`sweepExpiredRecoveries()`).
308
315
  */
309
316
  markRecovering(jobId: string, deadline?: Date, agentId?: string): Promise<void>;
317
+ /**
318
+ * Stamp the dispatch-acknowledgment deadline for a dispatched job.
319
+ * Only touches rows still in 'dispatched' for safety.
320
+ */
321
+ setAckDeadline(jobId: string, deadline: Date, agentId: string): Promise<void>;
322
+ /** Clear the ack deadline (agent answered, or the job left 'dispatched'). */
323
+ clearAckDeadline(jobId: string): Promise<void>;
324
+ /**
325
+ * List dispatched rows still awaiting an ack (non-null deadline). Used at
326
+ * coord boot (`Dispatcher.recoverState()`) to re-arm in-memory timers.
327
+ */
328
+ getDispatchedAwaitingAck(): Promise<Array<{
329
+ id: string;
330
+ runId: string;
331
+ agentId: string | null;
332
+ deadline: Date;
333
+ }>>;
334
+ /**
335
+ * List every dispatched row whose ack deadline is in the past. The caller
336
+ * (leader-gated `Dispatcher.sweepExpiredAckDeadlines`) requeues each via
337
+ * the atomic `requeue()` (WHERE status='dispatched'), so racing coords
338
+ * cannot double-requeue.
339
+ */
340
+ listExpiredAckDeadlines(now: Date): Promise<Array<{
341
+ id: string;
342
+ runId: string;
343
+ agentId: string | null;
344
+ }>>;
310
345
  /**
311
346
  * List every job currently in `recovering` state with its persisted
312
347
  * recovery deadline. Used at coord boot (`Dispatcher.recoverState()`)
@@ -325,7 +360,7 @@ export declare class JobQueue {
325
360
  /**
326
361
  * Sweep every `recovering` row whose `recovery_deadline` is in the
327
362
  * past, marking them `failed`. Returns the rows that flipped so the
328
- * caller can fire the per-job `onRecoveryTimeout` hook in process.
363
+ * caller can fire the per-job `onJobFailedPermanently` hook in process.
329
364
  *
330
365
  * Intended for the leader-gated `Dispatcher.sweepExpiredRecoveries`
331
366
  * tick — running on N coords would still be correct (the WHERE
@@ -343,6 +378,23 @@ export declare class JobQueue {
343
378
  * @returns true if the update affected a row (job was still recovering).
344
379
  */
345
380
  markFailedIfRecovering(jobId: string, _reason: string): Promise<boolean>;
381
+ /**
382
+ * Return a dispatched job to the pending queue for re-dispatch, bumping
383
+ * its attempt counter. Used when an agent explicitly rejects a dispatch
384
+ * (job.reject) and when a scaler-managed agent disconnects before the
385
+ * job started. Only flips rows still in 'dispatched' — a job that was
386
+ * concurrently completed / failed / cancelled is left untouched.
387
+ *
388
+ * @returns the post-increment dispatch_attempts, or null when the row
389
+ * was not in 'dispatched' state (nothing requeued).
390
+ */
391
+ requeue(jobId: string): Promise<number | null>;
392
+ /**
393
+ * Get the full QueuedJob row by ID regardless of status. Used by the
394
+ * dispatcher's redispatch path, which needs runsOnLabels / excludeLabels /
395
+ * resources to pick an agent or consult the scaler for a requeued job.
396
+ */
397
+ getFullJobById(jobId: string): Promise<QueuedJob | null>;
346
398
  /**
347
399
  * Mark a job as dispatched only if it is still in 'recovering' state.
348
400
  * Used when an agent reconnects and claims a recovering job.
@@ -219,7 +219,9 @@ export declare class ExecutionTracker {
219
219
  concurrency?: {
220
220
  cancelInProgress?: boolean;
221
221
  max?: number;
222
- }): Promise<void>;
222
+ },
223
+ /** Workflow-level wall-clock timeout in ms from the lock file. Sets the run deadline. */
224
+ workflowTimeoutMs?: number): Promise<void>;
223
225
  /**
224
226
  * Add additional jobs to an already-started execution run.
225
227
  * Used when build jobs are tracked early and regular jobs are dispatched later.
@@ -6,6 +6,7 @@
6
6
  * PATCH /api/v1/admin/environments/:name/policy — update policy fields
7
7
  * GET /api/v1/admin/environments?orgId=<id> — list environments
8
8
  * GET /api/v1/admin/environments/:name?orgId=<id> — show env + vars + bindings
9
+ * DELETE /api/v1/admin/environments/:name?orgId=<id> — delete env (cascades bindings/variables/overrides; pending held runs block with 409)
9
10
  * POST /api/v1/admin/environments/templates — create/update a template
10
11
  * GET /api/v1/admin/environments/:name/variables?orgId=<id> — list org-level variables
11
12
  * PUT /api/v1/admin/environments/:name/variables/:key?orgId=<id> — upsert variable
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Fleet log-collection admin routes.
3
+ *
4
+ * GET /admin/fleet-topology — enumerate the cluster (no fan-out) for
5
+ * `debug-bundle --fleet --list` / `--pick`.
6
+ * POST /admin/fleet-bundle — drive the recursive fan-out and stream the
7
+ * assembled nested ZIP back as an octet-stream.
8
+ *
9
+ * Both are protected by the same Bearer admin-token auth as the other admin
10
+ * routes (token-manager validation).
11
+ */
12
+ import { Hono } from 'hono';
13
+ import type { TokenManager } from '../secrets/token-manager.js';
14
+ import type { FleetRoutesDeps } from '../app.js';
15
+ export interface FleetAdminRouteDeps {
16
+ fleet: FleetRoutesDeps;
17
+ tokenManager: TokenManager;
18
+ }
19
+ export declare function createFleetRoutes(deps: FleetAdminRouteDeps): Hono;
20
+ //# sourceMappingURL=fleet.d.ts.map
@@ -39,6 +39,7 @@ export interface BareMetalScalerBackendOptions {
39
39
  }
40
40
  export declare class BareMetalScalerBackend implements ScalerBackend {
41
41
  readonly type: "bare-metal";
42
+ readonly spawnsOnLocalHost = true;
42
43
  readonly logsSource = "bare-metal";
43
44
  readonly maxAgents: number;
44
45
  private _labelSets;
@@ -89,6 +89,7 @@ export declare const firecrackerNetworkSchema: z.ZodObject<{
89
89
  bridgeName: z.ZodDefault<z.ZodString>;
90
90
  gateway: z.ZodDefault<z.ZodString>;
91
91
  netmask: z.ZodDefault<z.ZodString>;
92
+ table: z.ZodDefault<z.ZodString>;
92
93
  }, z.core.$strip>;
93
94
  /**
94
95
  * Zod schema for the complete scaler configuration file.
@@ -241,10 +242,10 @@ export declare const scalerFileSchema: z.ZodObject<{
241
242
  }, z.core.$strip>>;
242
243
  mandatoryLabels: z.ZodDefault<z.ZodArray<z.ZodString>>;
243
244
  roles: z.ZodPipe<z.ZodOptional<z.ZodArray<z.ZodEnum<{
245
+ all: "all";
244
246
  builder: "builder";
245
247
  "init-runner": "init-runner";
246
- all: "all";
247
- }>>>, z.ZodTransform<string[] | undefined, ("builder" | "init-runner" | "all")[] | undefined>>;
248
+ }>>>, z.ZodTransform<string[] | undefined, ("all" | "builder" | "init-runner")[] | undefined>>;
248
249
  resourceCap: z.ZodOptional<z.ZodPipe<z.ZodObject<{
249
250
  maxCpu: z.ZodOptional<z.ZodNumber>;
250
251
  maxMemory: z.ZodOptional<z.ZodString>;
@@ -272,6 +273,7 @@ export declare const scalerFileSchema: z.ZodObject<{
272
273
  bridgeName: z.ZodDefault<z.ZodString>;
273
274
  gateway: z.ZodDefault<z.ZodString>;
274
275
  netmask: z.ZodDefault<z.ZodString>;
276
+ table: z.ZodDefault<z.ZodString>;
275
277
  }, z.core.$strip>>;
276
278
  }, z.core.$strip>;
277
279
  /**
@@ -8,7 +8,7 @@
8
8
  */
9
9
  import { type ToolRequirement } from '@kici-dev/shared';
10
10
  import type { AgentTokenStore } from '../agent/token-store.js';
11
- import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
11
+ import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, SpawnContext, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
12
12
  /**
13
13
  * Result of runtime detection.
14
14
  */
@@ -49,6 +49,7 @@ export interface ContainerScalerBackendOptions {
49
49
  }
50
50
  export declare class ContainerScalerBackend implements ScalerBackend {
51
51
  readonly type: "container";
52
+ readonly spawnsOnLocalHost: boolean;
52
53
  readonly maxAgents: number;
53
54
  private _labelSets;
54
55
  private readonly name;
@@ -112,7 +113,7 @@ export declare class ContainerScalerBackend implements ScalerBackend {
112
113
  get logsSource(): string;
113
114
  get labelSets(): LabelSetConfig[];
114
115
  getActiveCount(): number;
115
- spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
116
+ spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
116
117
  getScalerContext(agentId: string): Record<string, unknown> | undefined;
117
118
  destroy(managedId: string): Promise<void>;
118
119
  /**
@@ -0,0 +1,27 @@
1
+ import { type ReapCounts } from './reap-orphans.js';
2
+ import type { ScalerConfig } from './index.js';
3
+ /** Default minimum free space before startup is allowed to proceed: 1 GiB. */
4
+ export declare const DEFAULT_DISK_GUARD_THRESHOLD_BYTES: number;
5
+ export interface DiskGuardResult {
6
+ reaped: boolean;
7
+ recovered: boolean;
8
+ freeBytesAfter: number;
9
+ }
10
+ interface DiskGuardOpts {
11
+ scalerConfig: ScalerConfig;
12
+ thresholdBytes?: number;
13
+ /** Injectable for tests. */
14
+ statfsFn?: (path: string) => Promise<{
15
+ bavail: bigint;
16
+ bsize: number;
17
+ }>;
18
+ reapFn?: (cfg: ScalerConfig) => Promise<ReapCounts>;
19
+ }
20
+ /**
21
+ * Run the disk guard. Returns reaped/recovered status; the caller decides
22
+ * whether to proceed or exit with backoff. No-ops (recovered=true) when there
23
+ * is no FC scaler or free space is already above threshold.
24
+ */
25
+ export declare function runDiskGuard(opts: DiskGuardOpts): Promise<DiskGuardResult>;
26
+ export {};
27
+ //# sourceMappingURL=disk-guard.d.ts.map
@@ -61,10 +61,14 @@ export interface FirecrackerScalerBackendOptions {
61
61
  memSizeMib?: number;
62
62
  /** Bridge interface name for TAP attachment */
63
63
  bridgeName: string;
64
+ /** Network CIDR (e.g. '10.0.0.0/24'); supplies the prefix for the bridge gateway CIDR. */
65
+ cidr?: string;
64
66
  /** Gateway IP for guest networking */
65
67
  gateway: string;
66
68
  /** Netmask for guest networking */
67
69
  netmask: string;
70
+ /** nft table name for host-network diagnostics. @default 'kici' */
71
+ table?: string;
68
72
  /** Token store for creating ephemeral agent auth tokens. Optional -- when undefined, no token is injected. */
69
73
  tokenStore?: AgentTokenStore;
70
74
  /** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
@@ -82,6 +86,7 @@ export interface FirecrackerScalerBackendOptions {
82
86
  }
83
87
  export declare class FirecrackerScalerBackend implements ScalerBackend {
84
88
  readonly type: "firecracker";
89
+ readonly spawnsOnLocalHost = true;
85
90
  readonly maxAgents: number;
86
91
  readonly logsSource = "firecracker-serial";
87
92
  /** AbortControllers for file tailing per managed VM (keyed by agent ID) */
@@ -98,8 +103,10 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
98
103
  private readonly vcpuCount;
99
104
  private readonly memSizeMib;
100
105
  private readonly bridgeName;
106
+ private readonly cidr;
101
107
  private readonly gateway;
102
108
  private readonly netmask;
109
+ private readonly table;
103
110
  private readonly tokenStore?;
104
111
  private readonly tokenTtlMs;
105
112
  private readonly roles;
@@ -119,6 +126,18 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
119
126
  getActiveCount(): number;
120
127
  spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
121
128
  getScalerContext(agentId: string): Record<string, unknown> | undefined;
129
+ /**
130
+ * Bridge config for host-network diagnostics (read-only snapshot).
131
+ * `bridgeCidr` is the gateway IP with the network prefix (e.g. '10.0.0.1/24'),
132
+ * which is exactly what `provisionBridge`/`verifyBridge` consume. The prefix
133
+ * comes from the configured network `cidr`; if absent, it is derived from the
134
+ * dotted `netmask`.
135
+ */
136
+ getBridgeConfig(): {
137
+ bridgeName: string;
138
+ bridgeCidr: string;
139
+ table: string;
140
+ };
122
141
  destroy(managedId: string): Promise<void>;
123
142
  shutdownAll(): Promise<void>;
124
143
  /**
@@ -211,6 +230,22 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
211
230
  * Promisified execFile wrapper with 30s timeout.
212
231
  */
213
232
  private execAsync;
233
+ /**
234
+ * Remove a VM's jailer chroot directory.
235
+ *
236
+ * On rootless nodes (requireSudo), the jailer chowns the chroot contents to
237
+ * the jailer uid/gid, leaving the inner directories owned by that uid with
238
+ * mode 0755 — so the orchestrator process (a different, non-root uid) has no
239
+ * write permission on them and a plain `rm` fails with EACCES. destroy() and
240
+ * both cleanupOrphans passes treat removal as best-effort and swallow that
241
+ * failure, so each spawn would otherwise leak a multi-GiB chroot until the
242
+ * data disk fills and the orchestrator crash-loops on ENOSPC (it can no
243
+ * longer write its own files to start, so the in-process orphan sweep never
244
+ * runs to recover). Reclaim ownership via the same sudo-wrapped `chown` path
245
+ * used for `ip` before the `rm` runs as ourselves. On root nodes (requireSudo
246
+ * false) the chown is skipped and `rm` works directly.
247
+ */
248
+ private removeChrootDir;
214
249
  /**
215
250
  * Get the API socket path for an agent's Firecracker VM.
216
251
  */
@@ -59,6 +59,8 @@ export interface ScalerStatus {
59
59
  type: string;
60
60
  activeCount: number;
61
61
  maxAgents: number;
62
+ /** Whether this backend spawns its agents on the orchestrator's own host. */
63
+ spawnsOnLocalHost: boolean;
62
64
  /** Label sets this backend can provision (each entry is a string[] of labels) */
63
65
  labelSets: string[][];
64
66
  /** Sum of `requests` reserved by this scaler's active agents. */
@@ -334,6 +336,12 @@ export declare class ScalerManager {
334
336
  * Returns null if the agent is not scaler-managed (standalone).
335
337
  */
336
338
  getBackendForAgent(agentId: string): string | null;
339
+ /**
340
+ * Get a backend instance by scaler name. Returns undefined for an unknown
341
+ * name. Used by diagnostics to reach backend-specific accessors (e.g. the
342
+ * Firecracker backend's `getBridgeConfig()`).
343
+ */
344
+ getBackend(name: string): ScalerBackend | undefined;
337
345
  /**
338
346
  * Get scaler-specific configuration metadata for a managed agent.
339
347
  * Returns undefined for non-scaler-managed (static) agents.
@@ -17,6 +17,10 @@ import type { NetworkPolicy } from './types.js';
17
17
  * - 'saddr': Match on source IP address (container backends)
18
18
  */
19
19
  export type NftMatchMode = 'iifname' | 'saddr';
20
+ /** RFC1918 private address ranges. */
21
+ export declare const RFC1918_RANGES: string[];
22
+ /** Cloud metadata service range (AWS/GCP/Azure link-local). */
23
+ export declare const METADATA_RANGE = "169.254.0.0/16";
20
24
  /**
21
25
  * Options for nft command execution.
22
26
  */
@@ -41,13 +45,35 @@ export declare function validateNftablesAvailability(opts?: NftOptions): Promise
41
45
  * Idempotent -- safe to call multiple times.
42
46
  */
43
47
  export declare function ensureKiciTable(opts?: NftOptions): Promise<void>;
48
+ /** A single nft rule operation: the verb decides chain placement. */
49
+ export interface NftRuleOp {
50
+ /** 'insert' prepends to the chain head; 'add' appends to the tail. */
51
+ verb: 'insert' | 'add';
52
+ /** Tokens after `nft <verb> rule ip kici forward`. */
53
+ tokens: string[];
54
+ }
55
+ /**
56
+ * Build the per-identifier isolation rule operations.
57
+ *
58
+ * Placement matters because nftables is first-match-wins and the `kici`
59
+ * forward chain is shared: it accumulates rules for every live agent plus
60
+ * any host-level baseline (e.g. wildcard `iifname "kici-*"` RFC1918 drops
61
+ * installed at host bootstrap). Accepts therefore go in via `insert`
62
+ * (chain head) so they beat any pre-existing drop that also matches the
63
+ * traffic; an appended accept after a wildcard drop is dead code. Drops go
64
+ * in via `add` (tail) — they only need to beat the chain's accept policy.
65
+ *
66
+ * Effective per-identifier order: gateway + allowlist accepts (head),
67
+ * RFC1918 / metadata / denyAll drops (tail).
68
+ */
69
+ export declare function buildIsolationRuleOps(matchClause: string[], gatewayIp: string, networkPolicy?: NetworkPolicy): NftRuleOp[];
44
70
  /**
45
71
  * Add network isolation rules for a specific network interface.
46
72
  * Used by Firecracker and container backends.
47
73
  *
48
- * Rule order (top to bottom):
49
- * 1. Accept gateway traffic (MUST be first -- inserted, not appended)
50
- * 2. Accept allowlisted CIDRs (if any)
74
+ * Rule placement (see buildIsolationRuleOps):
75
+ * 1. Accept gateway traffic (inserted at chain head)
76
+ * 2. Accept allowlisted CIDRs (inserted at chain head)
51
77
  * 3. Drop RFC1918 ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16)
52
78
  * 4. Drop cloud metadata (169.254.0.0/16)
53
79
  * 5. Drop all remaining traffic (only if denyAll is true)
@@ -0,0 +1,27 @@
1
+ import type { ScalerConfig } from './index.js';
2
+ /** Per-scaler orphan counts, keyed by scaler name. */
3
+ export type ReapCounts = Record<string, number>;
4
+ /**
5
+ * Reap orphan Firecracker resources (dead-VM chroots + TAPs) for every
6
+ * `firecracker` scaler in the config. Safe to run alongside a live
7
+ * orchestrator: the reap is liveness-driven and only touches dead VMs.
8
+ */
9
+ export declare function reapFirecrackerOrphans(scalerConfig: ScalerConfig): Promise<ReapCounts>;
10
+ /**
11
+ * Reap orphan container resources for every `container` scaler. Unlike the FC
12
+ * reap, container cleanup is unconditional (it removes every kici-managed
13
+ * container), so callers MUST ensure the orchestrator is NOT running before
14
+ * invoking this — otherwise live agents are killed. The CLI command enforces
15
+ * this via the /health gate.
16
+ */
17
+ export declare function reapContainerOrphans(scalerConfig: ScalerConfig): Promise<ReapCounts>;
18
+ /**
19
+ * Reap orphans for all supported backends. Firecracker is always reaped
20
+ * (liveness-driven, safe). Containers are reaped only when includeContainers
21
+ * is set (caller has confirmed the orchestrator is down).
22
+ */
23
+ export declare function reapAllOrphans(opts: {
24
+ scalerConfig: ScalerConfig;
25
+ includeContainers: boolean;
26
+ }): Promise<ReapCounts>;
27
+ //# sourceMappingURL=reap-orphans.d.ts.map