@kici-dev/orchestrator 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/__test-helpers__/mock-db.d.ts +2 -0
- package/dist/agent/dispatcher.d.ts +110 -6
- package/dist/agent/registry.d.ts +14 -0
- package/dist/app.d.ts +36 -0
- package/dist/approvals/apply-decision.d.ts +60 -0
- package/dist/approvals/approval-resolver.d.ts +66 -0
- package/dist/approvals/step-approval-bridge.d.ts +72 -0
- package/dist/approvals/team-membership-lookup.d.ts +13 -0
- package/dist/cache/agent-job-failed-error.d.ts +13 -0
- package/dist/cache/dispatch-cache-ref-tracker.d.ts +45 -0
- package/dist/cache/index.d.ts +2 -0
- package/dist/cache/user-cache.d.ts +116 -0
- package/dist/cancel/cancel-run.d.ts +56 -0
- package/dist/cli/api-client.d.ts +24 -0
- package/dist/cli/commands/environment.d.ts +1 -0
- package/dist/cli/commands/firecracker/index.d.ts +11 -0
- package/dist/cli/commands/firecracker/provision.d.ts +13 -0
- package/dist/cli/commands/firecracker/teardown.d.ts +3 -0
- package/dist/cli/commands/firecracker/verify.d.ts +3 -0
- package/dist/cli/commands/scaler.d.ts +18 -0
- package/dist/cli/kici-admin.d.ts +10 -1
- package/dist/cli/service/image-digests.d.ts +21 -0
- package/dist/cli.js +4148 -455
- package/dist/cluster/peer-client.d.ts +23 -1
- package/dist/cluster/peer-handler.d.ts +9 -1
- package/dist/cluster/peer-registry.d.ts +6 -0
- package/dist/config/schema.d.ts +4 -0
- package/dist/config.d.ts +13 -0
- package/dist/dashboard/handler.d.ts +92 -1
- package/dist/db/migrations/026_event_log_lockfile_corrupt.d.ts +11 -0
- package/dist/db/migrations/027_workflow_timeout.d.ts +20 -0
- package/dist/db/migrations/028_org_settings_user_cache.d.ts +4 -0
- package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +16 -0
- package/dist/db/migrations/030_held_runs_env_set_null.d.ts +13 -0
- package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +19 -0
- package/dist/db/migrations/032_org_settings_dispatch_ack_timeout.d.ts +14 -0
- package/dist/db/migrations/033_org_settings_approval.d.ts +19 -0
- package/dist/db/migrations/034_held_runs_generalize.d.ts +24 -0
- package/dist/db/types.d.ts +89 -3
- package/dist/diagnostics/bundle-writer.d.ts +1 -17
- package/dist/diagnostics/checks/firecracker-network.d.ts +13 -0
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/fleet-collector.d.ts +52 -0
- package/dist/diagnostics/fleet-constants.d.ts +8 -0
- package/dist/diagnostics/fleet-selection.d.ts +15 -0
- package/dist/diagnostics/fleet-topology.d.ts +47 -0
- package/dist/diagnostics/fleet-wiring.d.ts +60 -0
- package/dist/environments/environment-store.d.ts +14 -1
- package/dist/environments/held-runs.d.ts +69 -1
- package/dist/firecracker/host-network.d.ts +83 -0
- package/dist/firecracker/persist.d.ts +14 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +380 -40
- package/dist/lockfile-cache.d.ts +1 -1
- package/dist/metrics/prometheus.d.ts +8 -0
- package/dist/orchestrator-core.d.ts +13 -2
- package/dist/pipeline/dispatch-matched-workflow.d.ts +9 -0
- package/dist/pipeline/inline-eval.d.ts +17 -2
- package/dist/pipeline/process-webhook.d.ts +19 -0
- package/dist/pipeline/processor.d.ts +28 -1
- package/dist/pipeline/test-pipeline.d.ts +10 -0
- package/dist/providers/github/lock-file.d.ts +1 -1
- package/dist/providers/internal/lock-file-fetcher.d.ts +3 -2
- package/dist/queue/job-queue.d.ts +53 -1
- package/dist/reporting/execution-tracker.d.ts +3 -1
- package/dist/routes/admin-environments.d.ts +1 -0
- package/dist/routes/fleet.d.ts +20 -0
- package/dist/scaler/bare-metal-backend.d.ts +1 -0
- package/dist/scaler/config.d.ts +4 -2
- package/dist/scaler/container-backend.d.ts +3 -2
- package/dist/scaler/disk-guard.d.ts +27 -0
- package/dist/scaler/firecracker-backend.d.ts +35 -0
- package/dist/scaler/manager.d.ts +8 -0
- package/dist/scaler/nftables.d.ts +29 -3
- package/dist/scaler/reap-orphans.d.ts +27 -0
- package/dist/scaler/types.d.ts +28 -1
- package/dist/server.js +6138 -1587
- package/dist/stale-detector/stale-run-detector.d.ts +18 -0
- package/dist/stale-detector/workflow-deadline-detector.d.ts +49 -0
- package/dist/standalone.js +18499 -14925
- package/dist/storage/filesystem.d.ts +14 -4
- package/dist/storage/s3.d.ts +19 -5
- package/dist/storage/types.d.ts +32 -5
- package/dist/worker/in-memory-job-queue.d.ts +40 -7
- package/dist/ws/agent-handler.d.ts +45 -0
- package/dist/ws/dashboard-env-handler.d.ts +36 -0
- package/dist/ws/fleet-agent-collector.d.ts +23 -0
- package/dist/ws/platform-client.d.ts +10 -1
- package/installer-image-digests.json +7 -0
- package/package.json +15 -11
- package/sbom.spdx.json +62 -57
|
@@ -13,11 +13,20 @@
|
|
|
13
13
|
* exported function is a narrative orchestrator that threads the typed
|
|
14
14
|
* results through the pipeline.
|
|
15
15
|
*/
|
|
16
|
+
import { CacheRefScope } from '@kici-dev/engine';
|
|
16
17
|
import type { LockWorkflow, SimulatedEvent, WorkflowDecision } from '@kici-dev/engine';
|
|
17
18
|
import type { WebhookInfo } from '../webhook/handler.js';
|
|
18
19
|
import type { ProviderBundle } from '../provider-registry.js';
|
|
19
20
|
import type { TrustResolution } from '../security/trust-resolver.js';
|
|
20
21
|
import { type ProcessingDeps } from './processor.js';
|
|
22
|
+
/**
|
|
23
|
+
* Trusted refs (write+ contributor, default-branch) get the org-shared cache
|
|
24
|
+
* write scope; everyone else (fork PR, unknown/known-but-not-trusted) is
|
|
25
|
+
* confined to a per-run isolated write scope. Absent trust resolution =>
|
|
26
|
+
* isolated (fail-closed), so an unresolved trust state can never poison the
|
|
27
|
+
* org-shared cache.
|
|
28
|
+
*/
|
|
29
|
+
export declare function deriveCacheRefScope(trust: TrustResolution | undefined): CacheRefScope;
|
|
21
30
|
/**
|
|
22
31
|
* Context for dispatching a single matched workflow.
|
|
23
32
|
*
|
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
* in a sandboxed VM context at dispatch time, eliminating the init-job
|
|
6
6
|
* round-trip for pure functions.
|
|
7
7
|
*/
|
|
8
|
+
import type { LockJob } from '@kici-dev/engine';
|
|
8
9
|
/**
|
|
9
10
|
* Evaluate an inline expression that returns a string.
|
|
10
11
|
*
|
|
11
12
|
* @param expression - Serialized arrow function, e.g. '(event) => event.ref.split("/").pop()'
|
|
12
|
-
* @param event - Normalized
|
|
13
|
+
* @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
|
|
13
14
|
* @returns The string result
|
|
14
15
|
* @throws TypeError if result is not a string
|
|
15
16
|
* @throws Error on timeout or sandbox violation
|
|
@@ -19,10 +20,24 @@ export declare function evaluateInlineString(expression: string, event: object):
|
|
|
19
20
|
* Evaluate an inline expression that returns a Record<string, string>.
|
|
20
21
|
*
|
|
21
22
|
* @param expression - Serialized arrow function, e.g. '(event) => ({ NODE_ENV: event.env })'
|
|
22
|
-
* @param event - Normalized
|
|
23
|
+
* @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
|
|
23
24
|
* @returns The record result
|
|
24
25
|
* @throws TypeError if result is not a plain object
|
|
25
26
|
* @throws Error on timeout or sandbox violation
|
|
26
27
|
*/
|
|
27
28
|
export declare function evaluateInlineRecord(expression: string, event: object): Record<string, string>;
|
|
29
|
+
/**
|
|
30
|
+
* Evaluate a lock job's inline (pure-function) dynamic fields against the
|
|
31
|
+
* normalized event envelope (SimulatedEvent shape: { type, action,
|
|
32
|
+
* targetBranch, sourceBranch, payload, … }). The raw provider webhook body is
|
|
33
|
+
* nested at `event.payload` — the same shape rules and step contexts see.
|
|
34
|
+
*
|
|
35
|
+
* Throws a job-attributed Error when any expression throws; inline evaluation
|
|
36
|
+
* failures are immediate dispatch failures (no init-job fallback).
|
|
37
|
+
*/
|
|
38
|
+
export declare function evaluateInlineFields(lockJob: LockJob, event: object): {
|
|
39
|
+
inlineEnvironmentName: string | undefined;
|
|
40
|
+
inlineEnv: Record<string, string> | undefined;
|
|
41
|
+
inlineConcurrencyGroup: string | undefined;
|
|
42
|
+
};
|
|
28
43
|
//# sourceMappingURL=inline-eval.d.ts.map
|
|
@@ -16,8 +16,26 @@
|
|
|
16
16
|
* Internal helpers are pure phase functions returning typed results; the only
|
|
17
17
|
* top-level export is `processWebhook`, callable from server.ts / app.ts.
|
|
18
18
|
*/
|
|
19
|
+
import type { SimulatedEvent } from '@kici-dev/engine';
|
|
19
20
|
import type { WebhookInfo } from '../webhook/handler.js';
|
|
21
|
+
import type { ProviderBundle } from '../provider-registry.js';
|
|
22
|
+
import type { TrustResolution } from '../security/trust-resolver.js';
|
|
20
23
|
import { type ProcessingDeps } from './processor.js';
|
|
24
|
+
interface TrustOutcome {
|
|
25
|
+
trustResolution: TrustResolution | undefined;
|
|
26
|
+
/** Default 'base' for PR events; trust resolution may override to 'head'. */
|
|
27
|
+
lockFileSource: 'head' | 'base';
|
|
28
|
+
}
|
|
29
|
+
export declare function resolveTrustForPR(args: {
|
|
30
|
+
info: WebhookInfo;
|
|
31
|
+
deps: ProcessingDeps;
|
|
32
|
+
bundle: ProviderBundle;
|
|
33
|
+
event: SimulatedEvent;
|
|
34
|
+
payload: Record<string, unknown>;
|
|
35
|
+
resolvedOrgId: string;
|
|
36
|
+
repoIdentifier: string;
|
|
37
|
+
credentials: Record<string, unknown>;
|
|
38
|
+
}): Promise<TrustOutcome>;
|
|
21
39
|
/**
|
|
22
40
|
* Process a webhook through the complete pipeline.
|
|
23
41
|
*
|
|
@@ -35,4 +53,5 @@ import { type ProcessingDeps } from './processor.js';
|
|
|
35
53
|
* 10. Forward Platform trace + record event log
|
|
36
54
|
*/
|
|
37
55
|
export declare function processWebhook(info: WebhookInfo, deps: ProcessingDeps): Promise<void>;
|
|
56
|
+
export {};
|
|
38
57
|
//# sourceMappingURL=process-webhook.d.ts.map
|
|
@@ -30,10 +30,13 @@ import type { CheckRunReporter } from '../reporting/check-run-reporter.js';
|
|
|
30
30
|
import type { ExecutionTracker } from '../reporting/execution-tracker.js';
|
|
31
31
|
import type { AgentRegistry } from '../agent/registry.js';
|
|
32
32
|
import type { RunCoordinator } from '../cluster/coordinator.js';
|
|
33
|
+
import type { TeamMembershipLookup } from '../approvals/team-membership-lookup.js';
|
|
33
34
|
import type { LogStorage } from '../reporting/log-storage.js';
|
|
34
35
|
import type { SecretResolver } from '../secrets/secret-resolver.js';
|
|
35
36
|
import type { ContributorCache } from '../security/contributor-cache.js';
|
|
37
|
+
import type { AccessLogWriter } from '../audit/access-log.js';
|
|
36
38
|
import type { LockFile as FullLockFile, LockWorkflow, SimulatedEvent, WebhookNormalizer } from '@kici-dev/engine';
|
|
39
|
+
import { LockFileParseError } from '@kici-dev/engine';
|
|
37
40
|
import type { EventRouter } from '../events/event-router.js';
|
|
38
41
|
import type { RegistrationStore } from '../registration/registration-store.js';
|
|
39
42
|
import type { RegistrationIndex } from '../registration/registration-index.js';
|
|
@@ -158,7 +161,7 @@ export declare function resolveLockFileWithFallback(args: {
|
|
|
158
161
|
deliveryId: string;
|
|
159
162
|
}): Promise<{
|
|
160
163
|
lockFile: FullLockFile | null;
|
|
161
|
-
resolvedVia: 'inbound' | 'fallback' | 'miss';
|
|
164
|
+
resolvedVia: 'inbound' | 'fallback' | 'miss' | 'corrupt';
|
|
162
165
|
fallbackRoutingKey?: string;
|
|
163
166
|
/** The winning provider bundle when resolvedVia='fallback'. Used by the dispatch
|
|
164
167
|
* site to swap repoUrlBuilder and cloneTokenProvider (Layer 4 cross-provider fix). */
|
|
@@ -166,6 +169,10 @@ export declare function resolveLockFileWithFallback(args: {
|
|
|
166
169
|
/** The winning registration's providerContext when resolvedVia='fallback'.
|
|
167
170
|
* Carries installationId etc. for clone token issuance. */
|
|
168
171
|
fallbackCredentials?: Record<string, unknown>;
|
|
172
|
+
/** Set when resolvedVia='corrupt': the parse error seen while attempting to
|
|
173
|
+
* resolve a lock file. A valid fallback always wins over a corrupt inbound,
|
|
174
|
+
* so this is only surfaced when NOTHING resolved. */
|
|
175
|
+
corruptError?: LockFileParseError;
|
|
169
176
|
}>;
|
|
170
177
|
/**
|
|
171
178
|
* Resolve the customer/org ID for a routing key.
|
|
@@ -208,6 +215,17 @@ export declare function extractCommitMessage(event: string, payload: unknown): s
|
|
|
208
215
|
* Build a human-readable summary for a security hold check.
|
|
209
216
|
*/
|
|
210
217
|
export declare function buildSecurityHoldSummary(reason: string, tier: string, contributorUsername?: string): string;
|
|
218
|
+
/**
|
|
219
|
+
* Build the pending check-run description for a job/workflow approval hold,
|
|
220
|
+
* naming the clauses an approver must satisfy. `{team:X}` renders as
|
|
221
|
+
* "team X", `{user:Y}` as "user Y". An empty clause list (any eligible
|
|
222
|
+
* reviewer satisfies the hold) falls back to a generic message.
|
|
223
|
+
*/
|
|
224
|
+
export declare function summarizeApprovalClauses(clauses: ReadonlyArray<{
|
|
225
|
+
team: string;
|
|
226
|
+
} | {
|
|
227
|
+
user: string;
|
|
228
|
+
}>): string;
|
|
211
229
|
/**
|
|
212
230
|
* Dependencies for the processing pipeline.
|
|
213
231
|
* All injected for testability. Fully provider-agnostic.
|
|
@@ -278,6 +296,12 @@ export interface ProcessingDeps {
|
|
|
278
296
|
identityLinks?: IdentityLink[];
|
|
279
297
|
/** ci_trust permission levels per user ID from Platform push. Optional -- defaults to empty. */
|
|
280
298
|
orgMemberPermissions?: Map<string, PermissionLevel>;
|
|
299
|
+
/**
|
|
300
|
+
* Team-membership lookup pushed from the Platform (team name → member set).
|
|
301
|
+
* Consumed by the approval resolver to satisfy `{team}` approval clauses.
|
|
302
|
+
* Optional -- defaults to "no teams".
|
|
303
|
+
*/
|
|
304
|
+
teamMembershipLookup?: TeamMembershipLookup;
|
|
281
305
|
/** Global workflow policy for org-level permission enforcement. Optional -- if not set, global workflows are unrestricted. */
|
|
282
306
|
globalWorkflowPolicy?: GlobalWorkflowPolicy;
|
|
283
307
|
/** Inbound webhook delivery log writer. Optional -- if not set, deliveries are not persisted to event_log. */
|
|
@@ -291,6 +315,9 @@ export interface ProcessingDeps {
|
|
|
291
315
|
* created in server.ts and threaded through both the Platform-relay WS
|
|
292
316
|
* path and the generic webhook HTTP path. */
|
|
293
317
|
contributorCache?: ContributorCache;
|
|
318
|
+
/** Access-log writer for the orchestrator audit stream. Optional -- if not
|
|
319
|
+
* set, hold-creation audit rows (`held_run.request`) are skipped. */
|
|
320
|
+
accessLogWriter?: AccessLogWriter;
|
|
294
321
|
}
|
|
295
322
|
/**
|
|
296
323
|
* Check if any trigger in the lock file workflows uses path filters.
|
|
@@ -25,6 +25,8 @@ import type { BuildCoordinator } from '../cache/index.js';
|
|
|
25
25
|
import type { DepCache } from '../cache/index.js';
|
|
26
26
|
import type { PendingBuildTracker } from '../cache/index.js';
|
|
27
27
|
import type { SecretResolver } from '../secrets/secret-resolver.js';
|
|
28
|
+
import type { EnvironmentStore } from '../environments/environment-store.js';
|
|
29
|
+
import type { VariableStore } from '../environments/variable-store.js';
|
|
28
30
|
import type { LogStorage } from '../reporting/log-storage.js';
|
|
29
31
|
import type { Kysely } from 'kysely';
|
|
30
32
|
import type { Database } from '../db/types.js';
|
|
@@ -49,6 +51,10 @@ export interface TestTriggerInput {
|
|
|
49
51
|
uploadId?: string;
|
|
50
52
|
/** Fixture secret context mappings (optional). */
|
|
51
53
|
secrets?: Record<string, string>;
|
|
54
|
+
/** Base64 X25519+AES-GCM blob of the developer's local secrets ({flat, contexts}). */
|
|
55
|
+
encryptedSecrets?: string;
|
|
56
|
+
/** Base64 ephemeral CLI public key that encrypted `encryptedSecrets`. */
|
|
57
|
+
encryptedSecretsKey?: string;
|
|
52
58
|
/** Direct workflow run -- bypass triggers (optional). */
|
|
53
59
|
workflowName?: string;
|
|
54
60
|
/** Resolved overlay metadata for tarball download + decryption. */
|
|
@@ -97,6 +103,10 @@ export interface TestPipelineDeps {
|
|
|
97
103
|
logStorage?: LogStorage;
|
|
98
104
|
/** Database connection for environment protection checks. Optional. */
|
|
99
105
|
db?: Kysely<Database>;
|
|
106
|
+
/** Environment store for resolving environment ids in test dispatch parity. Optional. */
|
|
107
|
+
environmentStore?: EnvironmentStore;
|
|
108
|
+
/** Variable store for resolving environment variables in test dispatch parity. Optional. */
|
|
109
|
+
variableStore?: VariableStore;
|
|
100
110
|
}
|
|
101
111
|
/**
|
|
102
112
|
* Process a test trigger through the existing pipeline.
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* NOTE: This is the raw fetcher without caching. The orchestrator wraps
|
|
8
8
|
* this with an LRU cache (LockFileCache) for production use.
|
|
9
9
|
*/
|
|
10
|
-
import type
|
|
10
|
+
import { type LockFileFetcher, type LockFile } from '@kici-dev/engine';
|
|
11
11
|
import { type GitHubAppConfig } from './auth.js';
|
|
12
12
|
/**
|
|
13
13
|
* GitHub-specific implementation of LockFileFetcher.
|
|
@@ -25,10 +25,11 @@ export declare class InternalLockFileFetcher implements LockFileFetcher {
|
|
|
25
25
|
* Fetch the lock file from the local filesystem.
|
|
26
26
|
*
|
|
27
27
|
* @param repoIdentifier - Either a file:// URL or a relative path under repoBasePath
|
|
28
|
-
* @param
|
|
28
|
+
* @param ref - Git ref (ignored for filesystem access -- always reads current state)
|
|
29
29
|
* @param _credentials - Not used (file:// access needs no auth)
|
|
30
30
|
* @returns Parsed LockFile, or null if not found
|
|
31
|
+
* @throws LockFileParseError when the file is present but unparseable/invalid
|
|
31
32
|
*/
|
|
32
|
-
fetchLockFile(repoIdentifier: string,
|
|
33
|
+
fetchLockFile(repoIdentifier: string, ref: string, _credentials: unknown): Promise<LockFile | null>;
|
|
33
34
|
}
|
|
34
35
|
//# sourceMappingURL=lock-file-fetcher.d.ts.map
|
|
@@ -43,6 +43,13 @@ export declare enum DispatchQueueStatus {
|
|
|
43
43
|
Expired = "expired",
|
|
44
44
|
Recovering = "recovering"
|
|
45
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Maximum delivery attempts for a single dispatch_queue job. A job whose
|
|
48
|
+
* `dispatch_attempts` reaches this value is failed permanently instead of
|
|
49
|
+
* being requeued again. Bounds requeue loops from repeated job.reject /
|
|
50
|
+
* pre-start agent loss; `expires_at` is the time-based backstop.
|
|
51
|
+
*/
|
|
52
|
+
export declare const MAX_DISPATCH_ATTEMPTS = 5;
|
|
46
53
|
/**
|
|
47
54
|
* Input for enqueuing a job. Callers provide these fields;
|
|
48
55
|
* the queue generates id, status, created_at, and expires_at.
|
|
@@ -307,6 +314,34 @@ export declare class JobQueue {
|
|
|
307
314
|
* the leader-gated sweep (`sweepExpiredRecoveries()`).
|
|
308
315
|
*/
|
|
309
316
|
markRecovering(jobId: string, deadline?: Date, agentId?: string): Promise<void>;
|
|
317
|
+
/**
|
|
318
|
+
* Stamp the dispatch-acknowledgment deadline for a dispatched job.
|
|
319
|
+
* Only touches rows still in 'dispatched' for safety.
|
|
320
|
+
*/
|
|
321
|
+
setAckDeadline(jobId: string, deadline: Date, agentId: string): Promise<void>;
|
|
322
|
+
/** Clear the ack deadline (agent answered, or the job left 'dispatched'). */
|
|
323
|
+
clearAckDeadline(jobId: string): Promise<void>;
|
|
324
|
+
/**
|
|
325
|
+
* List dispatched rows still awaiting an ack (non-null deadline). Used at
|
|
326
|
+
* coord boot (`Dispatcher.recoverState()`) to re-arm in-memory timers.
|
|
327
|
+
*/
|
|
328
|
+
getDispatchedAwaitingAck(): Promise<Array<{
|
|
329
|
+
id: string;
|
|
330
|
+
runId: string;
|
|
331
|
+
agentId: string | null;
|
|
332
|
+
deadline: Date;
|
|
333
|
+
}>>;
|
|
334
|
+
/**
|
|
335
|
+
* List every dispatched row whose ack deadline is in the past. The caller
|
|
336
|
+
* (leader-gated `Dispatcher.sweepExpiredAckDeadlines`) requeues each via
|
|
337
|
+
* the atomic `requeue()` (WHERE status='dispatched'), so racing coords
|
|
338
|
+
* cannot double-requeue.
|
|
339
|
+
*/
|
|
340
|
+
listExpiredAckDeadlines(now: Date): Promise<Array<{
|
|
341
|
+
id: string;
|
|
342
|
+
runId: string;
|
|
343
|
+
agentId: string | null;
|
|
344
|
+
}>>;
|
|
310
345
|
/**
|
|
311
346
|
* List every job currently in `recovering` state with its persisted
|
|
312
347
|
* recovery deadline. Used at coord boot (`Dispatcher.recoverState()`)
|
|
@@ -325,7 +360,7 @@ export declare class JobQueue {
|
|
|
325
360
|
/**
|
|
326
361
|
* Sweep every `recovering` row whose `recovery_deadline` is in the
|
|
327
362
|
* past, marking them `failed`. Returns the rows that flipped so the
|
|
328
|
-
* caller can fire the per-job `
|
|
363
|
+
* caller can fire the per-job `onJobFailedPermanently` hook in process.
|
|
329
364
|
*
|
|
330
365
|
* Intended for the leader-gated `Dispatcher.sweepExpiredRecoveries`
|
|
331
366
|
* tick — running on N coords would still be correct (the WHERE
|
|
@@ -343,6 +378,23 @@ export declare class JobQueue {
|
|
|
343
378
|
* @returns true if the update affected a row (job was still recovering).
|
|
344
379
|
*/
|
|
345
380
|
markFailedIfRecovering(jobId: string, _reason: string): Promise<boolean>;
|
|
381
|
+
/**
|
|
382
|
+
* Return a dispatched job to the pending queue for re-dispatch, bumping
|
|
383
|
+
* its attempt counter. Used when an agent explicitly rejects a dispatch
|
|
384
|
+
* (job.reject) and when a scaler-managed agent disconnects before the
|
|
385
|
+
* job started. Only flips rows still in 'dispatched' — a job that was
|
|
386
|
+
* concurrently completed / failed / cancelled is left untouched.
|
|
387
|
+
*
|
|
388
|
+
* @returns the post-increment dispatch_attempts, or null when the row
|
|
389
|
+
* was not in 'dispatched' state (nothing requeued).
|
|
390
|
+
*/
|
|
391
|
+
requeue(jobId: string): Promise<number | null>;
|
|
392
|
+
/**
|
|
393
|
+
* Get the full QueuedJob row by ID regardless of status. Used by the
|
|
394
|
+
* dispatcher's redispatch path, which needs runsOnLabels / excludeLabels /
|
|
395
|
+
* resources to pick an agent or consult the scaler for a requeued job.
|
|
396
|
+
*/
|
|
397
|
+
getFullJobById(jobId: string): Promise<QueuedJob | null>;
|
|
346
398
|
/**
|
|
347
399
|
* Mark a job as dispatched only if it is still in 'recovering' state.
|
|
348
400
|
* Used when an agent reconnects and claims a recovering job.
|
|
@@ -219,7 +219,9 @@ export declare class ExecutionTracker {
|
|
|
219
219
|
concurrency?: {
|
|
220
220
|
cancelInProgress?: boolean;
|
|
221
221
|
max?: number;
|
|
222
|
-
}
|
|
222
|
+
},
|
|
223
|
+
/** Workflow-level wall-clock timeout in ms from the lock file. Sets the run deadline. */
|
|
224
|
+
workflowTimeoutMs?: number): Promise<void>;
|
|
223
225
|
/**
|
|
224
226
|
* Add additional jobs to an already-started execution run.
|
|
225
227
|
* Used when build jobs are tracked early and regular jobs are dispatched later.
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* PATCH /api/v1/admin/environments/:name/policy — update policy fields
|
|
7
7
|
* GET /api/v1/admin/environments?orgId=<id> — list environments
|
|
8
8
|
* GET /api/v1/admin/environments/:name?orgId=<id> — show env + vars + bindings
|
|
9
|
+
* DELETE /api/v1/admin/environments/:name?orgId=<id> — delete env (cascades bindings/variables/overrides; pending held runs block with 409)
|
|
9
10
|
* POST /api/v1/admin/environments/templates — create/update a template
|
|
10
11
|
* GET /api/v1/admin/environments/:name/variables?orgId=<id> — list org-level variables
|
|
11
12
|
* PUT /api/v1/admin/environments/:name/variables/:key?orgId=<id> — upsert variable
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fleet log-collection admin routes.
|
|
3
|
+
*
|
|
4
|
+
* GET /admin/fleet-topology — enumerate the cluster (no fan-out) for
|
|
5
|
+
* `debug-bundle --fleet --list` / `--pick`.
|
|
6
|
+
* POST /admin/fleet-bundle — drive the recursive fan-out and stream the
|
|
7
|
+
* assembled nested ZIP back as an octet-stream.
|
|
8
|
+
*
|
|
9
|
+
* Both are protected by the same Bearer admin-token auth as the other admin
|
|
10
|
+
* routes (token-manager validation).
|
|
11
|
+
*/
|
|
12
|
+
import { Hono } from 'hono';
|
|
13
|
+
import type { TokenManager } from '../secrets/token-manager.js';
|
|
14
|
+
import type { FleetRoutesDeps } from '../app.js';
|
|
15
|
+
export interface FleetAdminRouteDeps {
|
|
16
|
+
fleet: FleetRoutesDeps;
|
|
17
|
+
tokenManager: TokenManager;
|
|
18
|
+
}
|
|
19
|
+
export declare function createFleetRoutes(deps: FleetAdminRouteDeps): Hono;
|
|
20
|
+
//# sourceMappingURL=fleet.d.ts.map
|
|
@@ -39,6 +39,7 @@ export interface BareMetalScalerBackendOptions {
|
|
|
39
39
|
}
|
|
40
40
|
export declare class BareMetalScalerBackend implements ScalerBackend {
|
|
41
41
|
readonly type: "bare-metal";
|
|
42
|
+
readonly spawnsOnLocalHost = true;
|
|
42
43
|
readonly logsSource = "bare-metal";
|
|
43
44
|
readonly maxAgents: number;
|
|
44
45
|
private _labelSets;
|
package/dist/scaler/config.d.ts
CHANGED
|
@@ -89,6 +89,7 @@ export declare const firecrackerNetworkSchema: z.ZodObject<{
|
|
|
89
89
|
bridgeName: z.ZodDefault<z.ZodString>;
|
|
90
90
|
gateway: z.ZodDefault<z.ZodString>;
|
|
91
91
|
netmask: z.ZodDefault<z.ZodString>;
|
|
92
|
+
table: z.ZodDefault<z.ZodString>;
|
|
92
93
|
}, z.core.$strip>;
|
|
93
94
|
/**
|
|
94
95
|
* Zod schema for the complete scaler configuration file.
|
|
@@ -241,10 +242,10 @@ export declare const scalerFileSchema: z.ZodObject<{
|
|
|
241
242
|
}, z.core.$strip>>;
|
|
242
243
|
mandatoryLabels: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
243
244
|
roles: z.ZodPipe<z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
245
|
+
all: "all";
|
|
244
246
|
builder: "builder";
|
|
245
247
|
"init-runner": "init-runner";
|
|
246
|
-
|
|
247
|
-
}>>>, z.ZodTransform<string[] | undefined, ("builder" | "init-runner" | "all")[] | undefined>>;
|
|
248
|
+
}>>>, z.ZodTransform<string[] | undefined, ("all" | "builder" | "init-runner")[] | undefined>>;
|
|
248
249
|
resourceCap: z.ZodOptional<z.ZodPipe<z.ZodObject<{
|
|
249
250
|
maxCpu: z.ZodOptional<z.ZodNumber>;
|
|
250
251
|
maxMemory: z.ZodOptional<z.ZodString>;
|
|
@@ -272,6 +273,7 @@ export declare const scalerFileSchema: z.ZodObject<{
|
|
|
272
273
|
bridgeName: z.ZodDefault<z.ZodString>;
|
|
273
274
|
gateway: z.ZodDefault<z.ZodString>;
|
|
274
275
|
netmask: z.ZodDefault<z.ZodString>;
|
|
276
|
+
table: z.ZodDefault<z.ZodString>;
|
|
275
277
|
}, z.core.$strip>>;
|
|
276
278
|
}, z.core.$strip>;
|
|
277
279
|
/**
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { type ToolRequirement } from '@kici-dev/shared';
|
|
10
10
|
import type { AgentTokenStore } from '../agent/token-store.js';
|
|
11
|
-
import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
|
|
11
|
+
import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, SpawnContext, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
|
|
12
12
|
/**
|
|
13
13
|
* Result of runtime detection.
|
|
14
14
|
*/
|
|
@@ -49,6 +49,7 @@ export interface ContainerScalerBackendOptions {
|
|
|
49
49
|
}
|
|
50
50
|
export declare class ContainerScalerBackend implements ScalerBackend {
|
|
51
51
|
readonly type: "container";
|
|
52
|
+
readonly spawnsOnLocalHost: boolean;
|
|
52
53
|
readonly maxAgents: number;
|
|
53
54
|
private _labelSets;
|
|
54
55
|
private readonly name;
|
|
@@ -112,7 +113,7 @@ export declare class ContainerScalerBackend implements ScalerBackend {
|
|
|
112
113
|
get logsSource(): string;
|
|
113
114
|
get labelSets(): LabelSetConfig[];
|
|
114
115
|
getActiveCount(): number;
|
|
115
|
-
spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
|
|
116
|
+
spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
|
|
116
117
|
getScalerContext(agentId: string): Record<string, unknown> | undefined;
|
|
117
118
|
destroy(managedId: string): Promise<void>;
|
|
118
119
|
/**
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type ReapCounts } from './reap-orphans.js';
|
|
2
|
+
import type { ScalerConfig } from './index.js';
|
|
3
|
+
/** Default minimum free space before startup is allowed to proceed: 1 GiB. */
|
|
4
|
+
export declare const DEFAULT_DISK_GUARD_THRESHOLD_BYTES: number;
|
|
5
|
+
export interface DiskGuardResult {
|
|
6
|
+
reaped: boolean;
|
|
7
|
+
recovered: boolean;
|
|
8
|
+
freeBytesAfter: number;
|
|
9
|
+
}
|
|
10
|
+
interface DiskGuardOpts {
|
|
11
|
+
scalerConfig: ScalerConfig;
|
|
12
|
+
thresholdBytes?: number;
|
|
13
|
+
/** Injectable for tests. */
|
|
14
|
+
statfsFn?: (path: string) => Promise<{
|
|
15
|
+
bavail: bigint;
|
|
16
|
+
bsize: number;
|
|
17
|
+
}>;
|
|
18
|
+
reapFn?: (cfg: ScalerConfig) => Promise<ReapCounts>;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Run the disk guard. Returns reaped/recovered status; the caller decides
|
|
22
|
+
* whether to proceed or exit with backoff. No-ops (recovered=true) when there
|
|
23
|
+
* is no FC scaler or free space is already above threshold.
|
|
24
|
+
*/
|
|
25
|
+
export declare function runDiskGuard(opts: DiskGuardOpts): Promise<DiskGuardResult>;
|
|
26
|
+
export {};
|
|
27
|
+
//# sourceMappingURL=disk-guard.d.ts.map
|
|
@@ -61,10 +61,14 @@ export interface FirecrackerScalerBackendOptions {
|
|
|
61
61
|
memSizeMib?: number;
|
|
62
62
|
/** Bridge interface name for TAP attachment */
|
|
63
63
|
bridgeName: string;
|
|
64
|
+
/** Network CIDR (e.g. '10.0.0.0/24'); supplies the prefix for the bridge gateway CIDR. */
|
|
65
|
+
cidr?: string;
|
|
64
66
|
/** Gateway IP for guest networking */
|
|
65
67
|
gateway: string;
|
|
66
68
|
/** Netmask for guest networking */
|
|
67
69
|
netmask: string;
|
|
70
|
+
/** nft table name for host-network diagnostics. @default 'kici' */
|
|
71
|
+
table?: string;
|
|
68
72
|
/** Token store for creating ephemeral agent auth tokens. Optional -- when undefined, no token is injected. */
|
|
69
73
|
tokenStore?: AgentTokenStore;
|
|
70
74
|
/** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
|
|
@@ -82,6 +86,7 @@ export interface FirecrackerScalerBackendOptions {
|
|
|
82
86
|
}
|
|
83
87
|
export declare class FirecrackerScalerBackend implements ScalerBackend {
|
|
84
88
|
readonly type: "firecracker";
|
|
89
|
+
readonly spawnsOnLocalHost = true;
|
|
85
90
|
readonly maxAgents: number;
|
|
86
91
|
readonly logsSource = "firecracker-serial";
|
|
87
92
|
/** AbortControllers for file tailing per managed VM (keyed by agent ID) */
|
|
@@ -98,8 +103,10 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
|
|
|
98
103
|
private readonly vcpuCount;
|
|
99
104
|
private readonly memSizeMib;
|
|
100
105
|
private readonly bridgeName;
|
|
106
|
+
private readonly cidr;
|
|
101
107
|
private readonly gateway;
|
|
102
108
|
private readonly netmask;
|
|
109
|
+
private readonly table;
|
|
103
110
|
private readonly tokenStore?;
|
|
104
111
|
private readonly tokenTtlMs;
|
|
105
112
|
private readonly roles;
|
|
@@ -119,6 +126,18 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
|
|
|
119
126
|
getActiveCount(): number;
|
|
120
127
|
spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
|
|
121
128
|
getScalerContext(agentId: string): Record<string, unknown> | undefined;
|
|
129
|
+
/**
|
|
130
|
+
* Bridge config for host-network diagnostics (read-only snapshot).
|
|
131
|
+
* `bridgeCidr` is the gateway IP with the network prefix (e.g. '10.0.0.1/24'),
|
|
132
|
+
* which is exactly what `provisionBridge`/`verifyBridge` consume. The prefix
|
|
133
|
+
* comes from the configured network `cidr`; if absent, it is derived from the
|
|
134
|
+
* dotted `netmask`.
|
|
135
|
+
*/
|
|
136
|
+
getBridgeConfig(): {
|
|
137
|
+
bridgeName: string;
|
|
138
|
+
bridgeCidr: string;
|
|
139
|
+
table: string;
|
|
140
|
+
};
|
|
122
141
|
destroy(managedId: string): Promise<void>;
|
|
123
142
|
shutdownAll(): Promise<void>;
|
|
124
143
|
/**
|
|
@@ -211,6 +230,22 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
|
|
|
211
230
|
* Promisified execFile wrapper with 30s timeout.
|
|
212
231
|
*/
|
|
213
232
|
private execAsync;
|
|
233
|
+
/**
|
|
234
|
+
* Remove a VM's jailer chroot directory.
|
|
235
|
+
*
|
|
236
|
+
* On rootless nodes (requireSudo), the jailer chowns the chroot contents to
|
|
237
|
+
* the jailer uid/gid, leaving the inner directories owned by that uid with
|
|
238
|
+
* mode 0755 — so the orchestrator process (a different, non-root uid) has no
|
|
239
|
+
* write permission on them and a plain `rm` fails with EACCES. destroy() and
|
|
240
|
+
* both cleanupOrphans passes treat removal as best-effort and swallow that
|
|
241
|
+
* failure, so each spawn would otherwise leak a multi-GiB chroot until the
|
|
242
|
+
* data disk fills and the orchestrator crash-loops on ENOSPC (it can no
|
|
243
|
+
* longer write its own files to start, so the in-process orphan sweep never
|
|
244
|
+
* runs to recover). Reclaim ownership via the same sudo-wrapped `chown` path
|
|
245
|
+
* used for `ip` before the `rm` runs as ourselves. On root nodes (requireSudo
|
|
246
|
+
* false) the chown is skipped and `rm` works directly.
|
|
247
|
+
*/
|
|
248
|
+
private removeChrootDir;
|
|
214
249
|
/**
|
|
215
250
|
* Get the API socket path for an agent's Firecracker VM.
|
|
216
251
|
*/
|
package/dist/scaler/manager.d.ts
CHANGED
|
@@ -59,6 +59,8 @@ export interface ScalerStatus {
|
|
|
59
59
|
type: string;
|
|
60
60
|
activeCount: number;
|
|
61
61
|
maxAgents: number;
|
|
62
|
+
/** Whether this backend spawns its agents on the orchestrator's own host. */
|
|
63
|
+
spawnsOnLocalHost: boolean;
|
|
62
64
|
/** Label sets this backend can provision (each entry is a string[] of labels) */
|
|
63
65
|
labelSets: string[][];
|
|
64
66
|
/** Sum of `requests` reserved by this scaler's active agents. */
|
|
@@ -334,6 +336,12 @@ export declare class ScalerManager {
|
|
|
334
336
|
* Returns null if the agent is not scaler-managed (standalone).
|
|
335
337
|
*/
|
|
336
338
|
getBackendForAgent(agentId: string): string | null;
|
|
339
|
+
/**
|
|
340
|
+
* Get a backend instance by scaler name. Returns undefined for an unknown
|
|
341
|
+
* name. Used by diagnostics to reach backend-specific accessors (e.g. the
|
|
342
|
+
* Firecracker backend's `getBridgeConfig()`).
|
|
343
|
+
*/
|
|
344
|
+
getBackend(name: string): ScalerBackend | undefined;
|
|
337
345
|
/**
|
|
338
346
|
* Get scaler-specific configuration metadata for a managed agent.
|
|
339
347
|
* Returns undefined for non-scaler-managed (static) agents.
|
|
@@ -17,6 +17,10 @@ import type { NetworkPolicy } from './types.js';
|
|
|
17
17
|
* - 'saddr': Match on source IP address (container backends)
|
|
18
18
|
*/
|
|
19
19
|
export type NftMatchMode = 'iifname' | 'saddr';
|
|
20
|
+
/** RFC1918 private address ranges. */
|
|
21
|
+
export declare const RFC1918_RANGES: string[];
|
|
22
|
+
/** Cloud metadata service range (AWS/GCP/Azure link-local). */
|
|
23
|
+
export declare const METADATA_RANGE = "169.254.0.0/16";
|
|
20
24
|
/**
|
|
21
25
|
* Options for nft command execution.
|
|
22
26
|
*/
|
|
@@ -41,13 +45,35 @@ export declare function validateNftablesAvailability(opts?: NftOptions): Promise
|
|
|
41
45
|
* Idempotent -- safe to call multiple times.
|
|
42
46
|
*/
|
|
43
47
|
export declare function ensureKiciTable(opts?: NftOptions): Promise<void>;
|
|
48
|
+
/** A single nft rule operation: the verb decides chain placement. */
|
|
49
|
+
export interface NftRuleOp {
|
|
50
|
+
/** 'insert' prepends to the chain head; 'add' appends to the tail. */
|
|
51
|
+
verb: 'insert' | 'add';
|
|
52
|
+
/** Tokens after `nft <verb> rule ip kici forward`. */
|
|
53
|
+
tokens: string[];
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Build the per-identifier isolation rule operations.
|
|
57
|
+
*
|
|
58
|
+
* Placement matters because nftables is first-match-wins and the `kici`
|
|
59
|
+
* forward chain is shared: it accumulates rules for every live agent plus
|
|
60
|
+
* any host-level baseline (e.g. wildcard `iifname "kici-*"` RFC1918 drops
|
|
61
|
+
* installed at host bootstrap). Accepts therefore go in via `insert`
|
|
62
|
+
* (chain head) so they beat any pre-existing drop that also matches the
|
|
63
|
+
* traffic; an appended accept after a wildcard drop is dead code. Drops go
|
|
64
|
+
* in via `add` (tail) — they only need to beat the chain's accept policy.
|
|
65
|
+
*
|
|
66
|
+
* Effective per-identifier order: gateway + allowlist accepts (head),
|
|
67
|
+
* RFC1918 / metadata / denyAll drops (tail).
|
|
68
|
+
*/
|
|
69
|
+
export declare function buildIsolationRuleOps(matchClause: string[], gatewayIp: string, networkPolicy?: NetworkPolicy): NftRuleOp[];
|
|
44
70
|
/**
|
|
45
71
|
* Add network isolation rules for a specific network interface.
|
|
46
72
|
* Used by Firecracker and container backends.
|
|
47
73
|
*
|
|
48
|
-
* Rule
|
|
49
|
-
* 1. Accept gateway traffic (
|
|
50
|
-
* 2. Accept allowlisted CIDRs (
|
|
74
|
+
* Rule placement (see buildIsolationRuleOps):
|
|
75
|
+
* 1. Accept gateway traffic (inserted at chain head)
|
|
76
|
+
* 2. Accept allowlisted CIDRs (inserted at chain head)
|
|
51
77
|
* 3. Drop RFC1918 ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16)
|
|
52
78
|
* 4. Drop cloud metadata (169.254.0.0/16)
|
|
53
79
|
* 5. Drop all remaining traffic (only if denyAll is true)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { ScalerConfig } from './index.js';
|
|
2
|
+
/** Per-scaler orphan counts, keyed by scaler name. */
|
|
3
|
+
export type ReapCounts = Record<string, number>;
|
|
4
|
+
/**
|
|
5
|
+
* Reap orphan Firecracker resources (dead-VM chroots + TAPs) for every
|
|
6
|
+
* `firecracker` scaler in the config. Safe to run alongside a live
|
|
7
|
+
* orchestrator: the reap is liveness-driven and only touches dead VMs.
|
|
8
|
+
*/
|
|
9
|
+
export declare function reapFirecrackerOrphans(scalerConfig: ScalerConfig): Promise<ReapCounts>;
|
|
10
|
+
/**
|
|
11
|
+
* Reap orphan container resources for every `container` scaler. Unlike the FC
|
|
12
|
+
* reap, container cleanup is unconditional (it removes every kici-managed
|
|
13
|
+
* container), so callers MUST ensure the orchestrator is NOT running before
|
|
14
|
+
* invoking this — otherwise live agents are killed. The CLI command enforces
|
|
15
|
+
* this via the /health gate.
|
|
16
|
+
*/
|
|
17
|
+
export declare function reapContainerOrphans(scalerConfig: ScalerConfig): Promise<ReapCounts>;
|
|
18
|
+
/**
|
|
19
|
+
* Reap orphans for all supported backends. Firecracker is always reaped
|
|
20
|
+
* (liveness-driven, safe). Containers are reaped only when includeContainers
|
|
21
|
+
* is set (caller has confirmed the orchestrator is down).
|
|
22
|
+
*/
|
|
23
|
+
export declare function reapAllOrphans(opts: {
|
|
24
|
+
scalerConfig: ScalerConfig;
|
|
25
|
+
includeContainers: boolean;
|
|
26
|
+
}): Promise<ReapCounts>;
|
|
27
|
+
//# sourceMappingURL=reap-orphans.d.ts.map
|