@kici-dev/orchestrator 0.1.27 → 0.2.0

package/dist/diagnostics/checks/backup.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Backup-freshness diagnostic check.
+ *
+ * Reads the newest `backup_runs.created_at` and reports whether the
+ * orchestrator DB has a recent `kici-admin db backup`. FAIL when none was ever
+ * recorded, WARN when the newest is older than the effective staleness
+ * threshold, else PASS.
+ *
+ * The threshold is cluster-configurable: the strictest (minimum) non-null
+ * `org_settings.backup_staleness_warn_hours` across all orgs, falling back to
+ * `config.backupStalenessWarnHours` when no org sets an override. A stale
+ * whole-DB backup affects every org, so the tightest org requirement wins.
+ */
+import type { DiagnosticDeps, DiagnosticResult } from '../types.js';
+export declare const DEFAULT_BACKUP_STALENESS_WARN_HOURS = 24;
+export declare function resolveStalenessThresholdHours(deps: DiagnosticDeps): Promise<number>;
+export declare function checkBackupFreshness(deps: DiagnosticDeps): Promise<DiagnosticResult>;
+//# sourceMappingURL=backup.d.ts.map

package/dist/diagnostics/checks/index.d.ts

@@ -6,6 +6,7 @@
  */
 import type { DiagnosticCheck } from '../types.js';
 import { checkDbConnectivity } from './db.js';
+import { checkBackupFreshness } from './backup.js';
 import { checkWsToPlatform } from './ws.js';
 import { checkAgentConnectivity } from './agents.js';
 import { checkDiskSpace } from './disk.js';
@@ -15,5 +16,5 @@ import { checkScalerProvisioning } from './scaler.js';
 import { checkFirecrackerNetwork } from './firecracker-network.js';
 /** All diagnostic checks in display order. */
 export declare const defaultChecks: DiagnosticCheck[];
-export { checkDbConnectivity, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, checkScalerProvisioning, checkFirecrackerNetwork, };
+export { checkDbConnectivity, checkBackupFreshness, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, checkScalerProvisioning, checkFirecrackerNetwork, };
 //# sourceMappingURL=index.d.ts.map

package/dist/diagnostics/fleet-collector.d.ts

@@ -12,9 +12,9 @@ import { z } from 'zod';
 /** Per-node collection outcome recorded in the fleet manifest. */
 export declare const FleetNodeStatus: z.ZodEnum<{
     error: "error";
-    unreachable: "unreachable";
     ok: "ok";
     timeout: "timeout";
+    unreachable: "unreachable";
 }>;
 export type FleetNodeStatus = z.infer<typeof FleetNodeStatus>;
 export interface FleetCollectorDeps {

package/dist/drain/drain-controller.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Owns the coordinator's ephemeral "draining" state for pre-upgrade quiescing.
+ *
+ * While draining, the Dispatcher routes every new job to the Pending queue
+ * (Model A / job-level drain), the peer heartbeat advertises `draining: true`
+ * (siblings stop rerouting here), and the scaler stops spawning fresh agents.
+ * Jobs already executing on agents finish; `jobsRunning()` drops monotonically
+ * to zero, at which point the coordinator is quiesced and safe to restart.
+ *
+ * State is in-memory only: an upgrade restart comes back accepting and the
+ * existing startup recovery re-dispatches the Pending backlog.
+ */
+export interface DrainControllerDeps {
+    /** Sum of activeJobs across all connected agents (agentRegistry). */
+    activeJobsTotal: () => number;
+    /**
+     * Count of dispatch_queue rows in Dispatched status. In the single-coordinator
+     * drain-before-upgrade scenario this equals this instance's in-flight jobs; a
+     * per-instance ownership column does not exist on dispatch_queue, so a
+     * multi-coordinator cluster may over-count here (belt-and-suspenders only —
+     * `activeJobsTotal` is the primary signal, and both shrink monotonically once
+     * the dispatcher gate stops new claims).
+     */
+    dispatchedJobsOwned: () => Promise<number>;
+    /**
+     * Optional hook fired whenever the draining flag changes, used to drive the
+     * `kici_orchestrator_draining` metric. Not fired on a no-op (idempotent) call.
+     */
+    onChange?: (draining: boolean) => void;
+}
+export declare class DrainController {
+    private readonly deps;
+    private draining;
+    constructor(deps: DrainControllerDeps);
+    isDraining(): boolean;
+    startDrain(): void;
+    stopDrain(): void;
+    /**
+     * Live quiesce signal: jobs currently executing on agents versus any row this
+     * instance already claimed (Dispatched) but whose activeJobs increment has not
+     * yet landed. Since draining stops new claims, this set only shrinks, so a
+     * poll on it is guaranteed to converge to zero. Uses `Math.max` (not sum) — the
+     * Dispatched rows are the durable view of the same jobs the in-memory
+     * activeJobs counts, so summing would double-count the common case.
+     */
+    jobsRunning(): Promise<number>;
+    snapshot(): Promise<{
+        draining: boolean;
+        jobsRunning: number;
+    }>;
+}
+//# sourceMappingURL=drain-controller.d.ts.map

package/dist/events/batch-accumulator.d.ts

@@ -0,0 +1,63 @@
+import { type Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+/**
+ * Durable accumulator backing the `workflowsFailedBatch` trigger.
+ *
+ * A failed `workflow_complete` event that matches a `workflowsFailedBatch`
+ * registration is buffered into a per-(registration, window) row instead of
+ * dispatching now. The first failure opens the window; the Raft-leader-only
+ * scanner sweeps windows past `expires_at`, emits one `__workflows_failed_batch`
+ * event, and deletes the window (cascading its items). The state lives in
+ * Postgres (not an in-memory Map) so a node restart mid-window loses nothing.
+ */
+/** A failed run buffered into a batch window. */
+export interface BatchAccumulatorRun {
+    runId: string;
+    repoIdentifier: string;
+    workflowName: string;
+    failureClass?: string | null;
+    senderUsername?: string | null;
+}
+/** A window swept past its deadline, with the runs it accumulated. */
+export interface SweptBatchWindow {
+    windowId: string;
+    customerId: string;
+    registrationId: string;
+    routingKey: string;
+    repoIdentifier: string;
+    runs: BatchAccumulatorRun[];
+}
+export interface OpenBatchWindowInput {
+    registrationId: string;
+    customerId: string;
+    routingKey: string;
+    repoIdentifier: string;
+    accumulateForMs: number;
+}
+/**
+ * Open the accumulation window for a registration, or return the existing open
+ * one. `opened` is true only for the caller that created the window (the first
+ * failure of the burst) so exactly one caller schedules nothing beyond the DB
+ * row — the leader scanner owns the flush.
+ *
+ * Uses `INSERT ... ON CONFLICT (registration_id) DO NOTHING RETURNING id` for an
+ * atomic open-once: concurrent failures append to the same window rather than
+ * racing to open N windows.
+ */
+export declare function openOrGetBatchWindow(db: Kysely<Database>, input: OpenBatchWindowInput): Promise<{
+    windowId: string;
+    opened: boolean;
+}>;
+/** Append one failed run to an open window. */
+export declare function appendBatchItem(db: Kysely<Database>, input: {
+    windowId: string;
+    run: BatchAccumulatorRun;
+}): Promise<void>;
+/**
+ * Sweep every window whose `expires_at <= now`. For each, in a transaction, load
+ * its items, delete the window (cascading the items), and return the runs. Only
+ * the Raft leader calls this. Windows with no items (a run that was excluded
+ * after opening) still emit — the caller decides whether to skip an empty batch.
+ */
+export declare function sweepExpiredBatchWindows(db: Kysely<Database>, now: Date): Promise<SweptBatchWindow[]>;
+//# sourceMappingURL=batch-accumulator.d.ts.map

package/dist/events/circuit-breaker.d.ts

@@ -1,4 +1,5 @@
 import type { EventRouterConfig } from './types.js';
+import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
 /**
  * Circuit breaker for event loop detection and user-event rate limiting.
  *
@@ -9,8 +10,9 @@ import type { EventRouterConfig } from './types.js';
  */
 export declare class EventCircuitBreaker {
     private readonly config;
+    private readonly clusterSettings?;
     private rateLimiter;
-    constructor(config: EventRouterConfig);
+    constructor(config: EventRouterConfig, clusterSettings?: ClusterSettingsReader | undefined);
     /**
      * Check if the current chain depth is within limits.
      */
@@ -23,11 +25,15 @@ export declare class EventCircuitBreaker {
      * sliding 60-second window. Callers key user events by
      * `<sourceRoutingKey>:<eventName>`; system events (`__`-prefixed) are exempt
      * upstream and never reach here.
+     *
+     * The per-(source routing key + event) limit is the fleet-wide cluster
+     * tunable `event_router_rate_limit_per_workflow_per_minute` (live override
+     * wins), falling back to the configured default.
      */
-    checkRateLimit(rateKey: string): {
+    checkRateLimit(rateKey: string): Promise<{
         allowed: boolean;
         retryAfterMs?: number;
-    };
+    }>;
     /**
      * Clear all rate limit state (for testing).
      */

package/dist/events/event-emitter.d.ts

@@ -14,6 +14,29 @@ export interface WorkflowCompleteData {
         name: string;
         status: string;
     }>;
+    /** Why the run failed (`RunFailureClass`); carried on the event so Plane B can route on it. */
+    failureClass?: string;
+}
+/** One failed run carried in a workflows_failed_batch event. */
+export interface WorkflowsFailedBatchRun {
+    runId: string;
+    repo: string;
+    workflowName: string;
+    failureClass?: string;
+    senderUsername?: string;
+}
+/**
+ * Input data for emitting a __workflows_failed_batch system event.
+ * One event per swept accumulation window; `registrationId` targets the single
+ * subscribing workflow so the batch dispatches once.
+ */
+export interface WorkflowsFailedBatchData {
+    routingKey: string;
+    repo: string;
+    registrationId: string;
+    /** Total failed runs in the window (may exceed `runs.length` when capped). */
+    total: number;
+    runs: WorkflowsFailedBatchRun[];
 }
 /**
  * Input data for emitting a job_complete system event.
@@ -46,6 +69,14 @@ export declare class EventEmitter {
      * Returns the event ID as delivery receipt.
      */
     emitWorkflowComplete(data: WorkflowCompleteData): Promise<string>;
+    /**
+     * Emit a __workflows_failed_batch system event: one synthetic event per swept
+     * accumulation window carrying the batched failed runs. Targeted at the single
+     * subscribing registration via `payload.registrationId` so the batch workflow
+     * dispatches exactly once, no matter how many failures landed in the window.
+     * Returns the event ID as delivery receipt.
+     */
+    emitWorkflowsFailedBatch(data: WorkflowsFailedBatchData): Promise<string>;
     /**
      * Emit a __job_complete system event with rich metadata.
      * Returns the event ID as delivery receipt.

package/dist/events/event-retry-scanner.d.ts

@@ -1,10 +1,12 @@
 import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 import type { EventStore } from './event-store.js';
+import type { EventEmitter } from './event-emitter.js';
 import type { EventRouterConfig } from './types.js';
 interface EventRetryScannerOptions {
     db: Kysely<Database>;
     eventStore: EventStore;
+    eventEmitter: EventEmitter;
     config: EventRouterConfig;
 }
 /**
@@ -32,9 +34,9 @@ interface EventRetryScannerOptions {
 export declare class EventRetryScanner {
     private readonly db;
     private readonly eventStore;
+    private readonly eventEmitter;
     private readonly config;
-    private timer;
-    private isLeader;
+    private readonly scheduler;
     constructor(options: EventRetryScannerOptions);
     onBecomeLeader(): void;
     onLoseLeadership(): void;
@@ -43,6 +45,14 @@ export declare class EventRetryScanner {
      * One scanner tick. Public for tests; not on the hot path otherwise.
      */
     tick(): Promise<void>;
+    /**
+     * Sweep expired workflows_failed_batch accumulation windows and emit one
+     * `__workflows_failed_batch` event per window that accumulated at least one
+     * run. The emitted event flows back through the router and dispatches the
+     * subscribing workflow exactly once. The run list is capped at
+     * `BATCH_MAX_RUNS`; `total` carries the true count.
+     */
+    private processBatchWindows;
     private processExpiredLeases;
     private processDueRetries;
     private refreshDlqDepth;

package/dist/events/event-router.d.ts

@@ -7,6 +7,7 @@ import type { EventCircuitBreaker } from './circuit-breaker.js';
 import type { TrustStore } from './trust-store.js';
 import type { RegistrationIndex } from '../registration/registration-index.js';
 import type { EventRouterConfig, StoredEvent } from './types.js';
+import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
 /**
  * Input for emitting an internal event.
  */
@@ -41,6 +42,12 @@ export interface EventRouterOptions {
     circuitBreaker: EventCircuitBreaker;
     trustStore: TrustStore;
     config: EventRouterConfig;
+    /**
+     * Fleet-wide settings reader. When present, the per-event insert TTL and the
+     * max-dispatch-attempts cap are read live from cluster_settings, falling back
+     * to `config.eventTtlSeconds` / `config.maxDispatchAttempts`.
+     */
+    clusterSettings?: ClusterSettingsReader;
     onEventMatched: (event: StoredEvent, lockFile: LockFile, matchedWorkflows: WorkflowDecision[], context?: EventMatchContext) => Promise<void>;
     /** Registration index for DB-backed subscription matching. Events match against persistent registrations. */
     registrationIndex: RegistrationIndex;
@@ -66,6 +73,7 @@ export declare class EventRouter {
     private readonly circuitBreaker;
     private readonly trustStore;
     private readonly config;
+    private readonly clusterSettings?;
     private readonly onEventMatched;
     private readonly registrationIndex;
     private readonly nodeId;
@@ -139,6 +147,27 @@ export declare class EventRouter {
      * lease pattern fixes.
      */
     private processSubscriptions;
+    /**
+     * Partition matched decisions into "dispatch now" and "buffer into a batch
+     * window". A decision whose matched trigger is `workflows_failed_batch` and
+     * whose event is a failed `__workflow_complete` is buffered (and excluded from
+     * the returned dispatch set) so the whole burst notifies once per window.
+     *
+     * Self-exclusion: a failed `__workflow_complete` whose originating run was
+     * itself dispatched by a failure-lifecycle trigger is neither buffered nor
+     * dispatched — a broken notifier must not re-trigger the batch on its own
+     * failure. The chain-depth breaker remains the backstop for anything missed.
+     */
+    private bufferBatchDecisions;
+    /** Open (or reuse) the registration's batch window and append the failed run. */
+    private bufferBatchFailure;
+    /**
+     * True when the given run was dispatched by a failure-lifecycle trigger
+     * (`workflows_failed_batch` or a `workflow_complete` failed-status trigger),
+     * recorded in `execution_runs.trigger_decision.dispatchedByFailureLifecycle`
+     * at dispatch time. Used for batch self-exclusion.
+     */
+    private isFailureLifecycleRun;
     /**
      * Map a stored event to a trigger type string for RegistrationIndex lookup.
      *

package/dist/firecracker/host-network.d.ts

@@ -12,6 +12,8 @@
  * nft table name but have separate lifecycles; this module never touches the
  * per-VM rules.
  */
+import { FIRECRACKER_NET_INTERFACES } from './net-interfaces.js';
+export { FIRECRACKER_NET_INTERFACES };
 /** A single subprocess invocation (no shell). */
 export interface CommandSpec {
     bin: string;

package/dist/firecracker/net-interfaces.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Interface patterns the Firecracker host setup marks unmanaged in
+ * NetworkManager.
+ *
+ * A leaf module on purpose: it imports nothing, so an out-of-package consumer
+ * can read the constant without pulling the orchestrator runtime graph in with
+ * it. The NetworkManager drift watchdog (`hack/lib/nm-watchdog.ts`) derives its
+ * expected-pattern set from here rather than re-typing the patterns — a second
+ * copy desyncs silently the moment a pattern is added. `host-network.ts`
+ * re-exports it and renders `NM_CONF_CONTENT` from it, so the drop-in and the
+ * watchdog can never disagree.
+ */
+export declare const FIRECRACKER_NET_INTERFACES: readonly ['kici-*'];
+//# sourceMappingURL=net-interfaces.d.ts.map

package/dist/github-app-name-refresher/github-app-name-refresher.d.ts

@@ -15,14 +15,16 @@
  * then a `setInterval`; `stop()` clears it. Per-source errors are logged and
  * never abort the loop.
  */
+/** A source row as returned by listSources() — the slice this module needs. */
+export type SourceIdentityRow = {
+    routing_key: string;
+    provider: string;
+    name: string;
+    slug: string | null;
+};
 /** The slice of {@link SourceStore} this module needs. */
 export interface RefreshableSourceStore {
-    listSources(): Promise<Array<{
-        routing_key: string;
-        provider: string;
-        name: string;
-        slug: string | null;
-    }>>;
+    listSources(): Promise<SourceIdentityRow[]>;
     getSourceWithSecrets(routingKey: string): Promise<{
         provider: string;
         config: string | Record<string, unknown>;
@@ -50,10 +52,18 @@ export interface RefreshResult {
     oldSlug: string | null;
     newSlug: string;
 }
+/**
+ * Refresh one already-resolved GitHub source's identity from GitHub and persist
+ * it when the name or slug drifted. The caller supplies the row it already holds,
+ * so this never lists the sources table. Throws for a non-GitHub row or missing
+ * credentials.
+ */
+export declare function refreshResolvedGithubSource(sourceStore: Pick<RefreshableSourceStore, 'getSourceWithSecrets' | 'updateSource'>, row: SourceIdentityRow, fetchIdentity: FetchGithubAppIdentity): Promise<RefreshResult>;
 /**
  * Re-fetch one GitHub source's identity from GitHub and persist it when the
- * name or slug drifted. Shared by the daily task and `kici-admin source
- * refresh`. Throws for a missing or non-GitHub routing key.
+ * name or slug drifted. Resolves the row by routing key (one sources read) then
+ * delegates. Shared by `kici-admin source refresh`. Throws for a missing or
+ * non-GitHub routing key.
  */
 export declare function refreshGithubSourceIdentity(sourceStore: RefreshableSourceStore, routingKey: string, fetchIdentity: FetchGithubAppIdentity): Promise<RefreshResult>;
 export interface GithubAppNameRefresherDeps {

package/dist/index.d.ts

@@ -7,6 +7,8 @@ export { UserCache, DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, t
 export { PeerCredentialStore, createPeerCredentialStoreFromUrl, type PeerCredential, type CredentialFileData, } from './cluster/peer-credentials.js';
 export { PeerAuthCoordinator, type AuthDecision, type RejectionAction, } from './cluster/peer-auth-coordinator.js';
 export { JoinTokenManager, createJoinTokenManagerFromUrl } from './cluster/join-token.js';
+export { HeldRunStore, createHeldRunStoreFromUrl, SecurityHoldReason, } from './contexts/held-runs.js';
+export { TrustPolicyStore, createTrustPolicyStoreFromUrl, TrustPolicySource, DEFAULT_TRUST_POLICY, type StoredTrustPolicy, } from './security/trust-policy-store.js';
 export { AdminApiClient } from './cli/api-client.js';
 export { runGithubManifestSetup, type ManifestSetupOptions, type ManifestSetupDeps, } from './cli/commands/source-manifest.js';
 //# sourceMappingURL=index.d.ts.map