@kici-dev/orchestrator 0.1.3 → 0.1.5

package/dist/routes/admin-events.d.ts

@@ -13,6 +13,9 @@ import type { GenericSourceManager } from '../webhook/generic-sources.js';
 import type { TrustStore } from '../events/trust-store.js';
 import type { TokenManager } from '../secrets/token-manager.js';
 import type { RbacEnforcer, Role } from '../secrets/rbac.js';
+import type { AppConfig } from '../config.js';
+import type { ProviderRegistry } from '../provider-registry.js';
+import type { SecretResolver } from '../secrets/secret-resolver.js';
 /**
  * Dependencies for admin event routes.
  */
@@ -21,6 +24,21 @@ interface AdminEventRouteDeps {
     trustStore: TrustStore;
     tokenManager: TokenManager;
     rbac: RbacEnforcer;
+    /**
+     * The in-process bundle registry. The POST /generic-sources handler
+     * registers an internal / universal-git bundle into this registry
+     * immediately after the source row lands in the DB, so the next
+     * webhook against that source resolves the right normalizer without
+     * waiting for an orchestrator restart.
+     */
+    providerRegistry: ProviderRegistry;
+    /** Needed by `registerProviderBundleForSource` to gate internal-bundle
+     *  registration on `canServeGenericProviderType` and read the
+     *  `internalProviderRepoPath` / `internalProviderCloneUrl` config. */
+    config: AppConfig;
+    /** Required for universal-git source registration — `null` is allowed;
+     *  rows with `git_config` are skipped + metric-bumped in that case. */
+    secretResolver: SecretResolver | null;
     /**
      * Optional — when provided, the `POST /api/v1/admin/events/emit` route is
      * mounted so operators can INSERT into `kici_events` + `pg_notify` via HTTP.
@@ -33,6 +51,7 @@ type AdminEventEnv = {
     Variables: {
         role: Role;
         userId: string;
+        routingKey: string | null;
     };
 };
 /**

package/dist/routes/admin-maintenance.d.ts

@@ -14,9 +14,17 @@
  */
 import { Hono } from 'hono';
 import type { Kysely } from 'kysely';
+import type { Role } from '../secrets/rbac.js';
 interface MaintenanceRouteDeps {
     db: Kysely<any>;
 }
-export declare function createMaintenanceRoutes(deps: MaintenanceRouteDeps): Hono;
+type AdminMaintenanceEnv = {
+    Variables: {
+        role: Role;
+        userId: string;
+        routingKey: string | null;
+    };
+};
+export declare function createMaintenanceRoutes(deps: MaintenanceRouteDeps): Hono<AdminMaintenanceEnv>;
 export {};
 //# sourceMappingURL=admin-maintenance.d.ts.map

package/dist/routes/admin-org-settings.d.ts

@@ -11,11 +11,20 @@
  */
 import { Hono } from 'hono';
 import { type Kysely } from 'kysely';
+import { DashboardWriteOperation } from '@kici-dev/engine/protocol/dashboard-write-operations';
 import type { Database } from '../db/types.js';
 import type { RbacEnforcer, Role } from '../secrets/rbac.js';
+import type { AccessLogWriter } from '../audit/access-log.js';
 interface OrgSettingsRouteDeps {
     db: Kysely<Database>;
     rbac: RbacEnforcer;
+    /**
+     * Optional — when wired, each `dashboard_write_policy` flip emits one
+     * `access_log` row (`org_settings.dashboard_write_policy.update`)
+     * carrying the operation name + prior/next state in `actor_meta`.
+     * Reset calls additionally stamp `reset: true`.
+     */
+    accessLog?: AccessLogWriter;
 }
 type AdminEnv = {
     Variables: {
@@ -25,5 +34,5 @@ type AdminEnv = {
     };
 };
 export declare function createOrgSettingsRoutes(deps: OrgSettingsRouteDeps): Hono<AdminEnv>;
-export {};
+export { DashboardWriteOperation };
 //# sourceMappingURL=admin-org-settings.d.ts.map

package/dist/routes/admin-queue-execution.d.ts

@@ -20,6 +20,7 @@ type AdminQEEnv = {
     Variables: {
         role: Role;
         userId: string;
+        routingKey: string | null;
     };
 };
 export declare function createAdminQueueExecutionRoutes(deps: AdminQueueExecutionRoutesDeps): Hono<AdminQEEnv>;

package/dist/routes/admin-registrations.d.ts

@@ -24,6 +24,7 @@ type AdminRegEnv = {
     Variables: {
         role: Role;
         userId: string;
+        routingKey: string | null;
     };
 };
 /**

package/dist/routes/admin-runs.d.ts

@@ -44,6 +44,7 @@ type AdminRunEnv = {
     Variables: {
         role: Role;
         userId: string;
+        routingKey: string | null;
     };
 };
 /**

package/dist/routes/admin-scheduled-jobs.d.ts

@@ -29,6 +29,7 @@ type AdminEnv = {
     Variables: {
         role: Role;
         userId: string;
+        routingKey: string | null;
     };
 };
 export declare function createAdminScheduledJobsRoutes(deps: AdminScheduledJobsRoutesDeps): Hono<AdminEnv>;

package/dist/routes/admin-sources.d.ts

@@ -10,9 +10,17 @@
  */
 import { Hono } from 'hono';
 import type { SourceStore } from '../sources/source-store.js';
+import type { Role } from '../secrets/rbac.js';
 interface SourceRouteDeps {
     sourceStore: SourceStore;
 }
-export declare function createSourceRoutes(deps: SourceRouteDeps): Hono;
+type AdminSourcesEnv = {
+    Variables: {
+        role: Role;
+        userId: string;
+        routingKey: string | null;
+    };
+};
+export declare function createSourceRoutes(deps: SourceRouteDeps): Hono<AdminSourcesEnv>;
 export {};
 //# sourceMappingURL=admin-sources.d.ts.map

package/dist/routes/admin.d.ts

@@ -23,6 +23,7 @@ import type { BackendHealthChecker } from '../secrets/backend-health.js';
 import type { BackendSyncManager } from '@kici-dev/engine';
 import type { Kysely } from 'kysely';
 import type pg from 'pg';
+import type { AccessLogWriter } from '../audit/access-log.js';
 /**
  * Dependencies for admin API routes.
  */
@@ -83,6 +84,13 @@ export interface AdminRouteDeps {
      * response reports `reEncryptedConfigs: 0`.
      */
     sharedStore?: SharedConfigStore;
+    /**
+     * Optional -- attribution writer for routes that emit an `access_log`
+     * row directly (today: org-settings dashboard-write policy flips). When
+     * unset, those routes execute the mutation without recording — the
+     * write is best-effort, never gating.
+     */
+    accessLog?: AccessLogWriter;
 }
 /** Hono env type for admin routes with context variables. */
 type AdminEnv = {

package/dist/scaler/manager.d.ts

@@ -8,6 +8,7 @@
  */
 import type { ResourceRequest } from '@kici-dev/engine';
 import type { ScalerBackend, ScalerConfig, ScaleResult, ScalerEvent, ResourceCap, ValidationResult } from './types.js';
+import type { ScalerStateStore, ScalerStateRecovery } from './scaler-state-store.js';
 /**
  * Resolved per-job resource amounts (cpus + bytes) for both `requests` and
  * `limits`. The scaler manager produces this from the job's declared resources
@@ -125,6 +126,14 @@ export declare class ScalerManager {
      */
     private readonly onScalerEvent?;
     private readonly warmPool;
+    /**
+     * Optional DB-backed state store. When wired (production path), every
+     * mutation to `spawningAgents` / `agentJobCorrelation` / `reservations`
+     * is write-through-cached to Postgres so a coord crash mid-spawn no
+     * longer orphans agents, strands reservations, or loses correlation.
+     * Unit tests can omit the store and operate from in-memory Maps only.
+     */
+    private readonly stateStore?;
     constructor(deps: {
         config: ScalerConfig;
         backends: Array<{
@@ -133,6 +142,11 @@ export declare class ScalerManager {
         }>;
         /** Callback for relaying scaler events with runId/jobId context. */
         onScalerEvent?: (runId: string, jobId: string, event: ScalerEvent) => void;
+        /**
+         * Optional DB-backed state store. Tests omit it; production wires it
+         * up via the orchestrator-core bootstrap.
+         */
+        stateStore?: ScalerStateStore;
         /**
          * Optional machine-ledger options. When `machinePools` are configured,
          * the manager initializes a `MachineLedger` keyed off this directory and
@@ -316,6 +330,30 @@ export declare class ScalerManager {
      * entries would leak in spawningAgents forever.
      */
     private pruneStaleSpawningEntries;
+    private persistSpawningAgent;
+    private deleteSpawningAgentFromStore;
+    private persistReservation;
+    private deleteReservationFromStore;
+    private persistAgentJob;
+    private deleteAgentJobFromStore;
+    /**
+     * Hydrate the in-memory Maps from the DB-backed state store after a
+     * coord boot or Raft leader switch. Reconstructs:
+     *
+     *   - `spawningAgents` (with `boundJobId` preserved for eager-dispatch on register)
+     *   - `agentJobCorrelation` (so scaler-lifecycle events route correctly)
+     *   - `reservations` + `perScalerUsage` (so the cap-check critical
+     *     section reflects the cluster-wide truth, not the local empty
+     *     starting state)
+     *
+     * The `globalUsage` counter is recomputed from `perScalerUsage` to
+     * keep the cap math consistent. `eventBuffer` is NOT restored — events
+     * emitted by the previous coord before correlation are lost (see
+     * wishlist for the rationale).
+     *
+     * No-op when no store is wired (unit-test path).
+     */
+    recoverState(): Promise<ScalerStateRecovery>;
     private generateAgentId;
     /**
      * Start log forwarding for a scaler-managed agent if its backend supports LogCapture.

package/dist/scaler/scaler-state-store.d.ts

@@ -0,0 +1,102 @@
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+import type { ScalerEvent } from './types.js';
+/**
+ * Snapshot of a spawning-agent record. Mirrors the row shape in
+ * `scaler_spawning_agents`.
+ */
+export interface SpawningAgentSnapshot {
+    agentId: string;
+    scalerName: string;
+    labelSet: string[];
+    runId?: string;
+    jobId?: string;
+    boundJobId?: string;
+    spawnedAt: Date;
+}
+/**
+ * Snapshot of an agent-job correlation. Mirrors the row shape in
+ * `scaler_agent_jobs`.
+ */
+export interface AgentJobCorrelationSnapshot {
+    agentId: string;
+    runId: string;
+    jobId: string;
+}
+/**
+ * Snapshot of a resource reservation. Mirrors the row shape in
+ * `scaler_reservations`.
+ */
+export interface ReservationSnapshot {
+    agentId: string;
+    scalerName: string;
+    cpus: number;
+    memBytes: number;
+}
+/**
+ * DB persistence for `ScalerManager` HA-critical state.
+ *
+ * Backed by three tables — `scaler_spawning_agents`, `scaler_agent_jobs`,
+ * `scaler_reservations` — so a Raft leader switch / coord crash no
+ * longer:
+ *
+ *  - orphans an agent that is mid-spawn (lost `boundJobId` → eager
+ *    dispatch silently downgraded to a generic queue drain),
+ *  - strands a reservation (resource counted as used until the agent's
+ *    backend GC eventually disconnects, minutes later),
+ *  - drops the agent → run/job correlation (execution-tracker loses
+ *    scaler-lifecycle events emitted by the new coord).
+ *
+ * The consumer keeps the in-memory Maps as L1 caches. On boot /
+ * become-leader the caches are hydrated via `recoverState()`.
+ *
+ * `perScalerUsage` / `globalUsage` are NOT stored — they are derived
+ * state recomputed from `SUM(...) FROM scaler_reservations` on
+ * recovery, which means the on-disk reservation rows are the single
+ * source of truth for the cap-check critical section.
+ *
+ * The `eventBuffer` Map is also not persisted: events emitted before
+ * correlation are observability, not correctness. A coord crash before
+ * `correlateAgentToJob()` runs accepts losing those events (see the
+ * wishlist for the rationale).
+ */
+export declare class ScalerStateStore {
+    private readonly db;
+    constructor(db: Kysely<Database>);
+    upsertSpawningAgent(snapshot: SpawningAgentSnapshot): Promise<void>;
+    deleteSpawningAgent(agentId: string): Promise<void>;
+    listSpawningAgents(): Promise<SpawningAgentSnapshot[]>;
+    /**
+     * Delete every spawning-agent row whose `spawned_at` is older than the
+     * given cutoff. Used by the leader-gated GC sweep so a coord that
+     * crashed mid-spawn doesn't leave the row blocking the spawn-timeout
+     * detection forever. Returns the row count GC'd.
+     */
+    sweepStaleSpawningAgents(olderThan: Date): Promise<number>;
+    upsertAgentJob(snapshot: AgentJobCorrelationSnapshot): Promise<void>;
+    deleteAgentJob(agentId: string): Promise<void>;
+    listAgentJobs(): Promise<AgentJobCorrelationSnapshot[]>;
+    upsertReservation(snapshot: ReservationSnapshot): Promise<void>;
+    deleteReservation(agentId: string): Promise<void>;
+    listReservations(): Promise<ReservationSnapshot[]>;
+}
+/**
+ * Aggregate event surface for "the scaler manager fully replayed its
+ * state from the DB after a leader switch". Kept here (vs in
+ * manager.ts) so `ScalerManager.recoverState()` can declare a clean
+ * return type. `bufferedEventsLost` always returns 0 today — the
+ * `eventBuffer` Map is intentionally not persisted — but the field
+ * exists so a future buffer-table addition is type-compatible.
+ */
+export interface ScalerStateRecovery {
+    spawningAgentsRehydrated: number;
+    agentJobsRehydrated: number;
+    reservationsRehydrated: number;
+    bufferedEventsLost: number;
+}
+/**
+ * Re-export for the buffered-events note above; sole reason
+ * `ScalerEvent` is imported is to keep that comment compile-checked.
+ */
+export type { ScalerEvent };
+//# sourceMappingURL=scaler-state-store.d.ts.map

package/dist/secrets/routing-key-scope.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Helpers for enforcing the `routing_key` scope on admin tokens.
+ *
+ * `admin_tokens.routing_key` is an optional column. A token created with
+ * `kici-admin token create --routing-key <key>` is restricted to operations
+ * that target that exact routing key:
+ *
+ *   - Routes that take an explicit routing key (URL param, query param, or
+ *     request body) MUST gate access with `enforceRoutingKeyScope`. Routes
+ *     that target an entity by ID look up the row's `routing_key` column
+ *     first and pass that to `enforceRoutingKeyScope`.
+ *   - Routes that target an org or that operate orchestrator-wide (no
+ *     routing-key concept at all) MUST refuse routing-key tokens via
+ *     `requireUnscopedToken`.
+ *
+ * Both helpers return a Hono `Response` to short-circuit the handler when
+ * access is denied, or `null` to let the handler continue. The middleware
+ * mounted in each admin route file is responsible for calling
+ * `c.set('routingKey', tokenInfo.routingKey)` so these helpers can read it.
+ */
+import type { Context } from 'hono';
+/**
+ * Reject the request with 403 when the calling token has a routing-key
+ * scope and the request targets a different routing key.
+ *
+ * Pass `requested = null | undefined` for routes that explicitly do not
+ * carry a routing key — in that case the call is also rejected for
+ * scoped tokens. Use {@link requireUnscopedToken} when the route is
+ * orchestrator-wide or org-scoped (no per-routing-key variant exists at
+ * all) to make the intent explicit.
+ *
+ * Returns a `Response` to short-circuit the handler when access is
+ * denied, or `null` to let the handler continue.
+ */
+export declare function enforceRoutingKeyScope(c: Context, requested: string | null | undefined): Response | null;
+/**
+ * Reject the request with 403 when the calling token has any routing-key
+ * scope. Use this on routes that operate on orchestrator-wide state,
+ * org-level state, or any other surface where the routing-key concept
+ * does not apply.
+ */
+export declare function requireUnscopedToken(c: Context): Response | null;
+//# sourceMappingURL=routing-key-scope.d.ts.map