@kici-dev/orchestrator 0.1.15 → 0.1.17

package/dist/diagnostics/fleet-selection.d.ts

@@ -0,0 +1,15 @@
+import type { FleetSelection } from '@kici-dev/engine';
+import type { FleetTopology } from './fleet-topology.js';
+/** A per-orchestrator selection, keyed by orchestrator instanceId. Absent = prune. */
+export type ResolvedSelectionMap = Map<string, FleetSelection>;
+/**
+ * Resolve `--pick` selectors into a per-orchestrator FleetSelection map.
+ *
+ * No selectors -> every orchestrator gets `{ all: true }`. With selectors, an
+ * orchestrator is included only if at least one of its agents/workers (or the
+ * orchestrator itself) matches; its FleetSelection then carries exactly the
+ * matched agent ids and worker instanceIds. Orchestrators with no matches are
+ * omitted from the map (their branch is pruned).
+ */
+export declare function resolveSelection(topology: FleetTopology, selectors: string[]): ResolvedSelectionMap;
+//# sourceMappingURL=fleet-selection.d.ts.map

package/dist/diagnostics/fleet-topology.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Fleet topology enumeration.
+ *
+ * Builds the cluster tree for `debug-bundle --fleet --list` / `--pick` from the
+ * local agent registry plus the heartbeat-cached peer inventory — no fan-out.
+ * Each peer's cached agent list is already maintained by ~30s heartbeats, so
+ * enumeration is a cheap in-memory read.
+ */
+/** A node in the enumerated fleet topology. */
+export interface FleetTopologyNode {
+    kind: 'orchestrator' | 'agent';
+    id: string;
+    /** Cluster role for orchestrator nodes (coordinator | worker); undefined for agents. */
+    role?: 'coordinator' | 'worker';
+    hostname?: string;
+    labels: Record<string, string>;
+    /** Parent orchestrator instanceId; null for the collector (root) node. */
+    parentId: string | null;
+}
+export interface FleetTopology {
+    nodes: FleetTopologyNode[];
+}
+export interface FleetTopologyDeps {
+    /** This collector's own instanceId (the root orchestrator). */
+    instanceId: string;
+    /** This collector's role. */
+    role: 'coordinator' | 'worker';
+    /** This collector's hostname, if known. */
+    hostname?: string;
+    /** This node's directly-connected agents. */
+    listLocalAgents: () => {
+        agentId: string;
+        labels: string[];
+    }[];
+    /** The heartbeat-cached peer inventory (each peer + its cached agents). */
+    listPeers: () => {
+        instanceId: string;
+        role: 'coordinator' | 'worker';
+        hostname?: string;
+        agents: {
+            agentId: string;
+            labels: string[];
+        }[];
+    }[];
+}
+export declare function buildFleetTopology(deps: FleetTopologyDeps): FleetTopology;
+//# sourceMappingURL=fleet-topology.d.ts.map

package/dist/diagnostics/fleet-wiring.d.ts

@@ -0,0 +1,60 @@
+import type { PeerLogsCollectRequest, PeerToPeerMessage, FleetSelection } from '@kici-dev/engine';
+import type { AgentRegistry } from '../agent/registry.js';
+import type { PeerRegistry } from '../cluster/peer-registry.js';
+import type { PeerClient } from '../cluster/peer-client.js';
+import type { DiagnosticDeps } from './types.js';
+import { type FleetCollectorDeps } from './fleet-collector.js';
+import { type FleetTopology, type FleetTopologyDeps } from './fleet-topology.js';
+/** A peer-handler-like object exposing the dual-direction collect send. */
+export interface FleetPeerHandlerLike {
+    sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
+}
+/** A collector that issues a fleet.logs.request to an agent and awaits its bundle. */
+export interface FleetAgentRequester {
+    request: (requestId: string, agentId: string, send: () => void) => Promise<Buffer>;
+}
+export interface FleetRuntime {
+    instanceId: string;
+    role: 'coordinator' | 'worker';
+    /** Loop window for log files (hours). */
+    logWindowHours: number;
+    /** Per-node deadline (ms). */
+    timeoutMs: number;
+    /** Local agent log directory (KICI_LOG_DIR), if configured. */
+    logDir?: string;
+    agentRegistry: AgentRegistry;
+    peerRegistry: PeerRegistry;
+    fleetAgentCollector: FleetAgentRequester;
+    /** Outgoing peer clients, keyed by instanceId. */
+    peerClients: Map<string, PeerClient>;
+    /** Incoming peer-handler with the dual-direction collect send. */
+    peerHandler: FleetPeerHandlerLike;
+    /** Deps for the local createDebugBundle. */
+    diagnosticDeps: DiagnosticDeps;
+    /** Raw orchestrator config (redacted by createDebugBundle). */
+    config: Record<string, unknown>;
+    /** Cluster health endpoint for the local bundle's cluster/health.json. */
+    clusterHealthUrl?: string;
+}
+/**
+ * Build the FleetCollectorDeps for a given selection of this node's downstream
+ * agents/workers. The collector passes per-branch selection in when it knows
+ * which downstream subset each peer should gather.
+ */
+export declare function buildFleetCollectorDeps(runtime: FleetRuntime, perBranchSelection: (instanceId: string) => FleetSelection): FleetCollectorDeps;
+/** Build the FleetTopologyDeps for `--list` / `--pick` enumeration. */
+export declare function buildFleetTopologyDeps(runtime: FleetRuntime): FleetTopologyDeps;
+/** Enumerate this node's fleet topology (no fan-out). */
+export declare function getFleetTopology(runtime: FleetRuntime): FleetTopology;
+/**
+ * Collect this node's full subtree (root call: includeCoordinatorMesh=true) with
+ * an optional per-orchestrator selection map. Absent map entry => collect all.
+ */
+export declare function collectFleet(runtime: FleetRuntime, selectionByOrch: Map<string, FleetSelection> | null): Promise<Buffer>;
+/**
+ * Peer-side responder: on an inbound peer.logs.collect.request, assemble this
+ * node's subtree (with the request's loop guard + selection) and stream it back
+ * as peer.logs.collect.chunk frames, or a peer.logs.collect.error on failure.
+ */
+export declare function makeFleetCollectResponder(runtime: FleetRuntime): (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
+//# sourceMappingURL=fleet-wiring.d.ts.map

package/dist/entry-helpers.d.ts

@@ -5,14 +5,13 @@
  * Both entry points import these helpers instead of maintaining separate copies.
  */
 import { type SourceProvider, SourceSubtype } from '@kici-dev/engine';
-import { ProviderRegistry } from './provider-registry.js';
 /**
  * A provider source for Platform registration.
  *
  * `provider` is the coarse-grained Platform-routing family (`github` /
  * `gitlab` / `bitbucket` / `generic`). `subtype` is the fine-grained
  * source kind that survives all the way to the dashboard so universal-git,
- * generic_webhook, and internal sources stay distinguishable. `name` is
+ * generic_webhook, and local sources stay distinguishable. `name` is
  * the human-readable label shown in the runs view + sources tab.
  */
 export interface ProviderSource {
@@ -50,24 +49,6 @@ export declare function diffProviderSources(oldSources: ProviderSource[], newSou
     added: ProviderSource[];
     removed: ProviderSource[];
 };
-/**
- * Register the internal provider bundle for E2E testing when the
- * `internalProviderRepoPath` config field (KICI_INTERNAL_PROVIDER_REPO_PATH)
- * is set.
- *
- * When the field is present, this overwrites the previously-registered
- * `generic:default` bundle with one that has a lockFileFetcher and
- * repoUrlBuilder (needed for the webhook pipeline to match triggers and
- * dispatch jobs).
- *
- * When the field is not set, does nothing (leaves `generic:default` as-is).
- *
- * @returns true if the internal provider was registered, false otherwise
- */
-export declare function registerInternalProviderIfConfigured(registry: ProviderRegistry, config: {
-    internalProviderRepoPath?: string;
-    internalProviderCloneUrl?: string;
-}): boolean;
 /**
  * Decide whether the local orchestrator can serve a `generic_webhook_sources`
  * row with the given `provider_type`. Used at boot (and any future reload) to
@@ -79,15 +60,12 @@ export declare function registerInternalProviderIfConfigured(registry: ProviderR
  * - `'generic'` / `'universal-git'` → always servable (the bundles register
  *   without filesystem dependencies; per-row failures during universal-git
  *   bundle registration are already isolated in orchestrator-core.ts).
- * - `'internal'` → only servable when `internalProviderRepoPath` is set, the
- *   path exists on disk, and is a directory. This mirrors the fail-fast
- *   guard in `registerInternalProviderIfConfigured`. The check is binary
- *   per peer: a single `KICI_INTERNAL_PROVIDER_REPO_PATH` backs every
- *   internal source on that peer (per-source `repoIdentifier`s nest under
- *   the same base dir).
+ * - `'local'` → only servable when the ROW's own `git_config.repoBasePath`
+ *   exists as a directory on THIS peer. The check is per-source and per-peer:
+ *   a local repo present on one HA peer may be absent on another, so only the
+ *   peer that hosts the repo advertises the routing key. Mirrors the statSync
+ *   gate in `registerProviderBundleForSource`.
  * - Any unknown provider_type returns false to fail closed.
  */
-export declare function canServeGenericProviderType(providerType: string, config: {
-    internalProviderRepoPath?: string;
-}): boolean;
+export declare function canServeGenericProviderType(providerType: string, gitConfig?: string | Record<string, unknown> | null): boolean;
 //# sourceMappingURL=entry-helpers.d.ts.map

package/dist/environments/held-runs.d.ts

@@ -4,7 +4,8 @@
  * Manages the lifecycle: pending -> approved/rejected/expired.
  */
 import { type Kysely } from 'kysely';
-import type { Database, HeldRun } from '../db/types.js';
+import { type ApprovalRequirement, type ApproverClause, ApprovalDecision, HoldScope, TriggerSource } from '@kici-dev/engine';
+import type { Database, HeldRun, HeldRunApproval } from '../db/types.js';
 /** Status values for held runs (held_runs table). */
 export declare enum HeldRunStatus {
     Pending = "pending",
@@ -24,6 +25,53 @@ export interface CreateHeldRunData {
     /** Queue type: 'environment' (default) or 'security'. */
     queueType?: 'environment' | 'security';
 }
+/**
+ * Data required to create a generalized approval hold. Unlike the legacy
+ * environment-only `create()`, this carries the hold scope, trigger source,
+ * optional step index, and the normalized approval requirement.
+ */
+export interface CreateHoldData {
+    runId: string;
+    jobId: string;
+    /** Granularity of the held element. */
+    scope: HoldScope;
+    /** Step index within the job for step-scoped holds; omit otherwise. */
+    stepIndex?: number;
+    /** What triggered the hold (environment policy vs SDK requireApproval). */
+    triggerSource: TriggerSource;
+    /** The normalized requirement the hold must satisfy. */
+    requirement: ApprovalRequirement;
+    /** Environment id, when the hold originates from an environment policy. */
+    environmentId?: string | null;
+    /** Queue type: 'environment' (default) or 'security'. */
+    queueType?: 'environment' | 'security';
+    /**
+     * Held-run `hold_type` discriminator. Defaults to `'approval'` (reviewer
+     * holds). The workflow install gate sets `'wait_timer'` / `'concurrency'` so
+     * the automated release sweeps can find their rows.
+     */
+    holdType?: string;
+}
+/** A single decision to record against a hold. */
+export interface RecordDecisionData {
+    approverSub: string;
+    decision: ApprovalDecision;
+    /** Which requirement clauses this decision satisfied (for attribution). */
+    clausesSatisfied?: ApproverClause[];
+}
+/**
+ * The outcome of `release()` — describes how the held element must be resumed.
+ * The store only writes the terminal DB state; the caller performs the actual
+ * re-dispatch (job/workflow) or agent notification (step) using this signal.
+ */
+export interface ReleaseSignal {
+    holdId: string;
+    runId: string;
+    jobId: string;
+    scope: HoldScope;
+    /** Set only for step-scoped holds. */
+    stepIndex: number | null;
+}
 /** Options for listing held runs. */
 export interface ListHeldRunsOptions {
     status?: string;
@@ -35,6 +83,26 @@ export declare class HeldRunStore {
     constructor(db: Kysely<Database>);
     /** Create a new held run with pending status. */
     create(orgId: string, data: CreateHeldRunData): Promise<HeldRun>;
+    /**
+     * Create a generalized approval hold (workflow/job/step scope, explicit or
+     * environment trigger) carrying a normalized `ApprovalRequirement`. Returns
+     * the created row.
+     */
+    createHold(orgId: string, data: CreateHoldData): Promise<HeldRun>;
+    /** Record one approve/reject decision against a hold. */
+    recordDecision(heldRunId: string, data: RecordDecisionData): Promise<HeldRunApproval>;
+    /** List the recorded decisions for a hold, oldest first. */
+    listDecisions(heldRunId: string): Promise<HeldRunApproval[]>;
+    /** Get a single held run by id (org-scoped). Returns null if absent. */
+    getById(orgId: string, heldRunId: string): Promise<HeldRun | null>;
+    /**
+     * Release a hold whose approval requirement is satisfied. Flips the row to
+     * 'approved' and returns a `ReleaseSignal` describing how the caller must
+     * resume the element (re-dispatch for job/workflow, agent notification for
+     * step). Throws if the hold is not found or not pending. Approver attribution
+     * lives in `held_run_approvals`, not on the row.
+     */
+    release(orgId: string, heldRunId: string): Promise<ReleaseSignal>;
     /** Approve a pending held run. Throws if not found or not pending. */
     approve(orgId: string, heldRunId: string, approvedBy: string): Promise<HeldRun>;
     /** Reject a pending held run. Throws if not found or not pending. */
@@ -53,6 +121,30 @@ export declare class HeldRunStore {
     approveByQueueType(orgId: string, heldRunId: string, approvedBy: string, queueType: 'environment' | 'security'): Promise<HeldRun>;
     /** Get a held run by run ID and job ID. Returns null if not found. */
     getByRunAndJob(orgId: string, runId: string, jobId: string): Promise<HeldRun | null>;
+    /**
+     * List pending holds past their `expires_at`. Called by the stale detector
+     * BEFORE `expireOverdue()` so it can route each overdue hold by scope (step
+     * holds notify the waiting agent; job/workflow holds fail the run).
+     */
+    listOverdue(): Promise<HeldRun[]>;
+    /**
+     * Release overdue workflow `wait_timer` holds. The install-gate wait action
+     * pauses the workflow as a held run; on timer expiry it must RESUME (not
+     * fail like a reviewer-hold expiry). Flips each overdue pending
+     * `hold_type='wait_timer'`, `hold_scope='workflow'` row to `released` and
+     * returns a `ReleaseSignal` per row so the caller can resume the workflow.
+     * Runs BEFORE `expireOverdue()` so these rows leave the pending pool before
+     * the expire-and-fail sweep sees them.
+     */
+    releaseDueWaitHolds(): Promise<ReleaseSignal[]>;
+    /**
+     * Release the oldest pending workflow `concurrency` hold for a group. Called
+     * when a concurrency slot frees on run completion. Flips the oldest matching
+     * pending row to `released` and returns its `ReleaseSignal`, or null when no
+     * concurrency hold is queued for the group. `group` matches the held row's
+     * `environment_id` (workflow-level install concurrency keys on the env id).
+     */
+    releaseConcurrencyHold(orgId: string, group: string): Promise<ReleaseSignal | null>;
     /**
      * Expire overdue pending runs. Called by the stale detector.
      * Sets status to 'expired' and resolved_at to now() for all

package/dist/firecracker/host-network.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Firecracker host-network provisioning.
+ *
+ * Creates the per-coordinator bridge (kici-brN), assigns its gateway IP,
+ * marks kici-* interfaces unmanaged by NetworkManager, enables IP forwarding,
+ * and builds a disjoint, source-scoped nftables table for NAT + egress
+ * isolation. One pure command-builder drives live provisioning, the rendered
+ * boot script, and (read-only) verification.
+ *
+ * This is HOST setup, distinct from the runtime per-VM isolation in
+ * scaler/nftables.ts (added at spawn / removed at destroy). The two share the
+ * nft table name but have separate lifecycles; this module never touches the
+ * per-VM rules.
+ */
+/** A single subprocess invocation (no shell). */
+export interface CommandSpec {
+    bin: string;
+    args: string[];
+    /** Optional stdin payload (used for `nft -f -`). */
+    stdin?: string;
+}
+/** Host-bridge configuration for one Firecracker coordinator. */
+export interface FirecrackerBridgeConfig {
+    /** Bridge interface name, e.g. 'kici-br0'. */
+    bridgeName: string;
+    /** Gateway IP + prefix, e.g. '10.0.0.1/24'. */
+    bridgeCidr: string;
+    /** nft table name, e.g. 'kici' or 'kici_b'. */
+    table: string;
+    /** NAT egress interface; auto-detected from the default route when omitted. */
+    hostIface?: string;
+}
+/**
+ * Derive the network address (CIDR) from a gateway CIDR by masking host bits.
+ * '10.0.0.1/24' -> '10.0.0.0/24'.
+ */
+export declare function cidrToNetwork(cidr: string): string;
+/**
+ * Build the ordered command list that provisions one Firecracker host bridge.
+ * Pure — performs no I/O. `provisionBridge` executes these; `renderBootScript`
+ * serializes them.
+ *
+ * The nft `delete table`/`add table` here only ever touches `cfg.table`, so a
+ * coord-B provision never wipes coord A's table (and vice versa). Every
+ * forward/postrouting/MSS rule is source-scoped to the bridge subnet so two
+ * tables on the shared hooks do not cross-drop each other's traffic.
+ */
+export declare function buildBridgeCommands(cfg: FirecrackerBridgeConfig): CommandSpec[];
+export declare const NM_CONF_PATH = "/etc/NetworkManager/conf.d/90-kici-unmanaged.conf";
+export declare const NM_CONF_CONTENT: string;
+export type CommandRunner = (spec: CommandSpec) => Promise<{
+    stdout: string;
+}>;
+/** Writes the host-scoped NetworkManager conf. Injectable for tests. */
+export type FileWriter = (path: string, content: string) => Promise<void>;
+export interface ExecOptions {
+    /** Inject a runner for tests. */
+    runner?: CommandRunner;
+    /** Inject the NM-conf file writer for tests. */
+    writeNmConf?: FileWriter;
+    /** Wrap privileged bins with `sudo -n` (non-root orchestrator hosts). */
+    requireSudo?: boolean;
+}
+/** Resolve the default-route egress interface. */
+export declare function resolveHostIface(opts?: ExecOptions): Promise<string>;
+export interface BridgeHealth {
+    bridgeName: string;
+    bridgeExists: boolean;
+    bridgeUp: boolean;
+    addrPresent: boolean;
+    tablePresent: boolean;
+    healthy: boolean;
+    detail: string;
+}
+/** Provision (or heal) one Firecracker host bridge. Throws on any failure. */
+export declare function provisionBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<void>;
+/** Read-only health probe for one bridge. Never throws on a missing resource. */
+export declare function verifyBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<BridgeHealth>;
+/** Remove the bridge + its nft table. Leaves the host-scoped NM conf in place. */
+export declare function teardownBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<void>;
+/** Serialize the provisioning command list into a dependency-free boot script. */
+export declare function renderBootScript(cfg: FirecrackerBridgeConfig): string;
+//# sourceMappingURL=host-network.d.ts.map

package/dist/firecracker/persist.d.ts

@@ -0,0 +1,14 @@
+import { type FirecrackerBridgeConfig } from './host-network.js';
+export declare const BOOT_SCRIPT_DIR = "/usr/local/lib/kici-firecracker";
+export declare const SYSTEMD_DIR = "/etc/systemd/system";
+export declare function persistUnitName(bridgeName: string): string;
+export declare function renderPersistUnit(bridgeName: string): string;
+/**
+ * Install the boot script + unit and `systemctl enable` it (enable only — the
+ * deploy/CLI provisions the bridge live in the same run, so the unit fires on
+ * the NEXT boot rather than re-churning nft now).
+ */
+export declare function installPersist(cfg: FirecrackerBridgeConfig, opts?: {
+    requireSudo?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=persist.d.ts.map

package/dist/index.js

@@ -233,6 +233,9 @@ var S3CacheStorage = class {
 		items.sort((a, b) => b.created - a.created);
 		return items.map((i) => i.key);
 	}
+	async getMetadata(key) {
+		return this.readMeta(key);
+	}
 	async copy(srcKey, destKey) {
 		const srcObj = this.objectKey(srcKey);
 		const destObj = this.objectKey(destKey);
@@ -445,6 +448,9 @@ var FilesystemCacheStorage = class {
 		entries.sort((a, b) => b.mtime - a.mtime);
 		return entries.map((e) => e.key);
 	}
+	async getMetadata(key) {
+		return this.readMeta(key);
+	}
 	async copy(srcKey, destKey) {
 		const srcPath = this.resolvePath(srcKey);
 		const destPath = this.resolvePath(destKey);
@@ -741,9 +747,10 @@ var DepCache = class {
 * delete the temp), so a crashed save never leaves a corrupt final entry.
 *
 * Eviction: per-org byte quota plus the TTL the backing CacheStorage already
-* enforces lazily on access. On a save that pushes the org over quota, oldest
-* entries (by createdAt) are evicted until under quota; each eviction is
-* logged. The companion `.hash` / `.size` sidecar objects carry the integrity
+* enforces lazily on access. On a save that pushes the org over quota,
+* least-recently-used entries (oldest `lastAccessedAt`, read from the storage
+* backend's metadata) are evicted until under quota; each eviction is logged.
+* The companion `.hash` / `.size` sidecar objects carry the integrity
 * hash and size accounting outside the tarball's own (presigned, metadata-less)
 * upload.
 */
@@ -923,7 +930,7 @@ var UserCache = class {
 	async readHash(key) {
 		return (await this.storage.get(`${key}.hash`))?.toString("utf-8") || void 0;
 	}
-	/** Evict oldest entries for the org until total tarball size <= the per-org quota. */
+	/** Evict least-recently-used entries for the org until total tarball size <= the per-org quota. */
 	async enforceQuota(ref) {
 		const { quotaBytes } = await this.resolveLimits(ref.org);
 		const orgPrefix = this.orgPrefix(ref);
@@ -940,8 +947,19 @@ var UserCache = class {
 			});
 		}
 		if (total <= quotaBytes) return;
-		for (let i = sized.length - 1; i >= 0 && total > quotaBytes; i--) {
-			const { key, size } = sized[i];
+		const candidates = [];
+		for (const e of sized) {
+			const meta = await this.storage.getMetadata(e.key);
+			const lastAccessed = meta ? new Date(meta.lastAccessedAt).getTime() : 0;
+			candidates.push({
+				key: e.key,
+				size: e.size,
+				lastAccessed
+			});
+		}
+		candidates.sort((a, b) => a.lastAccessed - b.lastAccessed);
+		for (const { key, size } of candidates) {
+			if (total <= quotaBytes) break;
 			await this.storage.delete(key);
 			await this.storage.delete(`${key}.hash`);
 			await this.storage.delete(`${key}.size`);
@@ -973,6 +991,7 @@ const DEFAULT_EXPIRY_DAYS = 90;
 * Used by the coordinator to manage credentials for all cluster members.
 */
 var PeerCredentialStore = class {
+	db;
 	constructor(db) {
 		this.db = db;
 	}

package/dist/metrics/prometheus.d.ts

@@ -264,6 +264,8 @@ export declare const InstallSecretsDecisionReason: {
     readonly ProtectionRuleBlock: "protection_rule_block";
     readonly MissingToken: "missing_token";
     readonly MissingInstallEnv: "missing_install_env";
+    /** A protection gate held the workflow install (hold / wait / queue). */
+    readonly Held: "held";
 };
 export type InstallSecretsDecisionReason = (typeof InstallSecretsDecisionReason)[keyof typeof InstallSecretsDecisionReason];
 /**
@@ -280,8 +282,8 @@ export type InstallSecretsChannel = (typeof InstallSecretsChannel)[keyof typeof
  * Total install-secrets resolution decisions.
  *
  * Labels:
- * - decision: pass | reject
- * - reason: ok (only with `pass`), or one of the reject reasons enumerated in `InstallSecretsDecisionReason`
+ * - decision: pass | reject | hold
+ * - reason: ok (only with `pass`), held (only with `hold`), or one of the reject reasons enumerated in `InstallSecretsDecisionReason`
  *
  * Pass series are emitted exactly once per resolver call that does any work
  * (workflows declaring neither `registries:` nor `installEnv:` are NOT

package/dist/orchestrator-core.d.ts

@@ -22,7 +22,7 @@ import { DedupCache } from './webhook/dedup.js';
 import { ObserverRegistry } from './ws/observer-registry.js';
 import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
 import { SourceLocationStore } from './app.js';
-import { type PeerHeartbeat } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
 import { ScalerManager } from './scaler/index.js';
 import type { ScalerConfig } from './scaler/index.js';
 import type { CacheStorage } from './storage/index.js';
@@ -146,6 +146,14 @@ export interface OrchestratorSubsystems {
      * store. the metrics enforcement plan.
      */
     agentMetricsAggregator: AgentMetricsAggregator;
+    /**
+     * Peer-side fleet collect responder. On an inbound peer.logs.collect.request,
+     * assembles this node's subtree and streams it back. Wired into every outgoing
+     * PeerClient (the incoming peer-handler is wired directly in setupCluster).
+     * Armed after the fleet runtime is built, so calls before that resolve to a
+     * no-op.
+     */
+    fleetCollectResponder: (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
 }
 /**
  * Mode-specific hooks injected by each entry point.
@@ -202,6 +210,54 @@ export interface OrchestratorHooks {
      */
     startupLogMessage: (port: number) => string;
 }
+/**
+ * Normalize a lock job's `needs` array (strings, NeedsEntry objects, or
+ * NeedsGroupEntry objects) to the set of upstream BASE job names. Group entries
+ * are skipped here — group fan-in is resolved separately by the scheduler.
+ */
+export declare function upstreamBaseNamesFromNeeds(needs: unknown): string[];
+/**
+ * Build the downstream `upstreamJobOutputs` map keyed by BASE name. A base name
+ * that fanned into matrix children (rows with `matrix_values`) gets the
+ * `{ byMatrix, merged }` envelope; a single non-fanned row keeps the flat shape.
+ * Returns undefined when no upstream produced outputs.
+ */
+export declare function buildUpstreamOutputsByBase(baseNames: string[], rows: Array<{
+    job_name: string;
+    outputs: unknown;
+    matrix_values: unknown;
+}>): Record<string, Record<string, unknown>> | undefined;
+/**
+ * Group an upstream's child rows into the matrix outputs envelope
+ * `{ byMatrix: { '<suffix>': outputs }, merged: <last-write-wins> }`. The suffix
+ * is the text inside the `(...)` of each expanded child name; children are
+ * merged in name order (deterministic, matching dispatch order).
+ */
+export declare function buildMatrixOutputsEnvelope(baseName: string, children: Array<{
+    job_name: string;
+    parsed: Record<string, unknown>;
+}>): {
+    byMatrix: Record<string, Record<string, unknown>>;
+    merged: Record<string, unknown>;
+};
+/**
+ * Merge plain `outputs` and decrypted `secret outputs` from upstream jobs
+ * into the dispatch envelope's `secrets` + `upstreamJobOutputs` fields.
+ *
+ * For a fanned (matrix) upstream, the downstream sees a keyed envelope
+ * `{ byMatrix: { '<suffix>': outputs }, merged: <last-write-wins> }` under the
+ * BASE name; a non-fanned upstream keeps the flat outputs shape.
+ *
+ * The needs-aware scheduler guarantees upstreams are
+ * terminal before the downstream is dispatched, so the lookups below
+ * always find final values. Errors are swallowed (warn-logged) so a flaky
+ * upstream-outputs read doesn't block dispatch — the agent will still get
+ * its declared secrets, just without the merged upstream additions.
+ */
+export declare function mergeUpstreamOutputs(db: Kysely<Database>, runId: string, jobName: string, needs: unknown, dispatchSecrets: Record<string, string> | undefined, secretKey: string): Promise<{
+    mergedSecrets: Record<string, string> | undefined;
+    upstreamJobOutputs: Record<string, Record<string, unknown>> | undefined;
+}>;
 export declare function bootstrapOrchestrator(config: AppConfig, hooks: OrchestratorHooks, options?: {
     otelSdk?: {
         shutdown(): Promise<void>;

package/dist/pipeline/dispatch-matched-workflow.d.ts

@@ -91,6 +91,23 @@ export interface WorkflowDispatchContext {
 export interface DispatchMatchedWorkflowResult {
     /** Number of jobs successfully dispatched (non-rejected). */
     dispatchedJobCount: number;
+    /** True when the workflow install gate paused the dispatch (held run). */
+    held?: boolean;
+}
+/** Options controlling a (re-)dispatch of a matched workflow. */
+export interface DispatchMatchedWorkflowOptions {
+    /**
+     * Resume path: skip the workflow install protection gate (already satisfied)
+     * so secrets resolve directly and the dispatch flows into job dispatch.
+     */
+    skipInstallProtectionGate?: boolean;
+    /** The released held-run id being resumed (for logging / correlation). */
+    reuseHeldRunId?: string;
+    /**
+     * The run id whose `held` execution_runs row should be reused (flipped to
+     * pending) instead of inserting a fresh row.
+     */
+    reuseRunId?: string;
 }
 /**
  * Dispatch a single matched workflow.
@@ -107,5 +124,5 @@ export interface DispatchMatchedWorkflowResult {
  *   I. deferred init dispatch (fire-and-forget per job)
  *   J. deferred dynamic dispatch (fire-and-forget per dynamic entry)
  */
-export declare function dispatchMatchedWorkflow(ctx: WorkflowDispatchContext): Promise<DispatchMatchedWorkflowResult>;
+export declare function dispatchMatchedWorkflow(ctx: WorkflowDispatchContext, opts?: DispatchMatchedWorkflowOptions): Promise<DispatchMatchedWorkflowResult>;
 //# sourceMappingURL=dispatch-matched-workflow.d.ts.map