@kici-dev/orchestrator 0.0.0 → 0.1.1

package/dist/cluster/cluster-identity.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Cluster identity validation for split-brain prevention.
+ *
+ * In geo-distributed setups, orchestrators may use different connection strings
+ * for the same logical DB and S3. URL comparison would falsely flag them as
+ * different resources. cluster_id validates that orchestrators share the same
+ * logical DB, and the S3 sentinel validates they share the same logical bucket.
+ */
+import type { Kysely } from 'kysely';
+interface ClusterIdentityS3Client {
+    getObject(bucket: string, key: string): Promise<string | null>;
+    putObject(bucket: string, key: string, body: string): Promise<void>;
+}
+export interface ClusterIdentityDeps {
+    db: Kysely<any>;
+    s3Client?: ClusterIdentityS3Client;
+    storageBucket?: string;
+    /**
+     * Optional storage prefix (mirrors `KICI_STORAGE_PREFIX`). When set, the
+     * sentinel lives under that prefix (`<prefix>/.kici-cluster-id`) so multiple
+     * clusters can share the same physical bucket as long as they use distinct
+     * prefixes. When empty, the sentinel is at the bucket root.
+     */
+    storagePrefix?: string;
+    /**
+     * E2E escape hatch — when true, validateS3Sentinel logs a warning and
+     * returns early. Mirrors the orchestrator's
+     * `config.skipS3SentinelValidation` (KICI_SKIP_S3_SENTINEL_VALIDATION).
+     */
+    skipSentinelValidation?: boolean;
+}
+/**
+ * Build the S3 key for the cluster identity sentinel under an optional prefix.
+ * Strips any trailing slash from the prefix so callers can pass either form.
+ */
+export declare function clusterSentinelKey(prefix?: string): string;
+export declare class ClusterIdentity {
+    private readonly deps;
+    constructor(deps: ClusterIdentityDeps);
+    /** Read cluster_id from DB. Throws if not found (migration not run). */
+    getClusterId(): Promise<string>;
+    /**
+     * Validate S3 sentinel file matches DB cluster_id.
+     * - If no S3 configured, skip (filesystem-only deployment).
+     * - If sentinel doesn't exist, write it (first orch to use this bucket).
+     * - If sentinel exists and matches, success.
+     * - If sentinel exists and doesn't match, throw error (different DB/bucket mismatch).
+     */
+    validateS3Sentinel(): Promise<void>;
+    /**
+     * Validate cluster identity at startup.
+     * Logs cluster_id and instance_id for operational visibility, then runs
+     * S3 sentinel check. Throws on misconfiguration (e.g., different cluster
+     * using same DB/S3).
+     */
+    validateAtStartup(instanceId: string): Promise<string>;
+}
+export {};
+//# sourceMappingURL=cluster-identity.d.ts.map

package/dist/cluster/coordinator.d.ts

@@ -0,0 +1,204 @@
+/**
+ * Run coordinator for multi-orchestrator job routing.
+ *
+ * The receiving orchestrator becomes the run coordinator: it claims jobs
+ * it can dispatch locally and reroutes others to peers based on agent
+ * inventory. Per locked decisions:
+ * - "One orchestrator coordinates per webhook run"
+ * - "Claim-based splitting"
+ * - "Coordinator handles ALL check run reporting"
+ * - "Peers report step-by-step progress back to coordinator"
+ * - "Cancel mode: graceful -- finish current step, cancel remaining"
+ */
+import type { JobReroute, JobProgress, PeerToPeerMessage, ResourceRequest } from '@kici-dev/engine';
+import type { PeerRegistry } from './peer-registry.js';
+import type { PeerClient } from './peer-client.js';
+import type { Dispatcher } from '../agent/dispatcher.js';
+import type { ExecutionTracker } from '../reporting/execution-tracker.js';
+import type { CommitStatusReporter } from '../reporting/commit-status.js';
+export interface RunContext {
+    runId: string;
+    deliveryId: string;
+    routingKey: string;
+    event: string;
+    action: string | null;
+    provider: string;
+    payload: Record<string, unknown>;
+    repoIdentifier: string;
+    sha: string;
+    ref: string;
+    workflowName: string;
+    installationId?: number;
+    requestId?: string;
+    traceId?: string;
+    /** Pre-resolved clone token for workers without provider credentials. */
+    cloneToken?: string;
+}
+export interface JobToRoute {
+    jobName: string;
+    runsOnLabels: string[][];
+    jobConfig: Record<string, unknown>;
+    repoUrl: string;
+    ref: string;
+    sha: string;
+    sourceTarUrl?: string;
+    sourceTarHash?: string;
+    depsUrl?: string;
+    depsHash?: string;
+    /** Labels that the dispatched agent must NOT have. */
+    excludeLabels?: string[];
+    /**
+     * Per-job resource request and limit (K8s-style). Forwarded to
+     * `Dispatcher.dispatch()` so the scaler's per-scaler / per-orchestrator /
+     * per-machine cap accounting sees the typed value rather than relying on
+     * the `jobConfig.resources` JSON blob (which the dispatcher does not parse).
+     */
+    resources?: ResourceRequest;
+}
+export interface RouteResult {
+    localJobs: Array<{
+        jobName: string;
+        jobId: string;
+    }>;
+    /**
+     * Rerouted jobs include the pre-allocated jobId so the caller can
+     * register the execution_runs / execution_jobs rows under the same id
+     * the worker will report progress against.
+     */
+    reroutedJobs: Array<{
+        jobName: string;
+        peerId: string;
+        jobId: string;
+    }>;
+    failedJobs: Array<{
+        jobName: string;
+        reason: string;
+    }>;
+}
+export interface RunCoordinatorDeps {
+    instanceId: string;
+    peerRegistry: PeerRegistry;
+    dispatcher: Dispatcher;
+    executionTracker?: ExecutionTracker;
+    commitStatusReporter?: CommitStatusReporter;
+    getPeerClient: (instanceId: string) => PeerClient | undefined;
+    /** Fallback: send job.reroute via server-side (incoming) peer connection and wait for ACK. */
+    sendAndWaitAckViaHandler?: (targetInstanceId: string, msg: JobReroute, timeoutMs: number) => Promise<boolean>;
+    /** Fallback: send a message via server-side (incoming) peer connection (fire-and-forget). */
+    sendToPeerViaHandler?: (targetInstanceId: string, msg: PeerToPeerMessage) => boolean;
+    ackTimeoutMs?: number;
+    /** Stale peer timeout in ms. Default: 60000 (60s). */
+    staleTimeoutMs?: number;
+}
+export declare class RunCoordinator {
+    private readonly instanceId;
+    private readonly peerRegistry;
+    private readonly dispatcher;
+    private readonly executionTracker?;
+    private readonly commitStatusReporter?;
+    private readonly getPeerClient;
+    private readonly sendAndWaitAckViaHandler?;
+    private readonly sendToPeerViaHandler?;
+    private readonly ackTimeoutMs;
+    /**
+     * Tracks which jobs have been rerouted to which peers, keyed by runId.
+     * Used for cancel propagation and progress tracking.
+     */
+    private readonly reroutedJobs;
+    /**
+     * NAK tracking per peer: count of consecutive NAKs and backoff-until timestamp.
+     * Peers that repeatedly NAK are deprioritized via exponential backoff.
+     */
+    private readonly nakTracker;
+    /** Stale eviction timer handle. */
+    private staleEvictionTimer;
+    constructor(deps: RunCoordinatorDeps);
+    /**
+     * Route jobs for a webhook run. Claims jobs that local agents can handle,
+     * reroutes the rest to peers with matching capacity.
+     *
+     * Parallel fan-out: jobs targeting different peers are rerouted concurrently.
+     */
+    routeJobs(runContext: RunContext, jobs: JobToRoute[]): Promise<RouteResult>;
+    /**
+     * Handle a job reroute request received from a peer (this instance is the worker).
+     * Checks loop prevention, then attempts to dispatch locally.
+     */
+    handleIncomingReroute(msg: JobReroute): Promise<{
+        accepted: boolean;
+        reason?: string;
+    }>;
+    /**
+     * Handle peer job/step progress updates from a worker (or coord) peer.
+     *
+     * The discriminator `msg.kind` decides which downstream tracker call to
+     * make: 'job' updates feed `onJobStatus` (the path that drives run-level
+     * state transitions and the rerouted-job cleanup); 'step' updates feed
+     * `onStepStatus` (which only persists `execution_steps` rows). Without
+     * this split, every job-level event was silently funnelled into
+     * `onStepStatus` and the run stayed in `running` forever.
+     */
+    onPeerJobProgress(msg: JobProgress): void;
+    /**
+     * Handle peer job completion. Updates ExecutionTracker.
+     * The execution tracker's onJobStatus fires the needs-aware scheduler hook,
+     * which evaluates downstream jobs and dispatches newly-ready ones via onJobReadyCallback.
+     * The data parameter carries droppedJobs for drift reporting.
+     */
+    onPeerJobComplete(runId: string, jobId: string, status: string, timestamp: number, data?: Record<string, unknown>): void;
+    /**
+     * Cancel all rerouted jobs for a run. Sends peer.job.cancel to all
+     * peers that have jobs for this run.
+     *
+     * Per locked decision: "graceful -- finish current step, cancel remaining."
+     */
+    cancelRun(runId: string, reason: string): void;
+    /**
+     * Check if the peer registry has any connected peers.
+     */
+    hasConnectedPeers(): boolean;
+    /**
+     * Start the stale eviction timer. Calls peerRegistry.evictStalePeers()
+     * every staleTimeoutMs/2 to detect and remove stale peers.
+     */
+    startStaleEvictionTimer(staleTimeoutMs: number): void;
+    /**
+     * Stop the stale eviction timer.
+     */
+    stopStaleEvictionTimer(): void;
+    /**
+     * Get the NAK count for a peer (used in tests).
+     */
+    getNakCount(peerId: string): number;
+    /**
+     * Get the backoff-until timestamp for a peer (used in tests).
+     */
+    getBackoffUntil(peerId: string): number;
+    /**
+     * Attempt to reroute a job to a peer with matching capacity.
+     * Tries peers in order of available capacity (most capacity first).
+     * Returns on first successful ACK.
+     */
+    private rerouteJob;
+    /**
+     * Sort peers by available capacity for the given label sets.
+     * Peers with more available capacity come first.
+     */
+    private sortPeersByCapacity;
+    /**
+     * Compute total available capacity across matching agents on a peer.
+     *
+     * Applies the same `mandatoryLabels` gate the local label matcher
+     * applies — a gated agent only contributes capacity for label sets that
+     * include every gate label. Without it, the coordinator would route a
+     * job to a peer based on a gated agent's labels, and the peer would
+     * then refuse the dispatch in `AgentRegistry.findAvailable` because the
+     * agent's gate is not satisfied.
+     */
+    private computeAvailableCapacity;
+    /**
+     * Track a rerouted job for cancel propagation.
+     */
+    private trackReroutedJob;
+}
+//# sourceMappingURL=coordinator.d.ts.map

package/dist/cluster/health-api.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Cluster health HTTP endpoints.
+ *
+ * Provides visibility into cluster state for monitoring and debugging:
+ * - GET /cluster/health -- overall cluster health status
+ * - GET /cluster/peers -- per-peer details including connectivity
+ * - GET /cluster/runs -- active execution runs with job counts
+ *
+ * Follows the existing Hono route factory pattern from routes/health.ts.
+ */
+import { Hono } from 'hono';
+import type { RaftNode } from './raft.js';
+import type { PeerRegistry } from './peer-registry.js';
+import type { ExecutionTracker } from '../reporting/execution-tracker.js';
+import type { AgentRegistry } from '../agent/registry.js';
+export interface ClusterHealthRoutesDeps {
+    instanceId: string;
+    raft: RaftNode;
+    peerRegistry: PeerRegistry;
+    executionTracker: ExecutionTracker;
+    agentRegistry: AgentRegistry;
+}
+/**
+ * Create cluster health routes.
+ *
+ * @param deps - Dependencies for cluster state introspection
+ * @returns Hono app with /cluster/health, /cluster/peers, /cluster/runs endpoints
+ */
+export declare function createClusterHealthRoutes(deps: ClusterHealthRoutesDeps): Hono;
+//# sourceMappingURL=health-api.d.ts.map

package/dist/cluster/index.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Barrel export for the cluster coordination module.
+ *
+ * All cluster-related functionality is exported from this single entry point
+ * for clean imports in server.ts and standalone.ts.
+ */
+export { PeerRegistry } from './peer-registry.js';
+export type { PeerInfo, PeerAgentInfo, PeerRegistryOptions } from './peer-registry.js';
+export { PeerClient } from './peer-client.js';
+export type { PeerClientOptions } from './peer-client.js';
+export { createPeerHandler } from './peer-handler.js';
+export type { PeerHandlerDeps } from './peer-handler.js';
+export { PeerCredentialStore, readCredentialFile, writeCredentialFile, } from './peer-credentials.js';
+export type { PeerCredential, CredentialFileData } from './peer-credentials.js';
+export { RaftNode } from './raft.js';
+export type { RaftRole } from './raft.js';
+export { RaftStateStore } from './raft-state.js';
+export type { RaftPersistentState } from './raft-state.js';
+export { OrphanRecovery } from './orphan-recovery.js';
+export { RunCoordinator } from './coordinator.js';
+export type { RunContext, JobToRoute, RouteResult } from './coordinator.js';
+export { createClusterHealthRoutes } from './health-api.js';
+export type { ClusterHealthRoutesDeps } from './health-api.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/cluster/join-client.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Join client for zero-knowledge cluster bootstrap.
+ *
+ * Provides the joiner-side logic for `kici-admin join`:
+ * 1. Connect to Platform relay (WS) or direct peer (HTTP POST)
+ * 2. Send join.request with the join token
+ * 3. Receive join.response with encrypted config bundle
+ * 4. Decrypt bundle using token-derived AES-256-GCM key
+ * 5. Write decrypted config as local YAML file
+ *
+ * The token carries routing info (cleartext for Platform relay) and a secret
+ * (used for HKDF key derivation). Only the joiner and the token creator
+ * can derive the encryption key -- the Platform relay sees only ciphertext.
+ */
+import { type JoinRequest, type JoinResponse } from '@kici-dev/engine';
+import type { ConfigBundle } from './join-handler.js';
+interface JoinClientOptions {
+    token: string;
+    /** Platform WebSocket URL for relay mode (e.g., wss://platform.kici.dev/ws) */
+    platformUrl?: string;
+    /** Peer HTTP URL for direct mode (e.g., https://orch-1:8080) */
+    peerUrl?: string;
+    /** API key for Platform authentication (required for --platform mode) */
+    apiKey?: string;
+    /** Path to write the resulting local config YAML */
+    configPath?: string;
+}
+/**
+ * Local config structure written by the join client.
+ * Compatible with loadLocalConfig() from config/loader.ts.
+ */
+interface JoinLocalConfig {
+    database: {
+        url: string;
+    };
+    storage?: ConfigBundle['storage'];
+    secrets?: {
+        key: string;
+    };
+}
+/**
+ * Decrypt a base64-encoded encrypted config bundle using a derived encryption key.
+ */
+export declare function decryptAndParseBundle(encryptedB64: string, encryptionKey: Buffer): ConfigBundle;
+/**
+ * Build a JoinLocalConfig from a decrypted ConfigBundle.
+ * Maps bundle fields to the structure expected by loadLocalConfig().
+ */
+export declare function buildLocalConfig(bundle: ConfigBundle): JoinLocalConfig;
+/**
+ * Write a local config object to a YAML file.
+ */
+export declare function writeConfigFile(path: string, config: Record<string, unknown>): Promise<void>;
+export declare class JoinClient {
+    private readonly options;
+    constructor(options: JoinClientOptions);
+    /**
+     * Execute the join flow:
+     * 1. Send join.request with token to Platform relay or direct peer
+     * 2. Receive join.response with encrypted config bundle
+     * 3. Decrypt bundle using token-derived key
+     * 4. Write local config YAML
+     */
+    join(): Promise<void>;
+    /**
+     * Join via Platform relay: connect WS, authenticate, send join.request, receive join.response.
+     */
+    joinViaPlatform(request: JoinRequest): Promise<JoinResponse>;
+    /**
+     * Join via direct peer: POST to peer's join endpoint.
+     */
+    joinViaPeer(request: JoinRequest): Promise<JoinResponse>;
+}
+export {};
+//# sourceMappingURL=join-client.d.ts.map

package/dist/cluster/join-handler.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Join handler for processing join requests from new orchestrators.
+ *
+ * Supports both Platform relay and direct peer transport. When a new orchestrator
+ * sends a join.request with a kici_join_v1 token, this handler:
+ * 1. Validates the token (exists in DB, not expired, not consumed)
+ * 2. Builds a config bundle (DB URL, S3 config, secrets key, cluster ID)
+ * 3. Encrypts the bundle with the token-derived AES-256-GCM key
+ * 4. Consumes the token (one-time use)
+ * 5. Returns the encrypted bundle to the joiner
+ *
+ * The Platform relay sees only routing metadata and ciphertext -- zero-knowledge.
+ */
+import type { JoinRequest, JoinResponse } from '@kici-dev/engine';
+import type { Kysely } from 'kysely';
+import type { ClusterIdentity } from './cluster-identity.js';
+import type { SharedConfigStore } from '../config/shared-store.js';
+interface JoinHandlerDeps {
+    db: Kysely<any>;
+    sharedConfigStore: SharedConfigStore;
+    clusterIdentity: ClusterIdentity;
+    databaseUrl: string;
+}
+/**
+ * Config bundle distributed to new orchestrators via join token.
+ * Contains everything needed to configure and start an orchestrator instance.
+ */
+export interface ConfigBundle {
+    databaseUrl: string;
+    storage?: {
+        type?: 's3';
+        bucket?: string;
+        prefix?: string;
+        region?: string;
+        endpoint?: string;
+        forcePathStyle?: boolean;
+        logBucket?: string;
+    };
+    secretKey?: string;
+    clusterId: string;
+}
+export declare class JoinHandler {
+    private readonly deps;
+    private readonly tokenManager;
+    constructor(deps: JoinHandlerDeps);
+    /**
+     * Handle a join request from a new orchestrator.
+     * Validates token, builds config bundle, encrypts with token-derived key, consumes token.
+     * Echoes messageId from request for Platform relay correlation.
+     */
+    handleJoinRequest(request: JoinRequest): Promise<JoinResponse>;
+    /**
+     * Build the config bundle from SharedConfig + local config.
+     */
+    buildConfigBundle(): Promise<ConfigBundle>;
+}
+export {};
+//# sourceMappingURL=join-handler.d.ts.map

package/dist/cluster/join-token.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Join token manager for zero-knowledge cluster bootstrap.
+ *
+ * Token format: kici_join_v1.<base64url(routing_json)>.<random_256bit_hex>
+ * - Routing JSON: { orgId, routingKey, expiry } (cleartext for Platform relay routing)
+ * - Secret: 32 random bytes as hex (used for key derivation)
+ *
+ * Key derivation:
+ * - encryption_key = HKDF-SHA256(secret, salt="kici-join-encrypt", info="v1", length=32)
+ * - validation_hash = SHA-256(secret) (stored in DB for lookup)
+ *
+ * Config bundle encryption: AES-256-GCM with random 12-byte IV.
+ * Wire format: <12-byte IV><16-byte auth tag><ciphertext>
+ */
+import type { Kysely } from 'kysely';
+/**
+ * Silence the module-level logger (e.g. when a CLI is emitting JSON on stdout
+ * and a stray log line would break the contract).
+ */
+export declare function silenceJoinTokenLogger(): void;
+interface TokenRouting {
+    orgId: string;
+    routingKey: string;
+    expiry: number;
+    role: 'coordinator' | 'worker';
+}
+interface DerivedKeys {
+    encryptionKey: Buffer;
+    validationHash: string;
+}
+interface JoinTokenManagerDeps {
+    db: Kysely<any>;
+}
+export declare class JoinTokenManager {
+    private readonly deps;
+    constructor(deps: JoinTokenManagerDeps);
+    /**
+     * Create a new join token.
+     * Returns the full token string (only available at creation time).
+     */
+    createToken(opts: {
+        orgId: string;
+        routingKey: string;
+        createdBy: string;
+        role?: 'coordinator' | 'worker';
+        expiryMs?: number;
+    }): Promise<string>;
+    /**
+     * Validate a token: check hash exists in DB, not expired, not consumed.
+     * Returns routing info and derived keys on success.
+     */
+    validateToken(token: string): Promise<{
+        routing: TokenRouting;
+        keys: DerivedKeys;
+    }>;
+    /**
+     * Mark token as consumed (one-time use).
+     */
+    consumeToken(validationHash: string, consumedBy: string): Promise<void>;
+}
+/**
+ * Narrow detector for the "already been used" error thrown by validateToken().
+ * Used by peer-handler.ts to branch into the idempotent recovery path on
+ * legitimate mesh-join races (sibling peer-clients on the same peer identity
+ * racing on a shared join token across a multi-coordinator shared-DB mesh).
+ * Other validation failures (expired, not found, bad parse) MUST NOT be
+ * treated as recoverable.
+ */
+export declare function isTokenAlreadyUsedError(err: unknown): boolean;
+/**
+ * Parse a join token string into routing info and secret hex.
+ */
+export declare function parseToken(token: string): {
+    routing: TokenRouting;
+    secretHex: string;
+};
+/**
+ * Derive encryption key and validation hash from a token secret.
+ * - encryptionKey: HKDF-SHA256 with salt="kici-join-encrypt", info="v1"
+ * - validationHash: SHA-256 of secret (stored in DB for lookup)
+ */
+export declare function deriveKeys(secret: Buffer): DerivedKeys;
+/**
+ * Encrypt a config bundle with the derived encryption key.
+ * Format: <12-byte IV><16-byte auth tag><ciphertext>
+ */
+export declare function encryptBundle(plaintext: object, encryptionKey: Buffer): Buffer;
+/**
+ * Decrypt a config bundle with the derived encryption key.
+ * Expects format: <12-byte IV><16-byte auth tag><ciphertext>
+ */
+export declare function decryptBundle(data: Buffer, encryptionKey: Buffer): object;
+export {};
+//# sourceMappingURL=join-token.d.ts.map

package/dist/cluster/orphan-recovery.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Leader-only orphan run cleanup.
+ *
+ * Periodically scans for execution runs stuck in 'running' state whose
+ * coordinator orchestrator has crashed. Only the Raft leader runs recovery
+ * to prevent multiple orchestrators from racing to recover the same run.
+ *
+ * Recovery logic:
+ * 1. Find runs with status='running' and stale last update (> threshold)
+ * 2. Check if the coordinator is still a connected peer
+ * 3. If coordinator is disconnected/unknown, this is an orphan run
+ * 4. Check job statuses: terminal jobs stay, stuck jobs get marked failed
+ * 5. Finalize the run with computed overall status
+ */
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+import type { RaftNode } from './raft.js';
+import type { PeerRegistry } from './peer-registry.js';
+import type { ExecutionTracker } from '../reporting/execution-tracker.js';
+interface OrphanRecoveryDeps {
+    db: Kysely<Database>;
+    raft: RaftNode;
+    peerRegistry: PeerRegistry;
+    executionTracker: ExecutionTracker;
+    /** Scan interval in ms. Default: 60000 (1 minute). */
+    scanIntervalMs?: number;
+    /** How long a run must be stale before considered orphaned. Default: 5 minutes. */
+    staleThresholdMs?: number;
+    /** How long a stuck job must have no heartbeat before marking as failed. Default: 3 minutes. */
+    jobStuckThresholdMs?: number;
+}
+export declare class OrphanRecovery {
+    private readonly db;
+    private readonly raft;
+    private readonly peerRegistry;
+    private readonly executionTracker;
+    private readonly scanIntervalMs;
+    private readonly staleThresholdMs;
+    private readonly jobStuckThresholdMs;
+    private scanTimer;
+    constructor(deps: OrphanRecoveryDeps);
+    /**
+     * Start periodic orphan recovery scans.
+     * Only runs recovery logic when this node is the Raft leader.
+     */
+    start(): void;
+    /**
+     * Stop periodic orphan recovery scans.
+     */
+    stop(): void;
+    /**
+     * Scan for orphaned runs and recover them.
+     * Only runs when this node is the Raft leader.
+     */
+    scanForOrphans(): Promise<void>;
+    /**
+     * Attempt to recover a single potentially orphaned run.
+     */
+    private recoverRun;
+    /**
+     * Check if a coordinator with the given routing key is still connected
+     * as a peer in the cluster.
+     */
+    private isCoordinatorStillConnected;
+}
+export {};
+//# sourceMappingURL=orphan-recovery.d.ts.map