@opengeni/runtime 0.14.1 → 0.14.15

package/dist/sandbox/selfhosted/op-testing.d.ts

@@ -0,0 +1,108 @@
+import { ControlRequest, ControlResponse, OpAck, OpFrame, type AgentError, type OpExit } from "@opengeni/agent-proto";
+import type { ControlRpc } from "./control-rpc";
+import { type OpStreamSubscription, type OpStreamTransport } from "./op-transport";
+/**
+ * An in-memory `OpStreamTransport`: subscriptions are a subject→handler map,
+ * and everything the CLIENT publishes (its acks) is recorded for assertions
+ * and mirrored to `onPublish` (the fake runner's ack intake). `available =
+ * false` makes both primitives throw `OpStreamUnavailableError` — the
+ * legacy-fallback path.
+ */
+export declare class InMemoryOpStreamTransport implements OpStreamTransport {
+    available: boolean;
+    readonly published: Array<{
+        subject: string;
+        payload: Uint8Array;
+    }>;
+    onPublish: ((subject: string, payload: Uint8Array) => void) | undefined;
+    private readonly handlers;
+    subscribe(subject: string, onMessage: (payload: Uint8Array) => void): Promise<OpStreamSubscription>;
+    publish(subject: string, payload: Uint8Array): Promise<void>;
+    /** Deliver a payload to the subject's subscribers (the runner→client leg). */
+    deliver(subject: string, payload: Uint8Array): void;
+    /** Every OpAck the client published, decoded, in publish order. */
+    decodedAcks(): OpAck[];
+}
+/** One scripted data frame (the exit frame is synthesized from `exit`). */
+export interface FakeOpDataFrame {
+    channel: "stdout" | "stderr";
+    bytes: Uint8Array | string;
+}
+export interface FakeOpScript {
+    /** Data/progress frames in emission order (seq 1..N; the exit takes N+1). */
+    frames: Array<FakeOpDataFrame | "progress">;
+    /** Exit overrides (exitCode/timedOut/failureCode/…). Digests and totals are
+     *  computed from the scripted data frames unless overridden. */
+    exit?: Partial<OpExit>;
+    /** LIVE mode: the first attach replays nothing and triggers the (faulty)
+     *  live emission; later attaches replay from retention unfiltered. Default
+     *  false = the op is COMPLETE at start (pure collection). */
+    live?: boolean;
+    /** Keep a live op RUNNING with no frames until OpCancel is received. */
+    holdUntilCancel?: boolean;
+    /** Seqs whose LIVE emission is dropped (retention keeps them). */
+    dropLiveSeqs?: ReadonlySet<number>;
+    /** Seqs whose LIVE emission is duplicated (delivered twice back-to-back). */
+    duplicateLiveSeqs?: ReadonlySet<number>;
+    /** Swap each adjacent live pair (1,2 → 2,1) to exercise the stash path. */
+    reorderLivePairs?: boolean;
+    /** The first N OpStarts answer a DRAINING AgentError (admission pressure). */
+    drainingStarts?: number;
+    /** Refuse every OpStart with this error (e.g. a PROTOCOL refusal from a
+     *  runner that does not speak op-stream — the legacy-fallback race). */
+    startError?: AgentError;
+}
+interface FakeOpRun {
+    script: FakeOpScript;
+    frames: OpFrame[];
+    exitSeq: bigint;
+    exit: OpExit;
+    startCount: number;
+    attachCount: number;
+    drainingRemaining: number;
+    liveEmitted: boolean;
+    finalAcked: boolean;
+    acks: OpAck[];
+    highestGeneration: bigint;
+}
+/**
+ * The scripted fake runner: a `ControlRpc` that answers the op-stream control
+ * ops and emits frames through an `InMemoryOpStreamTransport`. Non-op-stream
+ * requests are delegated to `fallback` (a `MockAgentResponder`) so the
+ * legacy-fallback path is testable through the same instance.
+ */
+export declare class FakeOpRunner implements ControlRpc {
+    private readonly transport;
+    private readonly fallback;
+    private readonly workspaceId;
+    private readonly agentId;
+    private readonly scripts;
+    readonly runs: Map<string, FakeOpRun>;
+    /** Op ids that answer LOST (evicted) on query/attach — bounded-retention
+     *  collateral. */
+    readonly lostOps: Set<string>;
+    private readonly cancelledBeforeStart;
+    constructor(opts: {
+        transport: InMemoryOpStreamTransport;
+        workspaceId: string;
+        agentId: string;
+        /** Handles every non-op-stream ControlRequest (the legacy ops). */
+        fallback?: ControlRpc;
+    });
+    /** Script the op the NEXT OpStart with this id will run. */
+    script(opId: string, script: FakeOpScript): void;
+    private onAck;
+    request(subject: string, req: ControlRequest, opts: {
+        timeoutMs: number;
+    }): Promise<ControlResponse>;
+    private handleStart;
+    private handleAttach;
+    private handleQuery;
+    private handleCancel;
+    /** The LIVE flow: fault filters applied (drops, duplicates, adjacent
+     *  reordering); retention still holds every frame for the heal replay. */
+    private emitLive;
+    private statusOf;
+    private buildRun;
+}
+export {};

package/dist/sandbox/selfhosted/op-transport.d.ts

@@ -0,0 +1,60 @@
+/** The per-op frame subject (runner→server, fire-and-forget). Mirrors the
+ *  runner's wire constant: `agent.<ws>.<id>.op.<op_id>`. */
+export declare function opFrameSubject(workspaceId: string, agentId: string, opId: string): string;
+/** The per-agent ack subject (server→runner, fire-and-forget). Mirrors the
+ *  runner's wire constant: `agent.<ws>.<id>.ack`. */
+export declare function opAckSubject(workspaceId: string, agentId: string): string;
+/** A live subscription handle. `unsubscribe` is idempotent and never throws. */
+export interface OpStreamSubscription {
+    unsubscribe(): void;
+}
+/**
+ * The op-stream transport seam. `subscribe` MUST be established before the
+ * OpStart that makes frames flow (subscription-before-start is a protocol
+ * invariant: no frame is published before the consumer exists on a healthy
+ * path; a missed frame is healed via OpAttach replay regardless). `onMessage`
+ * is invoked in arrival order; a decode failure inside the handler must be
+ * contained by the caller (a torn frame must never kill the subscription).
+ */
+export interface OpStreamTransport {
+    subscribe(subject: string, onMessage: (payload: Uint8Array) => void): Promise<OpStreamSubscription>;
+    publish(subject: string, payload: Uint8Array): Promise<void>;
+}
+/** Thrown when the transport has no live connection: the op-stream path is
+ *  UNAVAILABLE (never silently degraded). The session catches this pre-start
+ *  and falls back to the legacy exec — the op provably never started. */
+export declare class OpStreamUnavailableError extends Error {
+    readonly name = "OpStreamUnavailableError";
+}
+/**
+ * The minimal NATS surface the transport needs — mirrors the `nats`
+ * `NatsConnection` subscribe/publish WITHOUT importing `nats` into the
+ * agent-loop-free runtime leaf (the worker injects the live `@opengeni/events`
+ * bus connection, exactly like `NatsRequestConnection` in control-rpc.ts).
+ * `subscribe` returns the nats.js Subscription shape: an async iterable of
+ * messages plus `unsubscribe()`.
+ */
+export interface NatsOpStreamConnection {
+    subscribe(subject: string): AsyncIterable<{
+        data: Uint8Array;
+    }> & {
+        unsubscribe(): void;
+    };
+    publish(subject: string, payload: Uint8Array): void;
+}
+/**
+ * The NATS-backed transport. Lazy memoized factory (identical discipline to
+ * `NatsControlRpc`): the connection resolves on first use; a null factory
+ * result or a dial failure surfaces `OpStreamUnavailableError` — the caller
+ * (the session) falls back to the LEGACY exec path rather than failing the op,
+ * so a NATS-less boot or a torn bus never strands a turn on a dead transport.
+ */
+export declare class NatsOpStreamTransport implements OpStreamTransport {
+    private readonly connect;
+    private connection;
+    private connecting;
+    constructor(connect: () => Promise<NatsOpStreamConnection | null>);
+    private resolveConnection;
+    subscribe(subject: string, onMessage: (payload: Uint8Array) => void): Promise<OpStreamSubscription>;
+    publish(subject: string, payload: Uint8Array): Promise<void>;
+}

package/dist/sandbox/selfhosted/retry-policy.d.ts

@@ -0,0 +1,106 @@
+import type { SelfhostedControlError } from "./control-rpc";
+/**
+ * The read-only, idempotent-safe control ops that MAY be retried after a TIMEOUT
+ * / `agent_reconnecting` blip.
+ *
+ * A timed-out request is AT-LEAST-ONCE on the wire: the agent may have RECEIVED
+ * and fully EXECUTED the op before its reply was lost, so re-issuing it is only
+ * safe when a second execution is observably identical to one — i.e. a pure read.
+ * Every MUTATING op (`exec`, `git`, `fsWrite`/`fsMove`/`fsMkdir`/`fsRemove`, and
+ * the pty/desktop input ops) is DELIBERATELY EXCLUDED: re-running a mutation that
+ * already ran on the machine is a real correctness hazard (a duplicated write, a
+ * second `git push`, a re-applied command). Only DRAINING (a pre-admission
+ * rejection — see `decideSelfhostedRetry`) is safe to retry for those.
+ */
+export declare const SELFHOSTED_IDEMPOTENT_READONLY_OPS: ReadonlySet<string>;
+/**
+ * DRAINING is a pre-admission backpressure rejection (the op never started), so it
+ * is safe to retry for ANY op kind. The budget is split by op class:
+ *
+ *  - NON-EXEC control ops (fs/git/desktop/pty) keep a SHORT budget — 3 retries,
+ *    worst-case backoff 500+1000+2000 = 3.5s (≤5s). Their start latency matters
+ *    (fs/git ops gate the model's next step), so a saturated machine should surface
+ *    quickly rather than stall.
+ *  - EXEC gets a LONG, patient budget — 10 retries, worst-case ≈55.5s (<60s). exec
+ *    start is not latency-critical, and the agent-side admission pool is (until a
+ *    later agent release) a FLAT 8 permits with no per-class split: a burst of slow
+ *    execs can hold every permit and blanket-DRAIN all fs/git ops. A patient exec
+ *    retry converts those DRAINING storms into mild queueing instead of a wall of
+ *    failed tool calls, and pairs with the deliberately modest exec deadline (which,
+ *    for the same 8-permit reason, defaults to 2min, not 5).
+ */
+export declare const SELFHOSTED_DRAINING_MAX_RETRIES = 3;
+export declare const SELFHOSTED_EXEC_DRAINING_MAX_RETRIES = 10;
+/** A read-only op that timed out is retried at most once (see the at-least-once
+ *  note above — even a read gets a single re-issue, not an unbounded loop). */
+export declare const SELFHOSTED_TIMEOUT_MAX_RETRIES = 1;
+/** A fault the transport synthesized PRE-SEND (no connection / no responder) — the
+ *  op provably never reached the machine, so it is safe to re-issue for ANY op kind.
+ *  Short bounded budget (≈3.5s summed): enough to ride out a NATS reconnect blip
+ *  quietly, not so long it stalls the model on a machine that is genuinely down. */
+export declare const SELFHOSTED_NEVER_SENT_MAX_RETRIES = 3;
+/** Full-jitter backoff base (ms). Delay N is sampled in `[0, base * factor^N)`. */
+export declare const SELFHOSTED_RETRY_BACKOFF_BASE_MS = 500;
+export declare const SELFHOSTED_RETRY_BACKOFF_FACTOR = 2;
+/** Per-delay ceiling: 500 → 1s → 2s → 4s → 8s, then flat at 8s. Bounds the summed
+ *  backoff — non-exec (3 retries) ≈3.5s ≤5s; exec (10 retries) ≈55.5s <60s. */
+export declare const SELFHOSTED_RETRY_BACKOFF_CAP_MS = 8000;
+export type SelfhostedRetryDecision = {
+    readonly action: "fail";
+} | {
+    readonly action: "retry";
+    readonly delayMs: number;
+};
+/**
+ * Full-jitter exponential backoff. Returns a delay in
+ * `[0, min(cap, base * factor^attempt))`. `jitter` is an injected sample in
+ * `[0, 1)` so the delay is fully deterministic under test.
+ */
+export declare function selfhostedRetryBackoffMs(attempt: number, jitter: number): number;
+/**
+ * PURE retry policy for one selfhosted control-op reply.
+ *
+ *  - `error.draining` (DRAINING — pre-admission host-work backpressure; the op was
+ *    rejected at the pool gate and NEVER started) → retry for ANY op kind, since
+ *    nothing executed. exec gets the long `SELFHOSTED_EXEC_DRAINING_MAX_RETRIES`
+ *    budget (≈60s of patient queueing under the flat 8-permit pool); every other op
+ *    keeps the short `SELFHOSTED_DRAINING_MAX_RETRIES` budget (≤5s).
+ *  - `error.neverSent` (a pre-send AGENT_OFFLINE synthesis — no connection / no
+ *    responder; the op provably never reached the machine) → retry for ANY op kind
+ *    up to `SELFHOSTED_NEVER_SENT_MAX_RETRIES`. At-least-once does NOT apply (nothing
+ *    executed), so even a mutation is safe to re-issue; this heals a NATS reconnect
+ *    blip quietly. A truly-offline machine surfaces honestly after the short budget.
+ *  - `error.reason === "agent_reconnecting"` (a TIMEOUT / transient blip AFTER send)
+ *    → retry at most `SELFHOSTED_TIMEOUT_MAX_RETRIES`, and ONLY for a read-only
+ *    idempotent-safe op (`SELFHOSTED_IDEMPOTENT_READONLY_OPS`). A timed-out
+ *    MUTATION is never retried here — it may already have run (at-least-once).
+ *  - Everything else → no retry at this layer:
+ *      · FENCED is retried by the routing proxy against a RE-RESOLVED backend
+ *        (retrying here under the same stale epoch would just re-fence);
+ *      · AGENT_OFFLINE / OS / PROTOCOL / PAYLOAD_TOO_LARGE / CONSENT_REQUIRED are
+ *        not transient and must surface to the caller.
+ *
+ * The draining and timeout budgets are SEPARATE counters so a read that first hit
+ * backpressure still keeps its single timeout re-issue (and vice versa); both are
+ * bounded, so the total number of attempts — and the summed backoff — is bounded.
+ */
+export declare function decideSelfhostedRetry(input: {
+    opKind: string;
+    error: SelfhostedControlError;
+    drainingRetries: number;
+    timeoutRetries: number;
+    /** Prior never-sent retries. Optional (default 0) — only the pre-send AGENT_OFFLINE
+     *  branch reads it, so callers that cannot hit that class may omit it. */
+    neverSentRetries?: number;
+    jitter: number;
+}): SelfhostedRetryDecision;
+/**
+ * The injected clock the retry loop drives — real in production, a deterministic
+ * fake in tests. `jitter()` returns a full-jitter sample in `[0, 1)`.
+ */
+export interface SelfhostedRetryClock {
+    sleep(ms: number): Promise<void>;
+    jitter(): number;
+}
+/** The production clock: a real timer + `Math.random()` jitter. */
+export declare const defaultSelfhostedRetryClock: SelfhostedRetryClock;