@cortexkit/subc-client 0.4.1 → 0.8.0

package/src/provider.ts

@@ -7,6 +7,7 @@ import {
   type ReconnectBackoff,
   type RequestOptions,
   type RouteTarget,
+  type SubcCallErrorKind,
 } from "./client.js";
 import { ConnectionFileError, readConnectionFile, type ConnectionInfo } from "./connection-file.js";
 import {
@@ -33,6 +34,7 @@ import {
   SocketWriteNotQueuedError,
   SocketWriteQueuedError,
   SubcSocket,
+  writeBorrowed,
 } from "./socket.js";
 
 const DEFAULT_HANDSHAKE_TIMEOUT_MS = 10_000;
@@ -53,7 +55,6 @@ export type ObservabilityKind = "snapshot" | "stream";
 export type InternalTransport = "bulk";
 export type StorageKind = "sqlite";
 export type StorageScope = "project";
-export type LeaseScope = "project";
 export type HealthStatus = "ok" | "degraded" | "failing";
 
 export interface HealthReport {
@@ -69,7 +70,6 @@ export interface ManifestInput {
   trust_tier: TrustTier;
   provides: ProviderRoleInput[];
   consumes: ConsumerRoleInput[];
-  scheduled_tasks: ScheduledTaskInput[];
   bindings: BindingsInput;
 }
 
@@ -97,6 +97,7 @@ export type ProviderRoleInput =
       config_schema: unknown;
       observability: ObservabilitySurfaceInput[];
       identity_scope: IdentityScope[];
+      concurrency: Concurrency;
     }
   | {
       role: "internal_service";
@@ -133,31 +134,6 @@ export type ConsumerRoleInput =
   | { role: "llm_client"; via: string; auth: string }
   | { role: "service_client"; of: string[] };
 
-export interface ScheduledTaskInput {
-  task_id: string;
-  eligibility: TaskEligibilityInput;
-  lease_scope: LeaseScope;
-  renews_during_calls: boolean;
-  toolset: string[];
-  model_policy: ModelPolicyInput;
-  step_cap: number;
-  circuit_breaker: CircuitBreakerInput;
-}
-
-export interface TaskEligibilityInput {
-  cooldown: string;
-  window: string;
-}
-
-export interface ModelPolicyInput {
-  tier: string;
-  fallback_chain: string[];
-}
-
-export interface CircuitBreakerInput {
-  identical_failures: number;
-}
-
 export interface BindingsInput {
   storage: StorageBindingInput;
   vault_grants: VaultGrantInput[];
@@ -185,6 +161,7 @@ export interface ManagementSurfaceManifestOptions {
   moduleId: string;
   operations: Array<string | ManagementOperationInput>;
   moduleVersion?: string;
+  concurrency?: Concurrency;
 }
 
 /**
@@ -312,6 +289,13 @@ interface OpenedProviderConnection {
   ack: ModuleHelloAckBody;
 }
 
+interface ReconnectCycle {
+  readonly generation: number;
+  socket: SubcSocket | null;
+  socketDied: boolean;
+  superseded: boolean;
+}
+
 export interface ModuleHelloAckBody {
   negotiated_ver: number;
   subc_ops: string[];
@@ -367,6 +351,7 @@ export class SubcProviderError extends Error {
   constructor(
     message: string,
     readonly code?: string,
+    readonly kind: SubcCallErrorKind = "terminal",
   ) {
     super(message);
   }
@@ -391,10 +376,10 @@ export function managementSurfaceManifest(opts: ManagementSurfaceManifestOptions
         config_schema: { type: "object" },
         observability: [],
         identity_scope: [],
+        concurrency: opts.concurrency ?? "module_managed",
       },
     ],
     consumes: [],
-    scheduled_tasks: [],
     bindings: {
       storage: {
         kind: "sqlite",
@@ -435,10 +420,10 @@ export class SubcProvider {
   private nextCorr = 1n;
   private ingressEpochDropCount = 0;
   private readonly requestGate = new AsyncPermitPool(DEFAULT_PROVIDER_HANDLER_CAPACITY);
-  private reconnecting: Promise<void> | null = null;
+  private reconnecting: ReconnectCycle | null = null;
   private generation = 1;
   private connectionEpoch = 1;
-  private stateQueue: ProviderConnectionState[] = [];
+  private stateQueue: Array<{ event: ProviderConnectionState; generation?: number; reconnect?: ReconnectCycle }> = [];
   private drainingStateQueue = false;
   private restoredDebounceToken = 0;
 
@@ -538,22 +523,22 @@ export class SubcProvider {
 
   cancel(handle: RouteHandle, corr: bigint): void {
     this.assertLiveHandle(handle);
-    void this.sendOn(
+    this.sendOn(
       this.sock,
       this.generation,
       buildFrame(FrameType.Cancel, controlFlags(), handle.channel, handle.epoch, corr, new Uint8Array(0)),
-    );
+    ).catch(() => undefined);
   }
 
   closeRoute(handle: RouteHandle): void {
     this.assertLiveHandle(handle);
     this.liveRoutes.delete(handle.channel);
     this.abortHandle(handle);
-    void this.sendOn(
+    this.sendOn(
       this.sock,
       this.generation,
       buildFrame(FrameType.Goodbye, controlFlags(), handle.channel, handle.epoch, 0n, new Uint8Array(0)),
-    );
+    ).catch(() => undefined);
   }
 
   /** Read the connection file, authenticate as a client, register the manifest with HELLO, and serve frames. */
@@ -587,12 +572,16 @@ export class SubcProvider {
     await this.closed;
   }
 
-  private static async openConnection(opts: NormalizedSubcProviderConnectOptions): Promise<OpenedProviderConnection> {
+  private static async openConnection(
+    opts: NormalizedSubcProviderConnectOptions,
+    onSocket?: (sock: SubcSocket) => void,
+  ): Promise<OpenedProviderConnection> {
     const conn = await readConnectionFile(opts.connectionFile);
     const deadline = Date.now() + (opts.handshakeTimeoutMs ?? DEFAULT_HANDSHAKE_TIMEOUT_MS);
     const endpoint = conn.endpoints[0]!;
     const sock = await SubcSocket.connect(endpoint.host, endpoint.port, deadline);
     try {
+      onSocket?.(sock);
       await authenticateClient(sock, conn, deadline);
       await sendFrame(sock, buildHelloFrame(opts));
       const ack = await expectHelloAck(sock, deadline);
@@ -748,11 +737,32 @@ export class SubcProvider {
       throw new SubcProviderError(`unsupported module control request ${request.op ?? "<missing op>"}`);
     }
 
-    const tentative = createRouteHandle(
-      numberField(request.route_channel, "route_channel"),
-      numberField(request.epoch, "epoch"),
-      this.connectionToken,
-    );
+    const boundChannel = numberField(request.route_channel, "route_channel");
+    const boundEpoch = numberField(request.epoch, "epoch");
+    // Implicit-replace rule (wire spec 3.3.0): the daemon never rebinds a live
+    // channel, but its route-gone GOODBYE to modules is best-effort, so a bind can
+    // arrive for a channel this endpoint still believes installed. A strictly
+    // higher epoch proves the daemon freed the old binding: tear the stale install
+    // down locally and proceed. Equal or lower epoch is a protocol violation the
+    // daemon cannot produce: reject the bind.
+    const stale = this.liveRoutes.get(boundChannel);
+    if (stale) {
+      if (boundEpoch <= stale.epoch) {
+        await this.sendError(
+          frame,
+          "route_rejected",
+          `route.bind epoch ${boundEpoch} does not supersede installed epoch ${stale.epoch} on channel ${boundChannel}`,
+          controlFlags(),
+          sock,
+          generation,
+        );
+        return;
+      }
+      this.liveRoutes.delete(stale.channel);
+      this.abortHandle(stale);
+      await this.opts.onRouteGone?.(stale);
+    }
+    const tentative = createRouteHandle(boundChannel, boundEpoch, this.connectionToken);
     const bindRequest: RouteBindRequest = {
       handle: tentative,
       target: request.target as RouteTarget,
@@ -856,8 +866,15 @@ export class SubcProvider {
     const releasePermit = await (this.requestGate ?? new AsyncPermitPool(DEFAULT_PROVIDER_HANDLER_CAPACITY)).acquire();
     const dataFlags = buildFlags(false, Priority.Interactive, false);
     try {
+      if (controller.signal.aborted) {
+        await this.sendError(frame, "cancelled", "request cancelled", dataFlags, sock, generation);
+        return;
+      }
       const body = await this.opts.handler(handle, frame.body, context);
-      if (controller.signal.aborted) return;
+      if (controller.signal.aborted) {
+        await this.sendError(frame, "cancelled", "request cancelled", dataFlags, sock, generation);
+        return;
+      }
       this.assertLiveHandle(handle);
       if (body === undefined) {
         await this.sendOn(
@@ -891,7 +908,11 @@ export class SubcProvider {
         throw new SubcProviderError("provider handler must return a Uint8Array or void", "invalid_handler_response");
       }
     } catch (error) {
-      if (error instanceof StaleRouteHandleError || controller.signal.aborted) return;
+      if (error instanceof StaleRouteHandleError) return;
+      if (controller.signal.aborted) {
+        await this.sendError(frame, "cancelled", "request cancelled", dataFlags, sock, generation);
+        return;
+      }
       await this.sendError(
         frame,
         error instanceof SubcProviderError && error.code ? error.code : "handler_error",
@@ -956,6 +977,10 @@ export class SubcProvider {
     sock: SubcSocket,
     generation: number,
   ): Promise<void> {
+    if (frame.header.channel !== 0) {
+      const handle = this.liveRoutes.get(frame.header.channel);
+      if (!handle || handle.epoch !== frame.header.epoch || !this.isLiveHandle(handle)) return;
+    }
     await this.sendOn(
       sock,
       generation,
@@ -977,43 +1002,79 @@ export class SubcProvider {
   }
 
   private handleUnexpectedDrop(sock: SubcSocket, generation: number, cause: Error): void {
+    if (this.closeStarted || this.sock !== sock || this.generation !== generation) return;
     this.cancelRestoredDebounce();
     this.abortGeneration(generation);
     this.generation += 1;
     sock.close();
-    this.enqueueConnectionState({ state: "down", cause });
-    this.scheduleReconnectAfterDrop(cause);
+    this.scheduleReconnectAfterDrop(cause, this.generation, sock);
   }
 
-  private scheduleReconnectAfterDrop(trigger: unknown): void {
-    if (this.closeStarted || this.reconnecting) return;
-    const promise = this.reconnectWithRetry(trigger)
+  private scheduleReconnectAfterDrop(cause: Error, generation: number, droppedSocket: SubcSocket): void {
+    if (this.closeStarted) return;
+
+    const previous = this.reconnecting;
+    if (previous) {
+      if (!this.shouldSupersedeReconnect(previous, generation, droppedSocket)) return;
+      previous.superseded = true;
+      previous.socket?.close();
+    }
+
+    const cycle: ReconnectCycle = {
+      generation,
+      socket: null,
+      socketDied: false,
+      superseded: false,
+    };
+    this.reconnecting = cycle;
+    this.enqueueConnectionState({ state: "down", cause });
+    void this.reconnectWithRetry(cycle)
       .catch((err) => {
-        if (!this.closeStarted) this.failFatal(err instanceof Error ? err : new SubcProviderError(String(err)));
+        if (this.isCurrentReconnect(cycle) && !this.closeStarted) {
+          this.failFatal(err instanceof Error ? err : new SubcProviderError(String(err)));
+        }
       })
       .finally(() => {
-        if (this.reconnecting === promise) this.reconnecting = null;
+        if (this.isCurrentReconnect(cycle)) this.reconnecting = null;
       });
-    this.reconnecting = promise;
   }
 
-  private async reconnectWithRetry(_trigger: unknown): Promise<void> {
+  private async reconnectWithRetry(cycle: ReconnectCycle): Promise<void> {
     let attempt = 0;
     let delay = this.opts.reconnectBackoff.baseMs;
 
     for (;;) {
+      if (!this.isCurrentReconnect(cycle)) return;
       if (this.closeStarted) throw new SubcProviderError("provider closed");
+
+      cycle.socket = null;
+      cycle.socketDied = false;
       attempt += 1;
-      this.enqueueConnectionState({ state: "reconnecting", attempt });
+      this.enqueueReconnectState(cycle, attempt);
       try {
-        const opened = await SubcProvider.openConnection(this.opts);
-        if (this.closeStarted) {
+        const opened = await SubcProvider.openConnection(this.opts, (sock) => {
+          if (!this.isCurrentReconnect(cycle)) {
+            sock.close();
+            return;
+          }
+          cycle.socket = sock;
+        });
+        if (!this.isCurrentReconnect(cycle) || this.closeStarted) {
           opened.sock.close();
-          throw new SubcProviderError("provider closed");
+          if (this.closeStarted) throw new SubcProviderError("provider closed");
+          return;
+        }
+
+        const epoch = this.replaceConnection(opened, cycle.generation);
+        this.reconnecting = null;
+        if (this.reconnecting !== null) {
+          throw new SubcProviderError("reconnect state must clear before restored", "reconnect_state");
         }
-        this.replaceConnection(opened);
+        this.scheduleRestored(cycle.generation, epoch);
         return;
       } catch (err) {
+        if (!this.isCurrentReconnect(cycle)) return;
+        if (cycle.socket) cycle.socketDied = true;
         if (this.closeStarted) throw err;
         if (!isProviderReconnectTransient(err)) throw err;
         // Providers retry transient failures indefinitely by design (a module
@@ -1026,7 +1087,20 @@ export class SubcProvider {
     }
   }
 
-    private replaceConnection(opened: OpenedProviderConnection): void {
+  private isCurrentReconnect(cycle: ReconnectCycle): boolean {
+    return !cycle.superseded && this.reconnecting === cycle && this.generation === cycle.generation;
+  }
+
+  private shouldSupersedeReconnect(cycle: ReconnectCycle, generation: number, droppedSocket: SubcSocket): boolean {
+    return cycle.generation < generation || cycle.socketDied || cycle.socket === droppedSocket;
+  }
+
+  private enqueueReconnectState(cycle: ReconnectCycle, attempt: number): void {
+    if (!this.isCurrentReconnect(cycle)) return;
+    this.enqueueConnectionState({ state: "reconnecting", attempt }, cycle.generation, cycle);
+  }
+
+  private replaceConnection(opened: OpenedProviderConnection, generation: number): number {
     this.sock.close();
     this.sock = opened.sock;
     this.currentConn = opened.conn;
@@ -1036,9 +1110,8 @@ export class SubcProvider {
     this.connectionToken = newConnectionToken();
     this.liveRoutes.clear();
     this.nextCorr = 1n;
-    const generation = this.generation;
     void this.readLoop(opened.sock, generation);
-    this.scheduleRestored(generation, this.connectionEpoch);
+    return this.connectionEpoch;
   }
 
   private scheduleRestored(generation: number, epoch: number): void {
@@ -1054,7 +1127,7 @@ export class SubcProvider {
           this.generation === generation &&
           this.connectionEpoch === epoch
         ) {
-          this.enqueueConnectionState({ state: "restored", epoch });
+          this.enqueueConnectionState({ state: "restored", epoch }, generation);
         }
       })
       .catch((err) => {
@@ -1068,9 +1141,9 @@ export class SubcProvider {
     this.restoredDebounceToken += 1;
   }
 
-  private enqueueConnectionState(event: ProviderConnectionState): void {
+  private enqueueConnectionState(event: ProviderConnectionState, generation?: number, reconnect?: ReconnectCycle): void {
     if (!this.opts.onConnectionState) return;
-    this.stateQueue.push(event);
+    this.stateQueue.push({ event, generation, reconnect });
     if (!this.drainingStateQueue) void this.drainConnectionStateQueue();
   }
 
@@ -1079,7 +1152,17 @@ export class SubcProvider {
     this.drainingStateQueue = true;
     try {
       while (this.stateQueue.length > 0) {
-        const event = this.stateQueue[0]!;
+        const queued = this.stateQueue[0]!;
+        if (
+          this.closeStarted ||
+          (queued.generation !== undefined && queued.generation !== this.generation) ||
+          queued.reconnect?.superseded
+        ) {
+          this.stateQueue.shift();
+          continue;
+        }
+
+        const { event } = queued;
         try {
           await this.opts.onConnectionState?.(event);
           this.stateQueue.shift();
@@ -1140,7 +1223,11 @@ function routeKey(handle: RouteHandle, corr: bigint): string {
 function providerErrorFromFrame(frame: Frame): SubcProviderError {
   try {
     const body = parseJson(frame.body) as { code?: string; message?: string };
-    return new SubcProviderError(body.message ?? "subc error", body.code);
+    return new SubcProviderError(
+      body.message ?? "subc error",
+      body.code,
+      body.code === "stale_route_epoch" || body.code === "unknown_channel" ? "not_sent" : "terminal",
+    );
   } catch {
     return new SubcProviderError(Buffer.from(frame.body).toString("utf8") || "subc error");
   }
@@ -1234,7 +1321,7 @@ function controlFlags(): number {
 }
 
 async function sendFrame(sock: SubcSocket, frame: Frame): Promise<void> {
-  await sock.write(encodeFrame(frame), Date.now() + WRITE_TIMEOUT_MS);
+  await writeBorrowed(sock, encodeFrame(frame), Date.now() + WRITE_TIMEOUT_MS);
 }
 
 async function expectHelloAck(sock: SubcSocket, deadline: number): Promise<ModuleHelloAckBody> {
@@ -1297,7 +1384,6 @@ function normalizeManifest(manifest: ManifestInput): ManifestInput {
     trust_tier: manifest.trust_tier,
     provides: manifest.provides.map(normalizeProviderRole),
     consumes: manifest.consumes.map(normalizeConsumerRole),
-    scheduled_tasks: manifest.scheduled_tasks.map(normalizeScheduledTask),
     bindings: {
       storage: {
         kind: manifest.bindings.storage.kind,
@@ -1358,6 +1444,7 @@ function normalizeProviderRole(role: ProviderRoleInput): ProviderRoleInput {
           kind: surface.kind,
         })),
         identity_scope: [...role.identity_scope],
+        concurrency: role.concurrency,
       };
     case "internal_service":
       return {
@@ -1381,23 +1468,3 @@ function normalizeConsumerRole(role: ConsumerRoleInput): ConsumerRoleInput {
   }
 }
 
-function normalizeScheduledTask(task: ScheduledTaskInput): ScheduledTaskInput {
-  return {
-    task_id: task.task_id,
-    eligibility: {
-      cooldown: task.eligibility.cooldown,
-      window: task.eligibility.window,
-    },
-    lease_scope: task.lease_scope,
-    renews_during_calls: task.renews_during_calls,
-    toolset: [...task.toolset],
-    model_policy: {
-      tier: task.model_policy.tier,
-      fallback_chain: [...task.model_policy.fallback_chain],
-    },
-    step_cap: task.step_cap,
-    circuit_breaker: {
-      identical_failures: task.circuit_breaker.identical_failures,
-    },
-  };
-}

package/src/socket.ts

@@ -44,6 +44,28 @@ export interface SocketWriteResult {
   completed: Promise<void>;
 }
 
+export function toWriteBuffer(bytes: Uint8Array): Buffer {
+  // Internal frame writes pass encodeFrame's fresh, single-use Uint8Array.
+  // Authentication writes likewise use fresh prefix/JSON storage and await each
+  // write without mutating it, so this view remains stable while Node drains it.
+  return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+}
+
+const WRITE_BORROWED = Symbol("subc.socket.writeBorrowed");
+const WRITE_TRACKED_BORROWED = Symbol("subc.socket.writeTrackedBorrowed");
+
+/** @internal Write fresh, single-use internal storage without copying it. */
+export function writeBorrowed(socket: SubcSocket, bytes: Uint8Array, deadlineMs: number): Promise<void> {
+  const borrowed = socket[WRITE_BORROWED];
+  return borrowed ? borrowed.call(socket, bytes, deadlineMs) : socket.write(bytes, deadlineMs);
+}
+
+/** @internal Track a write of fresh, single-use internal storage without copying it. */
+export function writeTrackedBorrowed(socket: SubcSocket, bytes: Uint8Array, deadlineMs: number): SocketWriteResult {
+  const borrowed = socket[WRITE_TRACKED_BORROWED];
+  return borrowed ? borrowed.call(socket, bytes, deadlineMs) : socket.writeTracked(bytes, deadlineMs);
+}
+
 interface Waiter {
   need: number;
   resolve: (bytes: Uint8Array) => void;
@@ -174,8 +196,24 @@ export class SubcSocket {
   }
 
   async write(bytes: Uint8Array, deadlineMs: number): Promise<void> {
+    await this.writeBuffer(Buffer.from(bytes), deadlineMs);
+  }
+
+  writeTracked(bytes: Uint8Array, deadlineMs: number): SocketWriteResult {
+    return this.writeTrackedBuffer(Buffer.from(bytes), deadlineMs);
+  }
+
+  async [WRITE_BORROWED](bytes: Uint8Array, deadlineMs: number): Promise<void> {
+    await this.writeBuffer(toWriteBuffer(bytes), deadlineMs);
+  }
+
+  [WRITE_TRACKED_BORROWED](bytes: Uint8Array, deadlineMs: number): SocketWriteResult {
+    return this.writeTrackedBuffer(toWriteBuffer(bytes), deadlineMs);
+  }
+
+  private async writeBuffer(buffer: Buffer, deadlineMs: number): Promise<void> {
     try {
-      await this.writeTracked(bytes, deadlineMs).completed;
+      await this.writeTrackedBuffer(buffer, deadlineMs).completed;
     } catch (err) {
       if (err instanceof SocketWriteNotQueuedError || err instanceof SocketWriteQueuedError) {
         throw err.cause ?? err;
@@ -184,7 +222,7 @@ export class SubcSocket {
     }
   }
 
-  writeTracked(bytes: Uint8Array, deadlineMs: number): SocketWriteResult {
+  private writeTrackedBuffer(buffer: Buffer, deadlineMs: number): SocketWriteResult {
     if (this.closedErr) {
       return {
         queued: false,
@@ -230,7 +268,7 @@ export class SubcSocket {
       }, remaining);
 
       try {
-        this.sock.write(Buffer.from(bytes), (err) => {
+        this.sock.write(buffer, (err) => {
           settle(() => {
             if (err) {
               reject(