create-windy 0.8.3 → 0.10.0

package/template/apps/agent-server/native/lifecycle.test.ts

@@ -0,0 +1,285 @@
+import { expect, test } from "bun:test";
+import { EventEmitter } from "node:events";
+import type { ChildProcess } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { WorkbenchManager, type NativeWorker } from "./manager.js";
+import { actorKey } from "./config.js";
+import { workerTransportSignal } from "./lifecycle.js";
+import {
+  authorizeWorkbench,
+  WorkbenchAuthorizationUnavailable,
+} from "./authorization.js";
+import { createBoundary } from "./boundary.js";
+import { workbenchHttp } from "./http.js";
+
+async function fixtureNativeWorker(authorize: () => Promise<boolean>) {
+  const home = await mkdtemp(join(tmpdir(), "windy-lifecycle-"));
+  const manager = new WorkbenchManager({
+    gateOrigin: "http://synthetic.invalid",
+    authorize,
+  });
+  const authority = {
+    actorId: "synthetic-user",
+    session: "s".repeat(43),
+    publicOrigin: "http://synthetic.invalid",
+  };
+  const child = Object.assign(new EventEmitter(), {
+    exitCode: 0,
+    signalCode: null,
+  }) as unknown as ChildProcess;
+  const worker: NativeWorker = {
+    uid: 20000,
+    key: actorKey(authority.actorId),
+    authority,
+    token: "t".repeat(43),
+    port: 0,
+    cookie: "synthetic=1",
+    child,
+    home,
+    controller: new AbortController(),
+    admissionController: new AbortController(),
+  };
+  manager.workers.set(worker.key, worker);
+  return {
+    manager,
+    worker,
+    async close() {
+      await manager.close();
+      await rm(home, { recursive: true, force: true });
+    },
+  };
+}
+
+function boundaryRequest(worker: NativeWorker, path = "/v1/chat/completions") {
+  return new Request(`http://synthetic.invalid/internal/boundary${path}`, {
+    method: "POST",
+    headers: { authorization: `Bearer ${worker.token}` },
+    body: "{}",
+  });
+}
+
+function httpRequest(worker: NativeWorker) {
+  return new Request("http://synthetic.invalid/internal/workbench/", {
+    headers: {
+      "x-windy-actor-id": worker.authority.actorId,
+      "x-windy-product-session": worker.authority.session,
+      "x-windy-public-origin": worker.authority.publicOrigin,
+    },
+  });
+}
+
+for (const scenario of ["503", "network"] as const) {
+  test(`${scenario} preserves worker but invalidates transports until fresh recovery`, async () => {
+    let healthy = false;
+    const fixture = await fixtureNativeWorker(() =>
+      authorizeWorkbench(
+        fixture.worker.authority,
+        fixture.worker.controller.signal,
+        {
+          productOrigin: "http://synthetic-product.invalid",
+          fetcher: async () => {
+            if (healthy)
+              return Response.json({
+                actor: { id: fixture.worker.authority.actorId, type: "user" },
+              });
+            if (scenario === "network")
+              throw new TypeError("synthetic network failure");
+            return new Response(null, { status: 503 });
+          },
+        },
+      ),
+    );
+    try {
+      const { manager, worker } = fixture;
+      const oldTransport = workerTransportSignal(worker);
+      await expect(manager.current(worker)).rejects.toBeInstanceOf(
+        WorkbenchAuthorizationUnavailable,
+      );
+      expect(manager.workers.get(worker.key)).toBe(worker);
+      expect(worker.controller.signal.aborted).toBe(false);
+      expect(worker.admissionController.signal.aborted).toBe(true);
+      expect(oldTransport.aborted).toBe(true);
+      await manager.inspect(worker);
+      expect(worker.controller.signal.aborted).toBe(false);
+      healthy = true;
+      expect(await manager.current(worker)).toBe(true);
+      expect(await manager.get(worker.authority)).toBe(worker);
+      expect(workerTransportSignal(worker).aborted).toBe(false);
+      expect(oldTransport.aborted).toBe(true);
+    } finally {
+      await fixture.close();
+    }
+  });
+}
+
+for (const status of [401, 403]) {
+  test(`confirmed ${status} denial stops and removes the worker`, async () => {
+    const fixture = await fixtureNativeWorker(() =>
+      authorizeWorkbench(
+        fixture.worker.authority,
+        fixture.worker.controller.signal,
+        {
+          productOrigin: "http://synthetic-product.invalid",
+          fetcher: async () => new Response(null, { status }),
+        },
+      ),
+    );
+    try {
+      expect(await fixture.manager.current(fixture.worker)).toBe(false);
+      expect(fixture.worker.controller.signal.aborted).toBe(true);
+      expect(fixture.worker.admissionController.signal.aborted).toBe(true);
+      expect(fixture.manager.workers.has(fixture.worker.key)).toBe(false);
+    } finally {
+      await fixture.close();
+    }
+  });
+}
+
+test("concurrent current and inspect share in-flight authorization, never cached success", async () => {
+  let calls = 0;
+  const gate = Promise.withResolvers<boolean>();
+  const fixture = await fixtureNativeWorker(async () => {
+    calls++;
+    return calls === 1 ? gate.promise : true;
+  });
+  try {
+    const first = fixture.manager.current(fixture.worker);
+    const second = fixture.manager.current(fixture.worker);
+    const poll = fixture.manager.inspect(fixture.worker);
+    const duplicatePoll = fixture.manager.inspect(fixture.worker);
+    expect(calls).toBe(1);
+    gate.resolve(true);
+    expect(await first).toBe(true);
+    expect(await second).toBe(true);
+    await Promise.all([poll, duplicatePoll]);
+    expect(calls).toBe(1);
+    expect(await fixture.manager.current(fixture.worker)).toBe(true);
+    expect(calls).toBe(2);
+  } finally {
+    await fixture.close();
+  }
+});
+
+test("quota is enforced even when authorization is temporarily unavailable", async () => {
+  const fixture = await fixtureNativeWorker(async () => {
+    throw new WorkbenchAuthorizationUnavailable("network");
+  });
+  try {
+    fixture.manager.config.workspaceBytes = 1;
+    await writeFile(
+      join(fixture.worker.home, "synthetic-quota.txt"),
+      "over-quota",
+    );
+    await fixture.manager.inspect(fixture.worker);
+    expect(fixture.worker.controller.signal.aborted).toBe(true);
+    expect(fixture.manager.workers.has(fixture.worker.key)).toBe(false);
+  } finally {
+    await fixture.close();
+  }
+});
+
+test("HTTP, MCP and model requests return 503 without forwarding on initial unavailable admission", async () => {
+  const fixture = await fixtureNativeWorker(async () => {
+    throw new WorkbenchAuthorizationUnavailable("upstream_status", 503);
+  });
+  const originalFetch = globalThis.fetch;
+  let forwarded = 0,
+    completions = 0;
+  // Every admission is injected above; any global fetch here would be a business forward.
+  globalThis.fetch = Object.assign(async () => {
+    forwarded++;
+    return new Response("unexpected");
+  }, originalFetch);
+  try {
+    const boundary = createBoundary({
+      manager: fixture.manager,
+      completion: async () => {
+        completions++;
+        return new Response("unexpected");
+      },
+    });
+    const responses = [
+      await workbenchHttp(httpRequest(fixture.worker), fixture.manager),
+      await boundary(boundaryRequest(fixture.worker, "/mcp")),
+      await boundary(boundaryRequest(fixture.worker)),
+    ];
+    for (const response of responses) {
+      expect(response.status).toBe(503);
+      expect(response.headers.get("retry-after")).toBe("2");
+    }
+    expect(forwarded).toBe(0);
+    expect(completions).toBe(0);
+    expect(fixture.worker.controller.signal.aborted).toBe(false);
+  } finally {
+    globalThis.fetch = originalFetch;
+    await fixture.close();
+  }
+});
+
+test("HTTP second authorization gate returns 503 before native transport", async () => {
+  let calls = 0;
+  const fixture = await fixtureNativeWorker(async () => {
+    if (++calls === 1) return true;
+    throw new WorkbenchAuthorizationUnavailable("network");
+  });
+  try {
+    expect(
+      (await workbenchHttp(httpRequest(fixture.worker), fixture.manager))
+        .status,
+    ).toBe(503);
+    expect(calls).toBe(2);
+    expect(fixture.worker.controller.signal.aborted).toBe(false);
+  } finally {
+    await fixture.close();
+  }
+});
+
+test("queued model admission fails closed with 503 after temporary loss, without second completion", async () => {
+  const oldGlobal = process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+  const oldNative = process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+  let healthy = true,
+    calls = 0,
+    completions = 0;
+  const queuedAdmission = Promise.withResolvers<void>();
+  const fixture = await fixtureNativeWorker(async () => {
+    calls++;
+    if (calls === 3) queuedAdmission.resolve();
+    if (!healthy)
+      throw new WorkbenchAuthorizationUnavailable("upstream_status", 503);
+    return true;
+  });
+  let first: Response | undefined;
+  try {
+    process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = "2";
+    process.env.AGENT_DSH_MAX_ACTIVE_RUNS = "2";
+    const boundary = createBoundary({
+      manager: fixture.manager,
+      completion: async () => {
+        completions++;
+        return new Response(new ReadableStream({ start() {} }));
+      },
+    });
+    first = await boundary(boundaryRequest(fixture.worker));
+    const queued = boundary(boundaryRequest(fixture.worker));
+    await queuedAdmission.promise;
+    // Allow initial admission's promise to settle and the request to join the FIFO.
+    await Bun.sleep(0);
+    healthy = false;
+    await first.body?.cancel();
+    expect((await queued).status).toBe(503);
+    expect(completions).toBe(1);
+    expect(calls).toBe(4);
+    expect(fixture.worker.controller.signal.aborted).toBe(false);
+    expect(fixture.worker.admissionController.signal.aborted).toBe(true);
+  } finally {
+    await first?.body?.cancel().catch(() => {});
+    if (oldGlobal === undefined)
+      delete process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+    else process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = oldGlobal;
+    if (oldNative === undefined) delete process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+    else process.env.AGENT_DSH_MAX_ACTIVE_RUNS = oldNative;
+    await fixture.close();
+  }
+});

package/template/apps/agent-server/native/lifecycle.ts

@@ -0,0 +1,136 @@
+import type { WorkbenchAuthority } from "./config.js";
+import type { NativeWorker } from "./manager.js";
+import { diskUsage } from "./storage.js";
+import { WorkbenchAuthorizationUnavailable } from "./authorization.js";
+
+export type WorkerStopReason =
+  | "authorization_denied"
+  | "session_replaced"
+  | "service_shutdown"
+  | "process_exit"
+  | "startup_failed"
+  | "quota_exceeded"
+  | "storage_check_failed"
+  | "maintenance_failed"
+  | "explicit_stop";
+
+/** Only safe operational categories are logged; never identities, tokens or raw errors. */
+export function workerLifecycleEvent(
+  worker: NativeWorker,
+  event: string,
+  reason?: string,
+) {
+  console.info(
+    JSON.stringify({
+      level: "info",
+      timestamp: new Date().toISOString(),
+      component: "agent-native",
+      event,
+      retentionClass: "operational",
+      pid: worker.child.pid,
+      ...(reason ? { reason } : {}),
+    }),
+  );
+}
+
+export function workerTransportSignal(worker: NativeWorker): AbortSignal {
+  return AbortSignal.any([
+    worker.controller.signal,
+    worker.admissionController.signal,
+  ]);
+}
+
+/** Share only in-flight checks. Every later request obtains fresh authorization. */
+export class WorkerLifecycle {
+  private readonly checks = new WeakMap<NativeWorker, Promise<boolean>>();
+  private readonly polls = new WeakMap<NativeWorker, Promise<void>>();
+  constructor(
+    private readonly authorize: (
+      authority: WorkbenchAuthority,
+      signal: AbortSignal,
+    ) => Promise<boolean>,
+    private readonly stop: (
+      worker: NativeWorker,
+      reason: WorkerStopReason,
+    ) => Promise<void>,
+  ) {}
+
+  current(worker: NativeWorker): Promise<boolean> {
+    if (worker.controller.signal.aborted) return Promise.resolve(false);
+    const pending = this.checks.get(worker);
+    if (pending) return pending;
+    const check = this.check(worker);
+    this.checks.set(worker, check);
+    void check.then(
+      () => this.checks.delete(worker),
+      () => this.checks.delete(worker),
+    );
+    return check;
+  }
+
+  private async check(worker: NativeWorker): Promise<boolean> {
+    let allowed: boolean;
+    try {
+      allowed = await this.authorize(
+        worker.authority,
+        worker.controller.signal,
+      );
+    } catch (error) {
+      if (worker.controller.signal.aborted) return false;
+      const unavailable =
+        error instanceof WorkbenchAuthorizationUnavailable
+          ? error
+          : new WorkbenchAuthorizationUnavailable("network");
+      if (!worker.admissionController.signal.aborted) {
+        worker.admissionController.abort(unavailable);
+        workerLifecycleEvent(
+          worker,
+          "worker.authorization.unavailable",
+          unavailable.status
+            ? `${unavailable.reason}:${unavailable.status}`
+            : unavailable.reason,
+        );
+      }
+      throw unavailable;
+    }
+    if (worker.controller.signal.aborted) return false;
+    if (!allowed) {
+      await this.stop(worker, "authorization_denied");
+      return false;
+    }
+    if (worker.admissionController.signal.aborted) {
+      worker.admissionController = new AbortController();
+      workerLifecycleEvent(worker, "worker.authorization.recovered");
+    }
+    return true;
+  }
+
+  inspect(worker: NativeWorker, quota: number): Promise<void> {
+    const existing = this.polls.get(worker);
+    if (existing) return existing;
+    const pending = this.poll(worker, quota);
+    this.polls.set(worker, pending);
+    void pending.then(
+      () => this.polls.delete(worker),
+      () => this.polls.delete(worker),
+    );
+    return pending;
+  }
+
+  private async poll(worker: NativeWorker, quota: number) {
+    if (worker.controller.signal.aborted) return;
+    try {
+      if (!(await this.current(worker))) return;
+    } catch (error) {
+      if (!(error instanceof WorkbenchAuthorizationUnavailable)) throw error;
+      // Admission failure closes transports, but quota remains independently enforced.
+    }
+    if (worker.controller.signal.aborted) return;
+    try {
+      if ((await diskUsage(worker.home, quota)) > quota)
+        await this.stop(worker, "quota_exceeded");
+    } catch {
+      await this.stop(worker, "storage_check_failed");
+    }
+  }
+}

package/template/apps/agent-server/native/manager.ts

@@ -0,0 +1,292 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { randomBytes, timingSafeEqual } from "node:crypto";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  actorKey,
+  type WorkbenchAuthority,
+  workbenchConfig,
+} from "./config.js";
+import { diskUsage, userStorage } from "./storage.js";
+import { nativeRequest } from "./request.js";
+import { WorkbenchSessionConflict } from "./session-conflict.js";
+import {
+  WorkerLifecycle,
+  workerLifecycleEvent,
+  type WorkerStopReason,
+} from "./lifecycle.js";
+
+export interface NativeWorker {
+  key: string;
+  uid: number;
+  authority: WorkbenchAuthority;
+  token: string;
+  port: number;
+  cookie: string;
+  child: ChildProcess;
+  controller: AbortController;
+  admissionController: AbortController;
+  home: string;
+  stopped?: Promise<void>;
+}
+
+export class WorkbenchManager {
+  readonly workers = new Map<string, NativeWorker>();
+  private pending = Promise.resolve();
+  readonly config = workbenchConfig();
+  private readonly lifecycle: WorkerLifecycle;
+  constructor(
+    private readonly options: {
+      gateOrigin: string;
+      authorize: (
+        authority: WorkbenchAuthority,
+        signal: AbortSignal,
+      ) => Promise<boolean>;
+    },
+  ) {
+    this.lifecycle = new WorkerLifecycle(options.authorize, (worker, reason) =>
+      this.stop(worker, reason),
+    );
+  }
+
+  async get(authority: WorkbenchAuthority): Promise<NativeWorker> {
+    let found!: NativeWorker;
+    const task = this.pending.then(async () => {
+      const key = actorKey(authority.actorId);
+      const existing = this.workers.get(key);
+      if (
+        existing &&
+        !existing.controller.signal.aborted &&
+        existing.authority.session === authority.session &&
+        existing.authority.publicOrigin === authority.publicOrigin
+      ) {
+        found = existing;
+        return;
+      }
+      if (existing && !existing.controller.signal.aborted)
+        throw new WorkbenchSessionConflict();
+      if (existing) await this.stop(existing, "session_replaced");
+      if (this.workers.size >= this.config.maxUsers)
+        throw new Error("WORKBENCH_CAPACITY");
+      found = await this.start(key, authority);
+    });
+    this.pending = task.catch(() => {});
+    await task;
+    return found;
+  }
+
+  byToken(token: string): NativeWorker | undefined {
+    return [...this.workers.values()].find(
+      (worker) =>
+        token.length === worker.token.length &&
+        timingSafeEqual(Buffer.from(token), Buffer.from(worker.token)),
+    );
+  }
+
+  authorizeAuthority(
+    authority: WorkbenchAuthority,
+    signal: AbortSignal,
+  ): Promise<boolean> {
+    const existing = this.workers.get(actorKey(authority.actorId));
+    if (
+      existing &&
+      existing.authority.session === authority.session &&
+      existing.authority.publicOrigin === authority.publicOrigin
+    )
+      return this.current(existing);
+    return this.options.authorize(authority, signal);
+  }
+
+  current(worker: NativeWorker): Promise<boolean> {
+    return this.lifecycle.current(worker);
+  }
+
+  inspect(worker: NativeWorker): Promise<void> {
+    return this.lifecycle.inspect(worker, this.config.workspaceBytes);
+  }
+
+  stop(
+    worker: NativeWorker,
+    reason: WorkerStopReason = "explicit_stop",
+  ): Promise<void> {
+    if (worker.stopped) return worker.stopped;
+    worker.controller.abort();
+    worker.admissionController.abort();
+    workerLifecycleEvent(worker, "worker.stopped", reason);
+    if (this.workers.get(worker.key) === worker)
+      this.workers.delete(worker.key);
+    worker.stopped = new Promise<void>((accept) => {
+      const kill = () => {
+        try {
+          if (worker.child.pid) process.kill(-worker.child.pid, "SIGKILL");
+        } catch {
+          /* The process group may already have exited. */
+        }
+      };
+      if (worker.child.exitCode !== null || worker.child.signalCode !== null) {
+        kill();
+        accept();
+        return;
+      }
+      const timer = setTimeout(() => {
+        kill();
+        accept();
+      }, 3000);
+      timer.unref();
+      worker.child.once("exit", () => {
+        clearTimeout(timer);
+        kill();
+        accept();
+      });
+      try {
+        if (worker.child.pid) process.kill(-worker.child.pid, "SIGTERM");
+      } catch {
+        clearTimeout(timer);
+        accept();
+      }
+    });
+    return worker.stopped;
+  }
+
+  async close() {
+    await Promise.all(
+      [...this.workers.values()].map((worker) =>
+        this.stop(worker, "service_shutdown"),
+      ),
+    );
+  }
+
+  private async start(
+    key: string,
+    authority: WorkbenchAuthority,
+  ): Promise<NativeWorker> {
+    const { uid, home, workspace } = await userStorage(this.config.root, key);
+    if (
+      (await diskUsage(home, this.config.workspaceBytes)) >
+      this.config.workspaceBytes
+    )
+      throw new Error("WORKBENCH_QUOTA");
+    const token = randomBytes(32).toString("base64url");
+    const appRoot =
+      process.env.WINDY_APP_ROOT ??
+      resolve(dirname(fileURLToPath(import.meta.url)), "..");
+    const entry = resolve(appRoot, "dist/native/worker.js");
+    const env = {
+      PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin",
+      LANG: "C.UTF-8",
+      TZ: "Asia/Shanghai",
+      HOME: home,
+      DSH_HOME: home,
+      DSH_MODEL: this.config.model,
+      WINDY_USER_EXTENSIONS: this.config.userExtensions ? "1" : "0",
+      DSH_CONTEXT_WINDOW: String(this.config.contextWindow),
+      DSH_MAX_OUTPUT_TOKENS: String(this.config.maxOutputTokens),
+      DSH_PERMISSION_MODE: "workspace-write",
+      DSH_TELEMETRY_DISABLED: "1",
+      WINDY_APP_ROOT: appRoot,
+      WINDY_GATE_URL: `${this.options.gateOrigin}/internal/boundary`,
+      WINDY_WORKER_TOKEN: token,
+      WINDY_PUBLIC_URL: `${authority.publicOrigin}/agent/`,
+      WINDY_WORKSPACE_ROOT: workspace,
+      PYTHONUTF8: "1",
+      PYTHONDONTWRITEBYTECODE: "1",
+      PIP_NO_INDEX: "1",
+      NARB_DISABLE_NATIVE_CACHE: "1",
+    };
+    const child = spawn(
+      "prlimit",
+      [
+        "--nproc=128:128",
+        "--nofile=512:512",
+        "--fsize=134217728:134217728",
+        process.execPath,
+        "--max-old-space-size=768",
+        entry,
+      ],
+      {
+        cwd: workspace,
+        env,
+        uid,
+        gid: uid,
+        detached: true,
+        stdio: ["ignore", "ignore", "ignore", "ipc"],
+      },
+    );
+    const worker: NativeWorker = {
+      key,
+      uid,
+      authority,
+      token,
+      port: 0,
+      cookie: "",
+      child,
+      controller: new AbortController(),
+      admissionController: new AbortController(),
+      home,
+    };
+    this.workers.set(key, worker);
+    child.once("exit", () => this.stop(worker, "process_exit"));
+    try {
+      await new Promise<void>((accept, reject) => {
+        const timer = setTimeout(
+          () => reject(new Error("WORKBENCH_STARTUP_TIMEOUT")),
+          45_000,
+        );
+        child.once("error", () => {
+          clearTimeout(timer);
+          reject(new Error("WORKBENCH_STARTUP_FAILED"));
+        });
+        child.once("exit", () => {
+          clearTimeout(timer);
+          reject(new Error("WORKBENCH_STARTUP_FAILED"));
+        });
+        child.on("message", (value) => {
+          const message = value as { type?: string; port?: number };
+          if (
+            message.type === "ready" &&
+            Number.isInteger(message.port) &&
+            message.port! > 0
+          ) {
+            worker.port = message.port!;
+            clearTimeout(timer);
+            accept();
+          } else if (message.type === "failed") {
+            clearTimeout(timer);
+            reject(new Error("WORKBENCH_STARTUP_FAILED"));
+          }
+        });
+      });
+      const bootstrap = await nativeRequest(
+        `http://127.0.0.1:${worker.port}/windy/internal-bootstrap`,
+        {
+          signal: AbortSignal.timeout(10_000),
+          headers: new Headers({
+            authorization: `Bearer ${token}`,
+            host: new URL(authority.publicOrigin).host,
+          }),
+        },
+      );
+      if (bootstrap.status !== 303)
+        throw new Error("WORKBENCH_BOOTSTRAP_FAILED");
+      worker.cookie = bootstrap.headers.get("set-cookie")?.split(";")[0] ?? "";
+      await bootstrap.body?.cancel();
+      if (!worker.cookie) throw new Error("WORKBENCH_BOOTSTRAP_FAILED");
+      workerLifecycleEvent(worker, "worker.started");
+      const timer = setInterval(() => {
+        void this.inspect(worker).catch(() =>
+          this.stop(worker, "maintenance_failed"),
+        );
+      }, 2000);
+      timer.unref();
+      worker.controller.signal.addEventListener(
+        "abort",
+        () => clearInterval(timer),
+        { once: true },
+      );
+      return worker;
+    } catch (error) {
+      await this.stop(worker, "startup_failed");
+      throw error;
+    }
+  }
+}