create-windy 0.8.3 → 0.10.0

package/template/apps/agent-server/native/socket-lifecycle.test.ts

@@ -0,0 +1,48 @@
+import { test, expect } from "bun:test";
+import { EventEmitter } from "node:events";
+import type { Server, IncomingMessage } from "node:http";
+import { PassThrough } from "node:stream";
+import { attachWorkbenchSockets } from "./socket.js";
+import type { WorkbenchManager } from "./manager.js";
+
+test("connection reset while worker admission is pending does not crash coordinator", async () => {
+  const server = new EventEmitter();
+  let release!: () => void;
+  const pending = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+  const manager = {
+    authorizeAuthority: async () => true,
+    get: async () => {
+      await pending;
+      throw Error("client disconnected");
+    },
+  } as unknown as WorkbenchManager;
+  const detach = attachWorkbenchSockets(server as Server, manager, () => true);
+  const socket = new PassThrough();
+  const request = {
+    url: "/internal/workbench/api/remote.mux",
+    headers: {
+      "x-windy-actor-id": "fixture-user",
+      "x-windy-product-session": "s".repeat(43),
+      "x-windy-public-origin": "http://localhost:18746",
+    },
+  } as unknown as IncomingMessage;
+  try {
+    server.emit("upgrade", request, socket, Buffer.alloc(0));
+    expect(() =>
+      socket.emit(
+        "error",
+        Object.assign(Error("read ECONNRESET"), {
+          code: "ECONNRESET",
+        }),
+      ),
+    ).not.toThrow();
+    expect(socket.destroyed).toBe(true);
+  } finally {
+    release();
+    await pending;
+    socket.destroy();
+    detach();
+  }
+});

package/template/apps/agent-server/native/socket.ts

@@ -0,0 +1,130 @@
+import type { IncomingMessage, Server } from "node:http";
+import type { Duplex } from "node:stream";
+import WebSocket, { WebSocketServer } from "ws";
+import { authorityFrom } from "./config.js";
+import { WorkbenchManager } from "./manager.js";
+import { workerTransportSignal } from "./lifecycle.js";
+import { isWorkbenchAuthorizationUnavailable } from "./authorization.js";
+import { WorkbenchSessionConflict } from "./session-conflict.js";
+
+export function attachWorkbenchSockets(
+  server: Server,
+  manager: WorkbenchManager,
+  authenticated: (headers: Headers) => boolean,
+) {
+  const wss = new WebSocketServer({
+    noServer: true,
+    maxPayload: 32 * 1024 * 1024,
+  });
+  const upgrade = async (
+    request: IncomingMessage,
+    socket: Duplex,
+    head: Buffer,
+  ) => {
+    // A browser may disconnect during worker startup/admission, before ws owns
+    // the raw socket. Handle resets throughout that asynchronous window.
+    socket.on("error", () => socket.destroy());
+    let transportSignal: AbortSignal | undefined;
+    let pendingUpstream: WebSocket | undefined;
+    try {
+      const url = new URL(request.url ?? "/", "http://agent.internal");
+      if (url.pathname !== "/internal/workbench/api/remote.mux" || url.search)
+        throw new Error("ROUTE_DENIED");
+      const headers = new Headers();
+      for (const [key, value] of Object.entries(request.headers))
+        if (typeof value === "string") headers.set(key, value);
+      if (!authenticated(headers)) throw new Error("UNAUTHENTICATED");
+      const authority = authorityFrom(new Request(url, { headers }));
+      if (
+        !(await manager.authorizeAuthority(
+          authority,
+          new AbortController().signal,
+        ))
+      )
+        throw new Error("UNAUTHENTICATED");
+      const worker = await manager.get(authority);
+      if (!(await manager.current(worker)) || socket.destroyed)
+        throw new Error("UNAUTHENTICATED");
+      const signal = workerTransportSignal(worker);
+      transportSignal = signal;
+      signal.throwIfAborted();
+      const upstream = new WebSocket(
+        `ws://127.0.0.1:${worker.port}/api/remote.mux`,
+        {
+          headers: {
+            host: new URL(authority.publicOrigin).host,
+            origin: authority.publicOrigin,
+            cookie: worker.cookie,
+          },
+          maxPayload: 32 * 1024 * 1024,
+          handshakeTimeout: 5000,
+        },
+      );
+      pendingUpstream = upstream;
+      const abortUpstream = () => upstream.terminate();
+      signal.addEventListener("abort", abortUpstream, { once: true });
+      upstream.once("close", () =>
+        signal.removeEventListener("abort", abortUpstream),
+      );
+      await new Promise<void>((accept, reject) => {
+        upstream.once("open", accept);
+        upstream.once("error", reject);
+      });
+      signal.throwIfAborted();
+      if (socket.destroyed) throw new Error("CLIENT_DISCONNECTED");
+      wss.handleUpgrade(request, socket, head, (client) => {
+        const close = () => {
+          client.terminate();
+          upstream.terminate();
+        };
+        signal.addEventListener("abort", close, {
+          once: true,
+        });
+        client.on("message", (data, binary) => {
+          if (
+            !signal.aborted &&
+            upstream.readyState === WebSocket.OPEN &&
+            upstream.bufferedAmount < 32 * 1024 * 1024
+          )
+            upstream.send(data, { binary });
+          else close();
+        });
+        upstream.on("message", (data, binary) => {
+          if (
+            !signal.aborted &&
+            client.readyState === WebSocket.OPEN &&
+            client.bufferedAmount < 32 * 1024 * 1024
+          )
+            client.send(data, { binary });
+          else close();
+        });
+        client.once("error", close);
+        upstream.once("error", close);
+        client.once("close", () => {
+          signal.removeEventListener("abort", close);
+          upstream.terminate();
+        });
+        upstream.once("close", () =>
+          client.close(1001, "Agent connection ended"),
+        );
+      });
+    } catch (error) {
+      pendingUpstream?.terminate();
+      if (!socket.destroyed)
+        socket.end(
+          error instanceof WorkbenchSessionConflict
+            ? "HTTP/1.1 409 Conflict\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n"
+            : isWorkbenchAuthorizationUnavailable(error) ||
+                isWorkbenchAuthorizationUnavailable(transportSignal?.reason)
+              ? "HTTP/1.1 503 Service Unavailable\r\nRetry-After: 2\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n"
+              : "HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n",
+        );
+    }
+  };
+  server.on("upgrade", upgrade);
+  return () => {
+    server.off("upgrade", upgrade);
+    for (const client of wss.clients) client.terminate();
+    wss.close();
+  };
+}

package/template/apps/agent-server/native/storage-lifecycle.test.ts

@@ -0,0 +1,218 @@
+import { expect, test } from "bun:test";
+import { EventEmitter } from "node:events";
+import type { ChildProcess } from "node:child_process";
+import {
+  chmod,
+  mkdir,
+  mkdtemp,
+  rename,
+  rm,
+  symlink,
+  unlink,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { diskUsage } from "./storage.js";
+import { WorkbenchManager, type NativeWorker } from "./manager.js";
+import { actorKey } from "./config.js";
+
+async function temporary(run: (home: string) => Promise<void>) {
+  const home = await mkdtemp(join(tmpdir(), "windy-storage-life-"));
+  try {
+    await run(home);
+  } finally {
+    await chmod(home, 0o700).catch(() => {});
+    await rm(home, { recursive: true, force: true });
+  }
+}
+
+test("normal quota totals files, stops above budget and excludes symbolic links", async () => {
+  await temporary(async (home) => {
+    const outside = await mkdtemp(join(tmpdir(), "windy-storage-outside-"));
+    try {
+      await mkdir(join(home, "nested"));
+      await writeFile(join(home, "a"), "123");
+      await writeFile(join(home, "nested", "b"), "4567");
+      await writeFile(join(outside, "private"), "not-counted");
+      await symlink(outside, join(home, "external-directory"));
+      await symlink(join(outside, "private"), join(home, "external-file"));
+      expect(await diskUsage(home, 1000)).toBe(7);
+      expect(await diskUsage(home, 2)).toBeGreaterThan(2);
+    } finally {
+      await rm(outside, { recursive: true, force: true });
+    }
+  });
+});
+
+test("missing, non-directory and symlink roots fail closed", async () => {
+  await temporary(async (home) => {
+    await expect(diskUsage(join(home, "missing"), 1000)).rejects.toMatchObject({
+      code: "ENOENT",
+    });
+    await writeFile(join(home, "file"), "synthetic");
+    await expect(diskUsage(join(home, "file"), 1000)).rejects.toThrow();
+    await mkdir(join(home, "directory"));
+    await symlink(join(home, "directory"), join(home, "link"));
+    await expect(diskUsage(join(home, "link"), 1000)).rejects.toThrow();
+  });
+});
+
+test.skipIf(process.getuid?.() === 0)(
+  "permission errors are not mistaken for disappeared children",
+  async () => {
+    await temporary(async (home) => {
+      const child = join(home, "restricted");
+      await mkdir(child);
+      await chmod(child, 0);
+      try {
+        await expect(diskUsage(home, 1000)).rejects.toMatchObject({
+          code: "EACCES",
+        });
+      } finally {
+        await chmod(child, 0o700);
+      }
+    });
+  },
+);
+
+test("root replacement during a walk is rejected even when the replacement exists", async () => {
+  await temporary(async (parent) => {
+    const home = join(parent, "home"),
+      moved = join(parent, "original");
+    await mkdir(home);
+    await writeFile(join(home, "fixture"), "synthetic");
+    const captured = Promise.withResolvers<void>();
+    const replaced = Promise.withResolvers<void>();
+    const scan = diskUsage(home, 1_000_000, {
+      async afterRootCaptured() {
+        captured.resolve();
+        await replaced.promise;
+      },
+    });
+    const outcome = scan.then(
+      () => undefined,
+      (error: unknown) => error,
+    );
+    try {
+      await captured.promise;
+      await rename(home, moved);
+      await mkdir(home);
+    } finally {
+      replaced.resolve();
+    }
+    expect(await outcome).toMatchObject({
+      message: "DSH quota root changed during inspection",
+    });
+  });
+});
+
+test("real file churn no longer stops a worker during quota inspection", async () => {
+  await temporary(async (home) => {
+    await Promise.all(
+      Array.from({ length: 128 }, (_, index) =>
+        writeFile(join(home, `fixture-${index}`), "synthetic"),
+      ),
+    );
+    const manager = new WorkbenchManager({
+      gateOrigin: "http://synthetic.invalid",
+      authorize: async () => true,
+    });
+    const worker: NativeWorker = {
+      uid: 20000,
+      key: actorKey("synthetic-user"),
+      authority: {
+        actorId: "synthetic-user",
+        session: "synthetic",
+        publicOrigin: "http://synthetic.invalid",
+      },
+      token: "synthetic",
+      port: 0,
+      cookie: "",
+      home,
+      child: Object.assign(new EventEmitter(), {
+        exitCode: 0,
+        signalCode: null,
+      }) as unknown as ChildProcess,
+      controller: new AbortController(),
+      admissionController: new AbortController(),
+    };
+    manager.workers.set(worker.key, worker);
+    let running = true,
+      mutations = 0;
+    const churn = (async () => {
+      while (running) {
+        const original = join(home, `fixture-${mutations % 128}`);
+        const moving = join(home, `moving-${mutations % 128}`);
+        await rename(original, moving);
+        await unlink(moving);
+        await writeFile(original, "synthetic");
+        mutations++;
+      }
+    })();
+    try {
+      for (let index = 0; index < 100; index++) await manager.inspect(worker);
+      expect(mutations).toBeGreaterThan(0);
+      expect(worker.controller.signal.aborted).toBe(false);
+      expect(manager.workers.get(worker.key)).toBe(worker);
+      manager.config.workspaceBytes = 1;
+      await manager.inspect(worker);
+      expect(worker.controller.signal.aborted).toBe(true);
+      expect(manager.workers.has(worker.key)).toBe(false);
+    } finally {
+      running = false;
+      await churn;
+      await manager.close();
+    }
+  });
+  // 100 次真实目录扫描与并发文件变动属于 I/O 集成验证，保留全部轮次与断言。
+}, 30_000);
+
+test("missing root still stops a worker through the real inspection path", async () => {
+  await temporary(async (home) => {
+    const manager = new WorkbenchManager({
+      gateOrigin: "http://synthetic.invalid",
+      authorize: async () => true,
+    });
+    const worker: NativeWorker = {
+      uid: 20000,
+      key: actorKey("synthetic-user"),
+      authority: {
+        actorId: "synthetic-user",
+        session: "synthetic",
+        publicOrigin: "http://synthetic.invalid",
+      },
+      token: "synthetic",
+      port: 0,
+      cookie: "",
+      home: join(home, "missing"),
+      child: Object.assign(new EventEmitter(), {
+        exitCode: 0,
+        signalCode: null,
+      }) as unknown as ChildProcess,
+      controller: new AbortController(),
+      admissionController: new AbortController(),
+    };
+    manager.workers.set(worker.key, worker);
+    try {
+      await manager.inspect(worker);
+      expect(worker.controller.signal.aborted).toBe(true);
+      expect(manager.workers.has(worker.key)).toBe(false);
+    } finally {
+      await manager.close();
+    }
+  });
+});
+
+test("deep directory trees fail closed before exhausting file descriptors", async () => {
+  await temporary(async (home) => {
+    let directory = home;
+    for (let depth = 0; depth < 66; depth++) {
+      directory = join(directory, "nested");
+      await mkdir(directory);
+    }
+    await expect(diskUsage(home, 1000)).rejects.toThrow(
+      "directory depth exceeded",
+    );
+  });
+});

package/template/apps/agent-server/native/storage.ts

@@ -0,0 +1,188 @@
+import {
+  chown,
+  chmod,
+  lstat,
+  mkdir,
+  open,
+  readFile,
+  readdir,
+  writeFile,
+} from "node:fs/promises";
+import { constants } from "node:fs";
+import { dirname, join } from "node:path";
+
+export async function userStorage(root: string, key: string) {
+  if (process.platform !== "linux" || process.getuid?.() !== 0)
+    throw new Error(
+      "Managed multi-user DSH requires Linux credential isolation",
+    );
+  const parent = dirname(root);
+  if (root === parent || parent === "/")
+    throw new Error("DSH requires a dedicated Agent data directory");
+  await mkdir(parent, { recursive: true, mode: 0o711 });
+  await protectCoordinatorFiles(parent);
+  await chown(parent, 0, 0);
+  await chmod(parent, 0o711);
+  await mkdir(root, { recursive: true, mode: 0o711 });
+  await chown(root, 0, 0);
+  await chmod(root, 0o711);
+  const ledgerPath = join(root, "users.json");
+  let ledger: Record<string, number> = {};
+  try {
+    ledger = JSON.parse(await readFile(ledgerPath, "utf8"));
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+  }
+  if (
+    !Object.values(ledger).every(
+      (uid) => Number.isSafeInteger(uid) && uid >= 20000 && uid < 60000,
+    )
+  )
+    throw new Error("Invalid DSH user ledger");
+  const uid = ledger[key] ?? Math.max(19999, ...Object.values(ledger)) + 1;
+  if (uid >= 60000) throw new Error("DSH user namespace exhausted");
+  if (!ledger[key]) {
+    ledger[key] = uid;
+    await writeFile(ledgerPath, JSON.stringify(ledger), { mode: 0o600 });
+  }
+  const home = join(root, key);
+  const workspace = join(home, "workspace");
+  for (const path of [home, workspace]) {
+    await mkdir(path, { recursive: true, mode: 0o700 });
+    // Bind privileged mutations to this exact directory inode. A worker can
+    // rename its workspace but cannot redirect root chown through a symlink.
+    const handle = await open(
+      path,
+      constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW,
+    );
+    try {
+      if (!(await handle.stat()).isDirectory())
+        throw new Error("Invalid user directory");
+      await handle.chown(0, 0);
+      await handle.chmod(0o700);
+      await handle.chown(uid, uid);
+    } finally {
+      await handle.close();
+    }
+  }
+  return { uid, home, workspace };
+}
+
+/** Symlinks do not widen quota walks or disclose another actor's file tree. */
+export async function diskUsage(
+  path: string,
+  budget: number,
+  observation?: { afterRootCaptured(): Promise<void> },
+): Promise<number> {
+  const root = await lstat(path);
+  if (!root.isDirectory()) throw new Error("Invalid DSH quota root");
+  // Allows a test to replace this exact observed inode before the walk starts.
+  if (observation) await observation.afterRootCaptured();
+  let size = 0;
+  let entries = 0;
+  const sameDirectory = (value: Awaited<ReturnType<typeof lstat>>) =>
+    value.isDirectory() && value.dev === root.dev && value.ino === root.ino;
+  const assertRoot = async () => {
+    if (!sameDirectory(await lstat(path)))
+      throw new Error("DSH quota root changed during inspection");
+  };
+  const walk = async (
+    directory: string,
+    isRoot = false,
+    depth = 0,
+  ): Promise<void> => {
+    if (depth > 64) throw new Error("DSH quota directory depth exceeded");
+    let handle;
+    try {
+      handle = await open(
+        directory,
+        constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW,
+      );
+    } catch (error) {
+      if (
+        !isRoot &&
+        ["ENOENT", "ELOOP", "ENOTDIR"].includes(
+          (error as NodeJS.ErrnoException).code ?? "",
+        )
+      )
+        return;
+      throw error;
+    }
+    try {
+      const identity = await handle.stat();
+      if (isRoot && !sameDirectory(identity))
+        throw new Error("DSH quota root changed during inspection");
+      // Production Linux walks the opened inode even when its pathname is renamed.
+      // Other platforms retain path traversal with explicit inode checks.
+      const bound =
+        process.platform === "linux" ? `/proc/self/fd/${handle.fd}` : directory;
+      let items;
+      try {
+        items = await readdir(bound, { withFileTypes: true });
+      } catch (error) {
+        if (!isRoot && (error as NodeJS.ErrnoException).code === "ENOENT")
+          return;
+        throw error;
+      }
+      if (process.platform !== "linux") {
+        let current;
+        try {
+          current = await lstat(directory);
+        } catch (error) {
+          if (!isRoot && (error as NodeJS.ErrnoException).code === "ENOENT")
+            return;
+          throw error;
+        }
+        if (
+          !current.isDirectory() ||
+          current.dev !== identity.dev ||
+          current.ino !== identity.ino
+        ) {
+          if (!isRoot) return; // The inspected child inode is no longer at this path.
+          throw new Error("DSH quota directory changed during inspection");
+        }
+      }
+      for (const item of items) {
+        if (++entries > 100_000)
+          throw new Error("DSH file count budget exceeded");
+        const child = join(bound, item.name);
+        let stat;
+        try {
+          stat = await lstat(child);
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
+          throw error;
+        }
+        if (stat.isSymbolicLink()) continue;
+        if (stat.isDirectory()) await walk(child, false, depth + 1);
+        else size += stat.size;
+        if (size > budget) return;
+      }
+    } finally {
+      await handle.close();
+    }
+  };
+  await walk(path, true);
+  await assertRoot();
+  return size;
+}
+
+/** Old Agent volumes may contain non-root-owned 0644 databases; traversal must not expose them. */
+async function protectCoordinatorFiles(parent: string) {
+  for (const entry of await readdir(parent, { withFileTypes: true })) {
+    if (!entry.isFile()) continue;
+    const handle = await open(
+      join(parent, entry.name),
+      constants.O_RDONLY | constants.O_NOFOLLOW,
+    );
+    try {
+      const stat = await handle.stat();
+      if (stat.isFile()) {
+        await handle.chown(0, 0);
+        await handle.chmod(0o600);
+      }
+    } finally {
+      await handle.close();
+    }
+  }
+}

package/template/apps/agent-server/native/stream-authorization.test.ts

@@ -0,0 +1,89 @@
+import { expect, test } from "bun:test";
+import { EventEmitter } from "node:events";
+import type { ChildProcess } from "node:child_process";
+import { WorkbenchManager, type NativeWorker } from "./manager.js";
+import { WorkbenchAuthorizationUnavailable } from "./authorization.js";
+import { createBoundary } from "./boundary.js";
+
+test("unavailable admission aborts an active model stream and releases its capacity without killing the worker", async () => {
+  const oldGlobal = process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+  const oldNative = process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+  process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = "2";
+  process.env.AGENT_DSH_MAX_ACTIVE_RUNS = "1";
+  let available = true;
+  const manager = new WorkbenchManager({
+    gateOrigin: "http://fixture",
+    authorize: async () => {
+      if (!available) throw new WorkbenchAuthorizationUnavailable("network");
+      return true;
+    },
+  });
+  const worker: NativeWorker = {
+    uid: 20000,
+    key: "fixture",
+    authority: {
+      actorId: "fixture",
+      session: "s".repeat(43),
+      publicOrigin: "http://fixture",
+    },
+    token: "t".repeat(43),
+    port: 0,
+    cookie: "",
+    home: "",
+    child: Object.assign(new EventEmitter(), {
+      exitCode: 0,
+      signalCode: null,
+    }) as ChildProcess,
+    controller: new AbortController(),
+    admissionController: new AbortController(),
+  };
+  manager.workers.set(worker.key, worker);
+  let calls = 0,
+    cancelled = 0;
+  let streamSignal: AbortSignal | undefined;
+  const boundary = createBoundary({
+    manager,
+    completion: async (request) => {
+      calls++;
+      streamSignal = request.signal;
+      return calls === 1
+        ? new Response(
+            new ReadableStream({
+              cancel() {
+                cancelled++;
+              },
+            }),
+          )
+        : new Response("recovered");
+    },
+  });
+  const request = () =>
+    new Request("http://fixture/internal/boundary/v1/chat/completions", {
+      method: "POST",
+      headers: { authorization: `Bearer ${worker.token}` },
+      body: "{}",
+    });
+  try {
+    const first = await boundary(request());
+    available = false;
+    await expect(manager.current(worker)).rejects.toBeInstanceOf(
+      WorkbenchAuthorizationUnavailable,
+    );
+    expect(streamSignal?.aborted).toBe(true);
+    expect((await first.body!.getReader().read()).done).toBe(true);
+    expect(cancelled).toBe(1);
+    expect(worker.controller.signal.aborted).toBe(false);
+    available = true;
+    const resumed = await boundary(request());
+    expect(await resumed.text()).toBe("recovered");
+    expect(calls).toBe(2);
+    expect(manager.workers.get(worker.key)).toBe(worker);
+  } finally {
+    await manager.close();
+    if (oldGlobal === undefined)
+      delete process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+    else process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = oldGlobal;
+    if (oldNative === undefined) delete process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+    else process.env.AGENT_DSH_MAX_ACTIVE_RUNS = oldNative;
+  }
+});