create-windy 0.8.3 → 0.9.1

package/template/apps/agent-server/native/manager.ts

@@ -0,0 +1,292 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { randomBytes, timingSafeEqual } from "node:crypto";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  actorKey,
+  type WorkbenchAuthority,
+  workbenchConfig,
+} from "./config.js";
+import { diskUsage, userStorage } from "./storage.js";
+import { nativeRequest } from "./request.js";
+import { WorkbenchSessionConflict } from "./session-conflict.js";
+import {
+  WorkerLifecycle,
+  workerLifecycleEvent,
+  type WorkerStopReason,
+} from "./lifecycle.js";
+
+export interface NativeWorker {
+  key: string;
+  uid: number;
+  authority: WorkbenchAuthority;
+  token: string;
+  port: number;
+  cookie: string;
+  child: ChildProcess;
+  controller: AbortController;
+  admissionController: AbortController;
+  home: string;
+  stopped?: Promise<void>;
+}
+
+export class WorkbenchManager {
+  readonly workers = new Map<string, NativeWorker>();
+  private pending = Promise.resolve();
+  readonly config = workbenchConfig();
+  private readonly lifecycle: WorkerLifecycle;
+  constructor(
+    private readonly options: {
+      gateOrigin: string;
+      authorize: (
+        authority: WorkbenchAuthority,
+        signal: AbortSignal,
+      ) => Promise<boolean>;
+    },
+  ) {
+    this.lifecycle = new WorkerLifecycle(options.authorize, (worker, reason) =>
+      this.stop(worker, reason),
+    );
+  }
+
+  async get(authority: WorkbenchAuthority): Promise<NativeWorker> {
+    let found!: NativeWorker;
+    const task = this.pending.then(async () => {
+      const key = actorKey(authority.actorId);
+      const existing = this.workers.get(key);
+      if (
+        existing &&
+        !existing.controller.signal.aborted &&
+        existing.authority.session === authority.session &&
+        existing.authority.publicOrigin === authority.publicOrigin
+      ) {
+        found = existing;
+        return;
+      }
+      if (existing && !existing.controller.signal.aborted)
+        throw new WorkbenchSessionConflict();
+      if (existing) await this.stop(existing, "session_replaced");
+      if (this.workers.size >= this.config.maxUsers)
+        throw new Error("WORKBENCH_CAPACITY");
+      found = await this.start(key, authority);
+    });
+    this.pending = task.catch(() => {});
+    await task;
+    return found;
+  }
+
+  byToken(token: string): NativeWorker | undefined {
+    return [...this.workers.values()].find(
+      (worker) =>
+        token.length === worker.token.length &&
+        timingSafeEqual(Buffer.from(token), Buffer.from(worker.token)),
+    );
+  }
+
+  authorizeAuthority(
+    authority: WorkbenchAuthority,
+    signal: AbortSignal,
+  ): Promise<boolean> {
+    const existing = this.workers.get(actorKey(authority.actorId));
+    if (
+      existing &&
+      existing.authority.session === authority.session &&
+      existing.authority.publicOrigin === authority.publicOrigin
+    )
+      return this.current(existing);
+    return this.options.authorize(authority, signal);
+  }
+
+  current(worker: NativeWorker): Promise<boolean> {
+    return this.lifecycle.current(worker);
+  }
+
+  inspect(worker: NativeWorker): Promise<void> {
+    return this.lifecycle.inspect(worker, this.config.workspaceBytes);
+  }
+
+  stop(
+    worker: NativeWorker,
+    reason: WorkerStopReason = "explicit_stop",
+  ): Promise<void> {
+    if (worker.stopped) return worker.stopped;
+    worker.controller.abort();
+    worker.admissionController.abort();
+    workerLifecycleEvent(worker, "worker.stopped", reason);
+    if (this.workers.get(worker.key) === worker)
+      this.workers.delete(worker.key);
+    worker.stopped = new Promise<void>((accept) => {
+      const kill = () => {
+        try {
+          if (worker.child.pid) process.kill(-worker.child.pid, "SIGKILL");
+        } catch {
+          /* The process group may already have exited. */
+        }
+      };
+      if (worker.child.exitCode !== null || worker.child.signalCode !== null) {
+        kill();
+        accept();
+        return;
+      }
+      const timer = setTimeout(() => {
+        kill();
+        accept();
+      }, 3000);
+      timer.unref();
+      worker.child.once("exit", () => {
+        clearTimeout(timer);
+        kill();
+        accept();
+      });
+      try {
+        if (worker.child.pid) process.kill(-worker.child.pid, "SIGTERM");
+      } catch {
+        clearTimeout(timer);
+        accept();
+      }
+    });
+    return worker.stopped;
+  }
+
+  async close() {
+    await Promise.all(
+      [...this.workers.values()].map((worker) =>
+        this.stop(worker, "service_shutdown"),
+      ),
+    );
+  }
+
+  private async start(
+    key: string,
+    authority: WorkbenchAuthority,
+  ): Promise<NativeWorker> {
+    const { uid, home, workspace } = await userStorage(this.config.root, key);
+    if (
+      (await diskUsage(home, this.config.workspaceBytes)) >
+      this.config.workspaceBytes
+    )
+      throw new Error("WORKBENCH_QUOTA");
+    const token = randomBytes(32).toString("base64url");
+    const appRoot =
+      process.env.WINDY_APP_ROOT ??
+      resolve(dirname(fileURLToPath(import.meta.url)), "..");
+    const entry = resolve(appRoot, "dist/native/worker.js");
+    const env = {
+      PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin",
+      LANG: "C.UTF-8",
+      TZ: "Asia/Shanghai",
+      HOME: home,
+      DSH_HOME: home,
+      DSH_MODEL: this.config.model,
+      WINDY_USER_EXTENSIONS: this.config.userExtensions ? "1" : "0",
+      DSH_CONTEXT_WINDOW: String(this.config.contextWindow),
+      DSH_MAX_OUTPUT_TOKENS: String(this.config.maxOutputTokens),
+      DSH_PERMISSION_MODE: "workspace-write",
+      DSH_TELEMETRY_DISABLED: "1",
+      WINDY_APP_ROOT: appRoot,
+      WINDY_GATE_URL: `${this.options.gateOrigin}/internal/boundary`,
+      WINDY_WORKER_TOKEN: token,
+      WINDY_PUBLIC_URL: `${authority.publicOrigin}/agent/`,
+      WINDY_WORKSPACE_ROOT: workspace,
+      PYTHONUTF8: "1",
+      PYTHONDONTWRITEBYTECODE: "1",
+      PIP_NO_INDEX: "1",
+      NARB_DISABLE_NATIVE_CACHE: "1",
+    };
+    const child = spawn(
+      "prlimit",
+      [
+        "--nproc=128:128",
+        "--nofile=512:512",
+        "--fsize=134217728:134217728",
+        process.execPath,
+        "--max-old-space-size=768",
+        entry,
+      ],
+      {
+        cwd: workspace,
+        env,
+        uid,
+        gid: uid,
+        detached: true,
+        stdio: ["ignore", "ignore", "ignore", "ipc"],
+      },
+    );
+    const worker: NativeWorker = {
+      key,
+      uid,
+      authority,
+      token,
+      port: 0,
+      cookie: "",
+      child,
+      controller: new AbortController(),
+      admissionController: new AbortController(),
+      home,
+    };
+    this.workers.set(key, worker);
+    child.once("exit", () => this.stop(worker, "process_exit"));
+    try {
+      await new Promise<void>((accept, reject) => {
+        const timer = setTimeout(
+          () => reject(new Error("WORKBENCH_STARTUP_TIMEOUT")),
+          45_000,
+        );
+        child.once("error", () => {
+          clearTimeout(timer);
+          reject(new Error("WORKBENCH_STARTUP_FAILED"));
+        });
+        child.once("exit", () => {
+          clearTimeout(timer);
+          reject(new Error("WORKBENCH_STARTUP_FAILED"));
+        });
+        child.on("message", (value) => {
+          const message = value as { type?: string; port?: number };
+          if (
+            message.type === "ready" &&
+            Number.isInteger(message.port) &&
+            message.port! > 0
+          ) {
+            worker.port = message.port!;
+            clearTimeout(timer);
+            accept();
+          } else if (message.type === "failed") {
+            clearTimeout(timer);
+            reject(new Error("WORKBENCH_STARTUP_FAILED"));
+          }
+        });
+      });
+      const bootstrap = await nativeRequest(
+        `http://127.0.0.1:${worker.port}/windy/internal-bootstrap`,
+        {
+          signal: AbortSignal.timeout(10_000),
+          headers: new Headers({
+            authorization: `Bearer ${token}`,
+            host: new URL(authority.publicOrigin).host,
+          }),
+        },
+      );
+      if (bootstrap.status !== 303)
+        throw new Error("WORKBENCH_BOOTSTRAP_FAILED");
+      worker.cookie = bootstrap.headers.get("set-cookie")?.split(";")[0] ?? "";
+      await bootstrap.body?.cancel();
+      if (!worker.cookie) throw new Error("WORKBENCH_BOOTSTRAP_FAILED");
+      workerLifecycleEvent(worker, "worker.started");
+      const timer = setInterval(() => {
+        void this.inspect(worker).catch(() =>
+          this.stop(worker, "maintenance_failed"),
+        );
+      }, 2000);
+      timer.unref();
+      worker.controller.signal.addEventListener(
+        "abort",
+        () => clearInterval(timer),
+        { once: true },
+      );
+      return worker;
+    } catch (error) {
+      await this.stop(worker, "startup_failed");
+      throw error;
+    }
+  }
+}

package/template/apps/agent-server/native/native.test.ts

@@ -0,0 +1,207 @@
+import { describe, expect, test } from "bun:test";
+import { createServer } from "node:http";
+import { mkdtemp, mkdir, symlink, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { actorKey, authorityFrom } from "./config.js";
+import { workerPath } from "./http.js";
+import { nativeRequest } from "./request.js";
+import { diskUsage } from "./storage.js";
+import { createBoundary } from "./boundary.js";
+import type { WorkbenchManager } from "./manager.js";
+
+describe("embedded DSH boundary", () => {
+  test("only server-bound login facts form an actor authority", () => {
+    const headers = {
+      "x-windy-actor-id": "actor-a",
+      "x-windy-product-session": "s".repeat(43),
+      "x-windy-public-origin": "https://windy.example",
+    };
+    expect(
+      authorityFrom(new Request("http://agent.internal/", { headers })).actorId,
+    ).toBe("actor-a");
+    for (const invalid of [
+      "https://attacker@windy.example",
+      "https://windy.example/path",
+      "file:///tmp",
+      "https://windy.example?x=1",
+    ])
+      expect(() =>
+        authorityFrom(
+          new Request("http://agent.internal/", {
+            headers: { ...headers, "x-windy-public-origin": invalid },
+          }),
+        ),
+      ).toThrow();
+    expect(() =>
+      authorityFrom(
+        new Request("http://agent.internal/", {
+          headers: { ...headers, "x-windy-actor-id": "../other-user" },
+        }),
+      ),
+    ).toThrow();
+    expect(actorKey("actor-a")).not.toBe(actorKey("actor-b"));
+  });
+
+  test("native private bootstrap and connect cannot be browser-proxied", () => {
+    for (const path of [
+      "/windy/internal-bootstrap",
+      "/windy/internal-other",
+      "/connect",
+      "/%2fescape",
+      "/%5cescape",
+    ])
+      expect(() =>
+        workerPath(`http://agent.internal/internal/workbench${path}`),
+      ).toThrow();
+    expect(
+      workerPath("http://agent.internal/internal/workbench/api/remote.mux"),
+    ).toBe("/api/remote.mux");
+  });
+
+  test("native transport preserves virtual Host and private Cookie audience", async () => {
+    const server = createServer((request, response) => {
+      response.writeHead(
+        request.headers.host === "windy.example" &&
+          request.headers.cookie === "private=native"
+          ? 200
+          : 401,
+      );
+      response.end("native");
+    });
+    await new Promise<void>((accept) => server.listen(0, "127.0.0.1", accept));
+    try {
+      const address = server.address();
+      if (!address || typeof address === "string")
+        throw new Error("Missing listener");
+      const response = await nativeRequest(
+        `http://127.0.0.1:${address.port}/`,
+        {
+          headers: new Headers({
+            host: "windy.example",
+            cookie: "private=native",
+          }),
+          signal: AbortSignal.timeout(3000),
+        },
+      );
+      expect(response.status).toBe(200);
+      expect(await response.text()).toBe("native");
+    } finally {
+      server.closeAllConnections();
+      await new Promise<void>((accept) => server.close(() => accept()));
+    }
+  });
+
+  test("quota walk counts generated files and never follows another user's tree", async () => {
+    const root = await mkdtemp(join(tmpdir(), "windy-user-quota-"));
+    try {
+      await mkdir(join(root, "own"));
+      await mkdir(join(root, "other"));
+      await writeFile(join(root, "own", "result"), "abc");
+      await writeFile(join(root, "other", "private"), "private-other-data");
+      await symlink(join(root, "other"), join(root, "own", "foreign"));
+      expect(await diskUsage(join(root, "own"), 100)).toBe(3);
+      expect(await diskUsage(join(root, "own"), 2)).toBeGreaterThan(2);
+    } finally {
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+});
+
+test("native capacity reserves a canonical slot and queues excess requests", async () => {
+  const oldGlobal = process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+  const oldNative = process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+  const worker = {
+    authority: { actorId: "actor-a" },
+    controller: new AbortController(),
+    admissionController: new AbortController(),
+  };
+  const manager = {
+    byToken: () => worker,
+    current: async () => true,
+  } as unknown as WorkbenchManager;
+  let calls = 0;
+  const completion = async () => {
+    calls++;
+    return new Response(new ReadableStream({ start() {} }));
+  };
+  const request = () =>
+    new Request("http://agent/internal/boundary/v1/chat/completions", {
+      method: "POST",
+      headers: { authorization: `Bearer ${"s".repeat(43)}` },
+      body: "{}",
+    });
+  try {
+    process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = "1";
+    process.env.AGENT_DSH_MAX_ACTIVE_RUNS = "1";
+    expect(
+      (await createBoundary({ manager, completion })(request())).status,
+    ).toBe(429);
+    expect(calls).toBe(0);
+    process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = "3";
+    process.env.AGENT_DSH_MAX_ACTIVE_RUNS = "32";
+    const boundary = createBoundary({ manager, completion });
+    const first = await boundary(request());
+    const second = await boundary(request());
+    const pending = boundary(request());
+    await new Promise((accept) => setTimeout(accept, 10));
+    expect(calls).toBe(2);
+    await first.body?.cancel();
+    const replacement = await pending;
+    expect(replacement.status).toBe(200);
+    await second.body?.cancel();
+    await replacement.body?.cancel();
+  } finally {
+    if (oldGlobal === undefined)
+      delete process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+    else process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = oldGlobal;
+    if (oldNative === undefined) delete process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+    else process.env.AGENT_DSH_MAX_ACTIVE_RUNS = oldNative;
+  }
+});
+
+test("queued native model requests reauthorize before dispatch", async () => {
+  const oldGlobal = process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+  const oldNative = process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+  let allowed = true,
+    calls = 0;
+  const worker = {
+    authority: { actorId: "actor-a" },
+    controller: new AbortController(),
+    admissionController: new AbortController(),
+  };
+  const manager = {
+    byToken: () => worker,
+    current: async () => allowed,
+  } as unknown as WorkbenchManager;
+  try {
+    process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = "2";
+    process.env.AGENT_DSH_MAX_ACTIVE_RUNS = "2";
+    const boundary = createBoundary({
+      manager,
+      completion: async () => {
+        calls++;
+        return new Response(new ReadableStream({ start() {} }));
+      },
+    });
+    const request = () =>
+      new Request("http://agent/internal/boundary/v1/chat/completions", {
+        method: "POST",
+        headers: { authorization: `Bearer ${"s".repeat(43)}` },
+        body: "{}",
+      });
+    const first = await boundary(request());
+    const queued = boundary(request());
+    await new Promise((accept) => setTimeout(accept, 10));
+    allowed = false;
+    await first.body?.cancel();
+    expect((await queued).status).toBe(401);
+    expect(calls).toBe(1);
+  } finally {
+    if (oldGlobal === undefined)
+      delete process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS;
+    else process.env.AI_PROVIDER_MAX_CONCURRENT_RUNS = oldGlobal;
+    if (oldNative === undefined) delete process.env.AGENT_DSH_MAX_ACTIVE_RUNS;
+    else process.env.AGENT_DSH_MAX_ACTIVE_RUNS = oldNative;
+  }
+});

package/template/apps/agent-server/native/office-preview.test.ts

@@ -0,0 +1,50 @@
+import { expect, test } from "bun:test";
+import { Config } from "@deepseek-ai/dsh-office-to-pdf";
+import { officePreviewPlugin } from "./office-preview";
+import { nativePackages } from "../install-native";
+
+test("Office preview uses valid bounded foreground-only configuration", () => {
+  const row = officePreviewPlugin();
+  const config = Config(row.config);
+  expect(row.id).toBe("office-to-pdf");
+  expect(config.maxConcurrentConversions).toBe(1);
+  expect(config.maxBackgroundConversions).toBe(0);
+  expect(config.maxSourceBytes).toBe(config.maxInputBytes);
+  expect(config.maxCachedBytes).toBeLessThanOrEqual(32 * 1024 * 1024);
+});
+
+test("both Linux architectures select the exact lock-pinned Office engine", async () => {
+  const parsed: unknown = Bun.JSONC.parse(
+    await Bun.file(new URL("../../../bun.lock", import.meta.url)).text(),
+  );
+  if (
+    !parsed ||
+    typeof parsed !== "object" ||
+    !("packages" in parsed) ||
+    !parsed.packages ||
+    typeof parsed.packages !== "object"
+  )
+    throw new Error("Invalid runtime lock");
+  const packages: Record<string, [string, string, unknown, string]> = {};
+  for (const [name, value] of Object.entries(parsed.packages)) {
+    if (
+      Array.isArray(value) &&
+      typeof value[0] === "string" &&
+      typeof value[1] === "string" &&
+      typeof value[3] === "string"
+    )
+      packages[name] = [value[0], value[1], value[2], value[3]];
+  }
+  const lock = { packages };
+  for (const architecture of ["arm64", "x64"]) {
+    const selected = nativePackages(lock, architecture).filter((pkg) =>
+      pkg.name.includes("libreoffice"),
+    );
+    expect(selected).toHaveLength(1);
+    expect(selected[0]?.name).toBe("@deepseek-ai/libreoffice-kit-wasm");
+    expect(selected[0]?.version).toBe("0.1.5");
+    expect(selected[0]?.integrity).toBe(
+      lock.packages["@deepseek-ai/libreoffice-kit-wasm"][3],
+    );
+  }
+});

package/template/apps/agent-server/native/office-preview.ts

@@ -0,0 +1,25 @@
+/** Host conversion stays within the existing isolated user's Agent worker. */
+export function officePreviewPlugin() {
+  return {
+    id: "office-to-pdf",
+    config: {
+      maxConcurrentConversions: 1,
+      maxBackgroundConversions: 0,
+      maxQueuedJobs: 2,
+      maxReaders: 4,
+      maxSourceBytes: 16 * 1024 * 1024,
+      maxInputBytes: 16 * 1024 * 1024,
+      maxOutputBytes: 20 * 1024 * 1024,
+      maxCachedEntries: 2,
+      maxCachedBytes: 32 * 1024 * 1024,
+      maxSourceEntries: 8,
+      timeoutMs: 60000,
+      maxArchiveEntries: 4096,
+      maxUncompressedBytes: 64 * 1024 * 1024,
+      maxFontFiles: 128,
+      maxFontFileBytes: 64 * 1024 * 1024,
+      maxLoadedFontBytes: 128 * 1024 * 1024,
+      fontDirectories: ["/usr/share/fonts/opentype/noto"],
+    },
+  };
+}