@unoverse-platform/studio 0.1.91 → 0.1.92

package/vendor/base/src/http/uwsFetchAdapter.ts

@@ -14,9 +14,14 @@
  *  - every write happens inside `res.cork()`.
  *  - body chunks are copied synchronously (uWS reuses the chunk's backing memory).
  */
-import { gzipSync } from "node:zlib";
+import { gzip as gzipCallback, createGzip, constants as zlibConstants } from "node:zlib";
+import { promisify } from "node:util";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 import type { HttpRequest, HttpResponse } from "uWebSockets.js";
 
+const gzipAsync = promisify(gzipCallback);
+
 const STATUS_TEXT: Record<number, string> = {
   200: "OK",
   202: "Accepted",
@@ -34,6 +39,91 @@ const statusLine = (code: number): string => `${code} ${STATUS_TEXT[code] ?? "OK
 const COMPRESSIBLE = /^(text\/|application\/(json|javascript|xml|vnd\.unoverse))/;
 /** Below this, the gzip header + CPU outweigh the shrink. */
 const COMPRESS_MIN_BYTES = 1024;
+/**
+ * BUFFER THEN SPILL. A single-shot body is held up to this many bytes so a small
+ * response (every MCP reply, most JSON) still goes out in ONE write with an exact
+ * length and the cached-gzip path. A body that runs past it is STREAMED from here:
+ * head first, then chunks as they arrive, gzip applied on the fly. Before this every
+ * response was buffered whole, so time-to-first-byte equalled time-to-last-byte, and a
+ * large proxied list cost its full size twice in memory (buffer plus gzip output).
+ */
+const STREAM_AFTER_BYTES = 64 * 1024;
+
+type PumpState = { aborted: boolean; reader?: ReadableStreamDefaultReader<Uint8Array>; drained?: () => void };
+
+/** Status, the upstream headers uWS must not see twice, encoding, CORS. Inside a cork. */
+function writeHead(res: HttpResponse, response: Response, gzip: boolean): void {
+  res.writeStatus(statusLine(response.status));
+  response.headers.forEach((value, key) => {
+    const k = key.toLowerCase();
+    // uWS generates these itself on every response. Forwarding an upstream copy
+    // (e.g. the engine's own `date`) yields a DUPLICATE header — nginx just warns
+    // and serves 200, but HTTP/3 (QUIC) rejects the malformed header block with
+    // QUIC_PACKET_READ_ERROR. Skip them so uWS's single copy is authoritative.
+    // access-control-*: writeCors() below is authoritative — forwarding an upstream
+    // copy too produced the doubled `access-control-allow-origin: *,*` seen live.
+    if (k === "content-length" || k === "date" || k === "server" || k.startsWith("access-control-")) return;
+    res.writeHeader(key, value);
+  });
+  if (gzip) {
+    res.writeHeader("Content-Encoding", "gzip");
+    // Encoding varies by the request's Accept-Encoding, so shared caches must key on it.
+    res.writeHeader("Vary", "Accept-Encoding");
+  }
+  writeCors(res);
+}
+
+/**
+ * One chunk onto the socket, honouring BACKPRESSURE. `res.write` returns false when
+ * uWS's socket buffer is full; the bytes are queued, but reading on regardless meant a
+ * stalled client (a backgrounded tab, a slow tunnel) grew that queue without bound
+ * while the engine kept streaming into it. Park until uWS reports the socket writable
+ * again, or the client aborts (onAborted wakes the parked pump).
+ */
+async function writeChunk(res: HttpResponse, state: PumpState, chunk: Uint8Array | string): Promise<void> {
+  let ok = true;
+  res.cork(() => {
+    ok = res.write(chunk);
+  });
+  if (ok || state.aborted) return;
+  await new Promise<void>((resolve) => {
+    state.drained = () => {
+      state.drained = undefined;
+      resolve();
+    };
+    res.onWritable(() => {
+      state.drained?.();
+      return true;
+    });
+  });
+}
+
+/**
+ * Compressed bytes for responses that carry an ETag, keyed on it. An ETag says the body
+ * is a fixed thing (the app shell, the embed bundle, keyed on the build's mtime), and
+ * those are the biggest payloads this server sends: 427KB of shell and 186KB of embed
+ * were gzipped afresh on EVERY 200, ~10-30ms of CPU each. Bounded and insertion-evicted;
+ * a new build has a new ETag, so a stale entry is never served, only aged out.
+ */
+const GZIP_CACHE_MAX = 32;
+const gzipCache = new Map<string, Buffer>();
+
+/** gzip `payload` OFF the event loop (zlib's threadpool), reusing the cached bytes
+ *  when the response names itself with an ETag. The old `gzipSync` ran on the same
+ *  loop that carries live audio frames and every SSE pump. */
+async function compress(payload: Buffer, etag: string | null, contentType: string): Promise<Buffer> {
+  const key = etag ? `${etag}|${contentType}|${payload.length}` : null;
+  if (key) {
+    const hit = gzipCache.get(key);
+    if (hit) return hit;
+  }
+  const gz = await gzipAsync(payload);
+  if (key) {
+    if (gzipCache.size >= GZIP_CACHE_MAX) gzipCache.delete(gzipCache.keys().next().value as string);
+    gzipCache.set(key, gz);
+  }
+  return gz;
+}
 
 function writeCors(res: HttpResponse): void {
   // The workbench is same-origin (via the Vite proxy) so needs none of this; kept so
@@ -92,9 +182,18 @@ export async function handleFetch(
   req: HttpRequest,
   handler: (request: Request) => Promise<Response>,
 ): Promise<void> {
-  const state: { aborted: boolean; reader?: ReadableStreamDefaultReader<Uint8Array> } = { aborted: false };
+  const state: PumpState = { aborted: false };
+  // THE HANDLER LEARNS OF THE DISCONNECT. `Request.signal` is how a Fetch handler is told
+  // its caller went away; without it a streaming handler (a node that runs for minutes)
+  // kept working for a client that had hung up, and an engine that had reaped a run could
+  // not stop the node it had started. Observed live 2026-09-05.
+  const abort = new AbortController();
   res.onAborted(() => {
     state.aborted = true;
+    abort.abort();
+    // A pump parked on backpressure must wake to see the abort, or it (and its
+    // keepalive timer) would hang for the life of the process.
+    state.drained?.();
     void state.reader?.cancel().catch(() => {});
   });
 
@@ -121,7 +220,7 @@ export async function handleFetch(
 
   let response: Response;
   try {
-    const init: RequestInit = { method, headers };
+    const init: RequestInit = { method, headers, signal: abort.signal };
     if (body && body.length) init.body = new Uint8Array(body); // BodyInit-compatible copy of the Buffer
     response = await handler(new Request(url, init));
   } catch (err) {
@@ -142,72 +241,109 @@ export async function handleFetch(
 
   const isSse = (response.headers.get("content-type") ?? "").startsWith("text/event-stream");
 
-  // Single-shot responses (everything except SSE) are buffered so large text payloads
-  // can be gzipped. Neither uWS nor the DO load balancer compresses, so without this
-  // every byte travels identity-encoded — the widget boot alone was ~660KB of JS/JSON
-  // that gzips to ~150KB. SSE must stream, so it is never buffered or compressed.
+  // Single-shot responses (everything except SSE): buffered up to STREAM_AFTER_BYTES so
+  // text payloads can be gzipped in one write (neither uWS nor the DO load balancer
+  // compresses — the widget boot alone was ~660KB of JS/JSON that gzips to ~150KB), and
+  // STREAMED past that so a large body's first bytes leave before its last arrive.
   if (!isSse) {
     const contentType = response.headers.get("content-type") ?? "";
     const acceptsGzip = /(^|,)\s*gzip\s*(;|,|$)/.test(headers["accept-encoding"] ?? "");
-    let payload: Buffer | null = null;
-    try {
-      payload = Buffer.from(await response.arrayBuffer());
-    } catch {
-      payload = null; // body unreadable — fall through and end empty below
+    const compressible = acceptsGzip && COMPRESSIBLE.test(contentType) && !response.headers.get("content-encoding");
+
+    // An ETag names a FIXED body (the app shell, the embed bundle): those are buffered
+    // whole so the cached-gzip path serves them, however large. Only ETag-less bodies
+    // (proxied lists, anything dynamic) spill into streaming.
+    const spillAfter = response.headers.get("etag") ? Infinity : STREAM_AFTER_BYTES;
+    const reader = response.body?.getReader();
+    state.reader = reader;
+    const prefix: Uint8Array[] = [];
+    let size = 0;
+    let complete = !reader;
+    if (reader) {
+      try {
+        for (;;) {
+          const { done, value } = await reader.read();
+          if (state.aborted) {
+            void reader.cancel().catch(() => {});
+            return;
+          }
+          if (done) {
+            complete = true;
+            break;
+          }
+          if (value?.length) {
+            prefix.push(value);
+            size += value.length;
+          }
+          if (size >= spillAfter) break;
+        }
+      } catch {
+        complete = true; // body unreadable — end empty below, as before
+        prefix.length = 0;
+        size = 0;
+      }
     }
-    if (state.aborted) return;
-    const shouldCompress =
-      acceptsGzip &&
-      payload !== null &&
-      payload.length >= COMPRESS_MIN_BYTES &&
-      COMPRESSIBLE.test(contentType) &&
-      !response.headers.get("content-encoding");
-    const body = shouldCompress && payload ? gzipSync(payload) : payload;
-    res.cork(() => {
-      res.writeStatus(statusLine(response.status));
-      response.headers.forEach((value, key) => {
-        const k = key.toLowerCase();
-        // uWS generates these itself on every response. Forwarding an upstream copy
-        // (e.g. the engine's own `date`) yields a DUPLICATE header — nginx just warns
-        // and serves 200, but HTTP/3 (QUIC) rejects the malformed header block with
-        // QUIC_PACKET_READ_ERROR. Skip them so uWS's single copy is authoritative.
-        // access-control-*: writeCors() below is authoritative — forwarding an upstream
-        // copy too produced the doubled `access-control-allow-origin: *,*` seen live.
-        if (k === "content-length" || k === "date" || k === "server" || k.startsWith("access-control-")) return;
-        res.writeHeader(key, value);
+
+    if (complete) {
+      const payload = size ? Buffer.concat(prefix) : null;
+      const shouldCompress = compressible && payload !== null && payload.length >= COMPRESS_MIN_BYTES;
+      const body = shouldCompress && payload ? await compress(payload, response.headers.get("etag"), contentType) : payload;
+      if (state.aborted) return;
+      res.cork(() => {
+        writeHead(res, response, shouldCompress);
+        // `res.end()` for the empty case, NEVER `endWithoutBody()`. On HTTP/1.1 with no
+        // Content-Length, endWithoutBody() leaves the response unterminated: the client
+        // waits on a reply that never finishes until the socket's idle timeout kills the
+        // connection ~12s later. Every bodyless response paid it — the MCP handshake's
+        // 202 ack among them, so every app open stalled ~10s before its first paint
+        // (observed live on bppunoverse 2026-08-21; reproduced in isolation: a JSON
+        // response 28ms, a null-body 202 11.9s). end() closes with content-length: 0.
+        if (body && body.length) res.end(body);
+        else res.end();
       });
-      if (shouldCompress) {
-        res.writeHeader("Content-Encoding", "gzip");
-        // Encoding varies by the request's Accept-Encoding, so shared caches must key on it.
-        res.writeHeader("Vary", "Accept-Encoding");
+      return;
+    }
+
+    // SPILL: the body is larger than the buffer. Head now, then the prefix and the rest
+    // as they arrive, chunked. Compression rides a streaming gzip that flushes per input
+    // chunk (Z_SYNC_FLUSH), so nothing waits inside zlib for a fuller block.
+    res.cork(() => writeHead(res, response, compressible));
+    const rest = async function* (): AsyncGenerator<Uint8Array> {
+      yield* prefix;
+      for (;;) {
+        const { done, value } = await reader!.read();
+        if (done || state.aborted) return;
+        if (value?.length) yield value;
       }
-      writeCors(res);
-      // `res.end()` for the empty case, NEVER `endWithoutBody()`. On HTTP/1.1 with no
-      // Content-Length, endWithoutBody() leaves the response unterminated: the client
-      // waits on a reply that never finishes until the socket's idle timeout kills the
-      // connection ~12s later. Every bodyless response paid it — the MCP handshake's
-      // 202 ack among them, so every app open stalled ~10s before its first paint
-      // (observed live on bppunoverse 2026-08-21; reproduced in isolation: a JSON
-      // response 28ms, a null-body 202 11.9s). end() closes with content-length: 0.
-      if (body && body.length) res.end(body);
-      else res.end();
-    });
+    };
+    let source: AsyncIterable<Uint8Array> = rest();
+    let gz: ReturnType<typeof createGzip> | null = null;
+    if (compressible) {
+      gz = createGzip({ flush: zlibConstants.Z_SYNC_FLUSH });
+      void pipeline(Readable.from(rest()), gz).catch(() => {});
+      source = gz as AsyncIterable<Uint8Array>;
+    }
+    try {
+      for await (const chunk of source) {
+        if (state.aborted) break;
+        await writeChunk(res, state, chunk);
+      }
+      if (!state.aborted) res.cork(() => res.end());
+    } catch {
+      if (!state.aborted) res.cork(() => res.end());
+    } finally {
+      if (state.aborted) {
+        void reader!.cancel().catch(() => {});
+        gz?.destroy();
+      }
+    }
     return;
   }
 
   // SSE: write head, then pump chunks until the transport closes the stream.
   const reader = response.body?.getReader();
   state.reader = reader;
-  res.cork(() => {
-    res.writeStatus(statusLine(response.status));
-    response.headers.forEach((value, key) => {
-      const k = key.toLowerCase();
-      // See the single-shot branch for why these are skipped.
-      if (k === "content-length" || k === "date" || k === "server" || k.startsWith("access-control-")) return;
-      res.writeHeader(key, value);
-    });
-    writeCors(res);
-  });
+  res.cork(() => writeHead(res, response, false));
 
   if (!reader) {
     if (!state.aborted) res.cork(() => res.end());
@@ -233,7 +369,8 @@ export async function handleFetch(
   // blind the canvas. Comment lines are ~18 bytes — frequency is free.
   const keepalive = isSse
     ? setInterval(() => {
-        if (!state.aborted) res.cork(() => res.write(": keepalive\n\n"));
+        // Not while parked on backpressure: a beat would only deepen the queue.
+        if (!state.aborted && !state.drained) res.cork(() => res.write(": keepalive\n\n"));
       }, 10_000)
     : null;
 
@@ -246,7 +383,7 @@ export async function handleFetch(
         return;
       }
       if (done) break;
-      if (value) res.cork(() => res.write(value));
+      if (value) await writeChunk(res, state, value);
     }
     if (!state.aborted) res.cork(() => res.end());
     if (isSse) console.log(`[unoverse:sse] pump CLOSED (stream ended) ${method} ${path} sid=${sid}`);

package/vendor/base/src/items/collect.ts

@@ -20,12 +20,15 @@
 import { readdirSync, readFileSync, existsSync, statSync } from "node:fs";
 import { join, relative, basename } from "node:path";
 import { fingerprintOf } from "./fingerprint.js";
+import { parse as parseYaml } from "yaml";
+import { readDiskPackages } from "../manifests/source.js";
 
 /** One publishable thing: what it is, what it contains, and what it was built on. */
 export interface CollectedItem {
   kind: string;
   name: string;
-  definition: { files: Record<string, string> };
+  /** The files as authored. A node row also carries its package envelope and folder name. */
+  definition: { files: Record<string, string>; package?: Record<string, unknown>; dir?: string };
   fingerprint: string;
   org: string;
   base_version: string;
@@ -106,7 +109,7 @@ export function collectProject(designRoot: string, project: string): CollectedIt
   // Apps keep their bare ids: those are org-qualified by convention
   // (`<org>-chat`). docs/unoverse/UNOVERSE_COMPONENT_ORGS.md.
   const QUALIFIED_KINDS = new Set(["component", "skill", "template"]);
-  const add = (kind: string, name: string, definition: { files: Record<string, string> }) =>
+  const add = (kind: string, name: string, definition: CollectedItem["definition"]) =>
     items.push({
       kind,
       name: QUALIFIED_KINDS.has(kind) ? `${project}/${name}` : name,
@@ -195,5 +198,49 @@ export function collectProject(designRoot: string, project: string): CollectedIt
     walk(blocksHome);
   }
 
+  // nodes/ — THE WORKSPACE'S NODE PACKAGES, one row per node, shaped exactly as the
+  // marketplace stores a node (core/items/catalogue.ts) so the row source composes them
+  // on the same path as disk: the raw files, plus the package envelope their $refs and
+  // credential shapes resolve against. The name is the node's `type`, bare: a saved
+  // workflow stores that identity and nothing else.
+  //
+  // Until 2026-09-05 deploy never looked here, so `deploy studio` shipped a developer's
+  // components and skills and silently left every node behind.
+  //
+  // NOT FROM THE PLATFORM TREE. The monorepo's nodes ship inside the core image and are
+  // read from disk there; publishing them as rows under whichever org happened to be
+  // deployed would double every platform node. A design system on disk is the mark of
+  // the platform tree (the same test designSystemDir makes), and a developer's workspace
+  // never has one.
+  const nodesHome = join(designRoot, "..", "nodes");
+  if (existsSync(nodesHome) && !existsSync(join(designRoot, "marketplace"))) {
+    for (const pkg of collectNodePackages(nodesHome)) {
+      for (const raw of pkg.nodes) {
+        const type = nodeType(raw.files["node.yaml"]);
+        if (!type) continue; // lint has already refused it; nothing to name a row by
+        add("node", type, {
+          package: { name: pkg.name, packageFile: pkg.packageFile ?? null, credentials: pkg.credentials, shared: pkg.shared },
+          dir: raw.dir,
+          files: raw.files,
+        });
+      }
+    }
+  }
+
   return items;
 }
+
+/** The workspace's node packages, read the way the platform reads its own. */
+function collectNodePackages(nodesHome: string) {
+  return readDiskPackages(nodesHome).filter((pkg) => pkg.name !== "marketplace");
+}
+
+/** `type:` off a node.yaml: the one scalar the row is named by. Unparseable means none. */
+function nodeType(nodeYaml: string | undefined): string {
+  try {
+    const doc = parseYaml(nodeYaml ?? "");
+    return typeof doc?.type === "string" ? doc.type : "";
+  } catch {
+    return "";
+  }
+}

package/vendor/base/src/items/publish.ts

@@ -12,6 +12,8 @@
  *
  * See docs/architecture/DECLARATIVE_NODES.md §9.
  */
+import { existsSync } from "node:fs";
+import { join } from "node:path";
 import { collectProject, type CollectedItem } from "./collect.js";
 
 /** One lint finding. `error` blocks a publish; `warn` and `hint` inform. */
@@ -46,6 +48,18 @@ export interface PublishPlan {
 export async function lintForPublish(designRoot: string, project: string): Promise<{ problems: Finding[]; errors: Finding[] }> {
   const { lintDefinitions } = await import("../lint/design/index.mjs");
   const result = lintDefinitions(designRoot);
+  /**
+   * NODES TOO. A workspace keeps its node packages beside design/ (the same `nodes/` the
+   * Studio Nodes screen reads), and until 2026-09-05 this gate never looked at them: a
+   * node with an events row out of order or a host missing from allowedHosts deployed as
+   * cleanly as a correct one. The node linter is the same library the platform's own CI
+   * runs. A workspace with no nodes/ folder has nothing to lint and is not an error.
+   */
+  const nodesHome = join(designRoot, "..", "nodes");
+  if (existsSync(nodesHome)) {
+    const { lintNodes } = await import("../lint/nodes/index.mjs");
+    result.problems.push(...lintNodes(nodesHome).problems);
+  }
   /**
    * SCOPED TO THE PROJECT BEING DEPLOYED, on paths that may be relative. The old
    * check looked for "/design/" with a leading slash, which a relative path never

package/vendor/base/src/lint/nodes/index.d.mts

@@ -0,0 +1,28 @@
+/**
+ * Types for the node linter, which is plain JavaScript.
+ *
+ * Same arrangement as ../design/index.d.mts, for the same reason: the linter stays .mjs so
+ * the CLIs run it under bare node, and TypeScript consumers (the publish gate) get a shape
+ * rather than `any`.
+ */
+
+export interface Finding {
+  level: "error" | "warn" | "hint";
+  file: string;
+  line?: number;
+  msg: string;
+}
+
+export interface NodesResult {
+  problems: Finding[];
+  derivedKinds: Map<string, string>;
+  nodeCount: number;
+  pkgCount: number;
+  nodesHome: string;
+}
+
+/** Lint every node package under `nodesHome`. Prints nothing, throws nothing for a lint failure. */
+export function lintNodes(nodesHome: string): NodesResult;
+
+/** True when anything would fail a build. */
+export function hasErrors(result: NodesResult): boolean;

package/vendor/base/src/lint/nodes/index.mjs

@@ -20,7 +20,8 @@
  *   4  dry run      `unoverse node test`, executes against testData, not here
  */
 import { readdirSync, existsSync, statSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { join, resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
 import { problems, derivedKinds, reset, report } from "../context.mjs";
 import { loadSchemas } from "./schema.mjs";
 import { lintPackage } from "./package.mjs";
@@ -32,9 +33,26 @@ import { lintPackage } from "./package.mjs";
  * missing directory or unreadable schema is itself reported, so a caller gets one shape
  * of answer whatever went wrong.
  */
+/**
+ * Where the node schemas are, for this run.
+ *
+ * The platform's own tree carries them at `nodes/_schema/`, and every node file points at
+ * them with `$schema:` for editor squiggles. A DEVELOPER'S WORKSPACE DOES NOT: it holds only
+ * their packages, and a linter that then warned "no schema found" on every file was
+ * validating nothing while looking busy. So the schemas travel with the linter. In dist
+ * they sit beside this file (copy-lint.mjs puts them there, and the CLI vendors that
+ * folder); from source they are the monorepo's copy, five levels up.
+ */
+function schemaHome(home) {
+  const here = dirname(fileURLToPath(import.meta.url));
+  for (const c of [join(home, "_schema"), join(here, "_schema"), resolve(here, "../../../../../apps/unoverse/nodes/_schema")])
+    if (existsSync(c)) return c;
+  return join(home, "_schema");
+}
+
 export function lintNodes(nodesHome) {
   const home = resolve(nodesHome);
-  reset(home, join(home, "_schema"));
+  reset(home, schemaHome(home));
 
   if (!existsSync(home)) {
     report("error", home, "no nodes directory here");