@sproutboat/toolchain 0.4.10 → 0.4.11

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @sproutboat/toolchain
 
+## 0.4.11
+
+### Patch Changes
+
+- 6d72af7: Fix `TypedArray.from` with array-like inputs in native binaries, and regenerate Porffor's builtins table when its compiler is unpacked through a symlinked temporary path. This preserves the key bytes in repeated HMAC signing with `Uint8Array.from({ length: 16 }, mapFn)`.
+
 ## 0.4.10
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/toolchain",
-  "version": "0.4.10",
+  "version": "0.4.11",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/acquire.test.ts

@@ -29,6 +29,20 @@ async function fixture(): Promise<{ archive: string; sha256: string }> {
       "  return -1;\n" +
       "}\n",
     "compiler/index.js": "          '-xc', '-', '-c',\n          uSocketsArchive,\n          '-lm'\n",
+    "compiler/precompile.js":
+      "const fs = require('node:fs');\n" +
+      "const path = require('node:path');\n" +
+      "fs.writeFileSync(path.join(__dirname, 'builtins_precompiled.js'), fs.readFileSync(path.join(__dirname, 'builtins/typedarray.js')));\n",
+    "compiler/builtins_precompiled.js": "fixture:precompiled\n",
+    "compiler/builtins/typedarray.js":
+      "export const __${name}_from = (arg: any, mapFn: any): ${name} => {\n" +
+      "  const arr: any[] = Porffor.array.new(4);\n" +
+      "  let len: i32 = 0;\n" +
+      "  if (Porffor.type(arg) == Porffor.TYPES.array) {\n" +
+      "    let i: i32 = 0;\n" +
+      "    for (const x of arg) arr[i++] = x;\n" +
+      "    len = i;\n" +
+      "  }\n\n  arr.length = len;\n\n  return new ${name}(arr);\n};\n",
     "compiler/uwebsockets.js":
       "static const size_t REQUEST_BODY_MAX_BYTES = 1024u * 1024u;\n" +
       "int porf_native_fetch_read_value(struct jsval value, const char** out_buf, size_t* out_len, char** out_owned);\n" +
@@ -202,6 +216,35 @@ test("a corrupted warm cache is replaced from the verified archive", async () =>
   expect(await readFile(join(root, "compiler/render.js"), "utf8")).toContain('getenv("PORT")');
 });
 
+test("a cache built by an older toolchain version is rebuilt, not silently reused", async () => {
+  const { archive, sha256 } = await fixture();
+  const cacheRoot = await mkdtemp(join(tmpdir(), "sb-porffor-stale-toolchain-"));
+  temporary.push(cacheRoot);
+  let requests = 0;
+  const options = {
+    cacheRoot,
+    url: "fixture",
+    expectedSha256: sha256,
+    fetcher: async () => {
+      requests += 1;
+      return new Response(Bun.file(archive));
+    },
+  };
+  const root = await ensurePorffor(options);
+  const manifestPath = join(root, ".sproutboat-complete");
+  const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+  expect(manifest.toolchainVersion).toBeTruthy();
+  // Simulate a cache left behind by an older @sproutboat/toolchain publish
+  // (baronunread/sproutboat#205): a patch.ts fix landing in a new version used
+  // to sit unused because the cache key only tracked the upstream commit.
+  await rm(manifestPath);
+  await writeFile(manifestPath, JSON.stringify({ ...manifest, toolchainVersion: "0.0.0-stale" }));
+  await ensurePorffor(options);
+  expect(requests).toBe(2);
+  const rebuilt = JSON.parse(await readFile(manifestPath, "utf8"));
+  expect(rebuilt.toolchainVersion).toBe(manifest.toolchainVersion);
+});
+
 test("an interrupted stale lock is recovered", async () => {
   const { archive, sha256 } = await fixture();
   const cacheRoot = await mkdtemp(join(tmpdir(), "sb-porffor-interrupted-"));

package/src/acquire.ts

@@ -8,6 +8,12 @@ import { resolve } from "node:path";
 import vendoredPorfforArchive from "../vendor/porffor-8f01541.tar.gz" with { type: "file" };
 import { ensurePorfforPatched } from "./patch";
 import { PORFFOR_ARCHIVE_SHA256, PORFFOR_ARCHIVE_URL, PORFFOR_COMMIT_FULL } from "./pin";
+// Folded into the cache manifest below so a toolchain release that changes
+// patch.ts's output invalidates every machine's cache automatically
+// (baronunread/sproutboat#205 — a patch fix published as a new version used
+// to sit unused behind a still-"complete" cache keyed only on the upstream
+// Porffor commit).
+import { version as TOOLCHAIN_VERSION } from "../package.json" with { type: "json" };
 
 export class PorfforToolchainError extends Error {
   constructor(
@@ -31,6 +37,9 @@ const required = [
   "compiler/render.js",
   "compiler/index.js",
   "compiler/uwebsockets.js",
+  "compiler/precompile.js",
+  "compiler/builtins/typedarray.js",
+  "compiler/builtins_precompiled.js",
 ];
 async function digest(path: string): Promise<string> {
   return createHash("sha256")
@@ -45,9 +54,14 @@ async function complete(dir: string, expectedArchive = PORFFOR_ARCHIVE_SHA256):
     const manifest = JSON.parse(await readFile(resolve(dir, ".sproutboat-complete"), "utf8")) as {
       commit?: string;
       archiveSha256?: string;
+      toolchainVersion?: string;
       files?: Record<string, string>;
     };
-    if (manifest.commit !== PORFFOR_COMMIT_FULL || manifest.archiveSha256 !== expectedArchive)
+    if (
+      manifest.commit !== PORFFOR_COMMIT_FULL ||
+      manifest.archiveSha256 !== expectedArchive ||
+      manifest.toolchainVersion !== TOOLCHAIN_VERSION
+    )
       return false;
     for (const file of required)
       if (!manifest.files?.[file] || (await digest(resolve(dir, file))) !== manifest.files[file])
@@ -185,7 +199,7 @@ export async function ensurePorffor(options: AcquireOptions = {}): Promise<strin
     );
     await writeFile(
       resolve(stage, ".sproutboat-complete"),
-      JSON.stringify({ commit: PORFFOR_COMMIT_FULL, archiveSha256: actual, files }),
+      JSON.stringify({ commit: PORFFOR_COMMIT_FULL, archiveSha256: actual, toolchainVersion: TOOLCHAIN_VERSION, files }),
       { mode: 0o444 },
     );
     await rename(stage, dir);

package/src/patch.ts

@@ -1,6 +1,7 @@
 /**
  * Idempotent, marker-guarded in-place edits to Porffor's generated C
- * (`compiler/render.js`, `compiler/index.js`, `compiler/uwebsockets.js`). Run
+ * (`compiler/render.js`, `compiler/index.js`, `compiler/uwebsockets.js`,
+ * `compiler/builtins/typedarray.js`). Run
  * from the build path, not a `postinstall` hook: package managers block
  * dependency lifecycle scripts by default, so a published `postinstall` would
  * silently not run.
@@ -9,7 +10,7 @@
  * older version of this module still receives the newer edits. All of them are
  * tracked in patches/UPSTREAM.md and go away as Porffor closes the gaps.
  */
-import { readFile, writeFile } from "node:fs/promises";
+import { readFile, realpath, writeFile } from "node:fs/promises";
 import { resolve } from "node:path";
 
 // --- compiler/render.js: the native-fetch server, rendered as C text ---
@@ -167,6 +168,28 @@ const CFLAGS_INJECT =
   "          ...(process.env.SB_EXTRA_CFLAGS ? process.env.SB_EXTRA_CFLAGS.split(' ').filter(Boolean) : []),\n";
 const CFLAGS_MARKER = "SB_EXTRA_CFLAGS";
 
+// Porffor's TypedArray.from only handles iterables. An array-like input such
+// as { length: 16 } silently becomes an empty typed array, including HMAC
+// keys made with Uint8Array.from({ length: 16 }, mapFn). The compiler uses a
+// precompiled builtin table, so the edited source must be precompiled again.
+const TYPED_ARRAY_FROM_ANCHOR =
+  "    len = i;\n  }\n\n  arr.length = len;\n\n  return new ${name}(arr);\n};";
+const TYPED_ARRAY_FROM_INJECT =
+  "    len = i;\n" +
+  "  } else {\n" +
+  "    // sproutboat: TypedArray.from accepts array-like objects.\n" +
+  "    let count = ecma262.ToIntegerOrInfinity(Porffor.object.get(arg, 'length'));\n" +
+  "    if (count < 0) count = 0;\n" +
+  "    if (count > 2147483643) throw new RangeError('Invalid TypedArray length (over maximum supported length)');\n" +
+  "    if (Porffor.type(mapFn) != Porffor.TYPES.undefined && Porffor.type(mapFn) != Porffor.TYPES.function)\n" +
+  "      throw new TypeError('Called TypedArray.from with a non-function mapFn');\n" +
+  "    for (let i: i32 = 0; i < count; i++) {\n" +
+  "      arr[i] = Porffor.type(mapFn) == Porffor.TYPES.undefined ? arg[i] : mapFn(arg[i], i);\n" +
+  "    }\n" +
+  "    len = count;\n" +
+  "  }\n\n  arr.length = len;\n\n  return new ${name}(arr);\n};";
+const TYPED_ARRAY_FROM_MARKER = "sproutboat: TypedArray.from accepts array-like objects";
+
 /**
  * #56 — make the inbound request-body limit configurable.
  *
@@ -830,6 +853,34 @@ async function patchCompilerArgs(root: string): Promise<void> {
   if (changed) await writeFile(file, src);
 }
 
+export async function patchTypedArrayFrom(root: string): Promise<void> {
+  const file = resolve(root, "compiler/builtins/typedarray.js");
+  const src = await readFile(file, "utf8");
+  if (!src.includes(TYPED_ARRAY_FROM_MARKER) && !src.includes(TYPED_ARRAY_FROM_ANCHOR)) {
+    throw new Error(`could not patch Porffor's TypedArray.from: anchor not found in ${file}`);
+  }
+  if (!src.includes(TYPED_ARRAY_FROM_MARKER))
+    await writeFile(file, src.replace(TYPED_ARRAY_FROM_ANCHOR, TYPED_ARRAY_FROM_INJECT));
+  const table = resolve(root, "compiler/builtins_precompiled.js");
+  const before = await readFile(table);
+  // Porffor only writes the table when import.meta.url matches argv[1].
+  // macOS's /var -> /private/var symlink makes a noncanonical argv[1] skip it.
+  const precompile = await realpath(resolve(root, "compiler/precompile.js"));
+  const child = Bun.spawn(["node", precompile], {
+    cwd: root,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [code, stdout, stderr] = await Promise.all([
+    child.exited,
+    new Response(child.stdout).text(),
+    new Response(child.stderr).text(),
+  ]);
+  if (code !== 0) throw new Error(`could not precompile Porffor's TypedArray.from: ${stderr || stdout}`);
+  if (!src.includes(TYPED_ARRAY_FROM_MARKER) && before.equals(await readFile(table)))
+    throw new Error(`Porffor precompile did not update ${table}`);
+}
+
 /** The native-fetch server source: $PORT support + the #165 console sink. Exported for tests. */
 export async function patchRenderJs(root: string): Promise<void> {
   const file = resolve(root, "compiler/render.js");
@@ -876,5 +927,6 @@ export async function ensurePorfforPatched(root: string): Promise<void> {
   await patchCompilerArgs(root);
   await patchUwebsockets(root);
   await patchRenderJs(root);
+  await patchTypedArrayFrom(root);
   done.add(root);
 }