@sproutboat/toolchain 0.4.2 → 0.4.4

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @sproutboat/toolchain
 
+## 0.4.4
+
+### Patch Changes
+
+- d8b7877: Add native standalone R2 direct-download tickets with bounded, backpressure-aware
+  file reads, including upgrades for existing patched Porffor caches.
+
+## 0.4.3
+
+### Patch Changes
+
+- 0b0a666: Add a native standalone R2 direct-upload bridge that streams transfer-ticket
+  request chunks to a temporary blob file, validates the digest, and atomically
+  publishes the object without buffering the upload in the Porffor request body.
+
 ## 0.4.2
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/toolchain",
-  "version": "0.4.2",
+  "version": "0.4.4",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/acquire.test.ts

@@ -40,6 +40,10 @@ async function fixture(): Promise<{ archive: string; sha256: string }> {
       "  i32 slot = 0;\n  for (auto [key, value] : *req) {\n    slot++;\n  }\n" +
       "  *((i32*)(porf_mem + headers_ptr)) = slot;\n\n  return headers_ptr;\n}\n" +
       "static void on_request(uWS::HttpResponse<false>* res, uWS::HttpRequest* req) {\n" +
+      "  const std::string_view method = req->getCaseSensitiveMethod();\n" +
+      "  const i32 method_ptr = get_method_ptr(method);\n" +
+      "  if (method_ptr == 0) return;\n" +
+      "  __porffor_js_enter();\n  const i32 url_ptr = alloc_request_url(req);\n" +
       "  const i32 headers_ptr = collect_headers(req);\n}\n" +
       "static bool is_forbidden_response_header(std::string_view key) {\n" +
       '  return key == "connection" ||\n' +

package/src/patch.test.ts

@@ -76,6 +76,11 @@ static i32 collect_headers(uWS::HttpRequest* req) {
 }
 
 static void on_request(uWS::HttpResponse<false>* res, uWS::HttpRequest* req) {
+  const std::string_view method = req->getCaseSensitiveMethod();
+  const i32 method_ptr = get_method_ptr(method);
+  if (method_ptr == 0) return;
+  __porffor_js_enter();
+  const i32 url_ptr = alloc_request_url(req);
   const i32 headers_ptr = collect_headers(req);
 }
 
@@ -159,6 +164,13 @@ test("#156: status-line fallback synthesizes a line for unlisted codes, idempote
     expect(once.match(/case 302: return "302 Found";/g)).toHaveLength(1);
     // Body-limit edit still rides along.
     expect(once).toContain("sb_request_body_max");
+    // #195: a valid transfer ticket is intercepted before JS request roots and
+    // the ordinary PendingRequest body allocation path.
+    expect(once).toContain("sb_r2_transfer_open");
+    expect(once).toContain("sb_r2_transfer_write");
+    expect(once).toContain("try_handle_r2_transfer(res, req, method)");
+    expect(once).toContain("sb_r2_transfer_download_open");
+    expect(once).toContain("sb_r2_transfer_download_v1");
 
     // #163: collect_headers takes res, drops a client-sent x-sb-remote-addr, and
     // appends the real peer; the call site passes res through.

package/src/patch.ts

@@ -413,6 +413,128 @@ const WRITE_RESPONSE_INJECT =
   "}";
 const WRITE_RESPONSE_MARKER = "porf_native_fetch_read_raw_bytes(body_value";
 
+// #195: reserve a direct-transfer ticket before native-fetch allocates the
+// ordinary in-memory PendingRequest. uWS then gives each wire chunk directly
+// to the runtime C ABI, which owns the temporary object file.
+const R2_TRANSFER_ANCHOR = "static void on_request(uWS::HttpResponse<false>* res, uWS::HttpRequest* req) {\n";
+const R2_TRANSFER_INJECT = `extern "C" {
+  struct sb_r2_transfer_ctx;
+  int sb_r2_transfer_open(const char*, const char*, size_t, sb_r2_transfer_ctx**);
+  int sb_r2_transfer_write(sb_r2_transfer_ctx*, const char*, size_t);
+  int sb_r2_transfer_finish(sb_r2_transfer_ctx*);
+  void sb_r2_transfer_abort(sb_r2_transfer_ctx*);
+  int sb_r2_transfer_download_open(const char*, const char*, sb_r2_transfer_ctx**);
+  size_t sb_r2_transfer_download_size(sb_r2_transfer_ctx*);
+  size_t sb_r2_transfer_download_read(sb_r2_transfer_ctx*, size_t, char*, size_t);
+  void sb_r2_transfer_download_close(sb_r2_transfer_ctx*);
+}
+static bool sb_r2_transfer_path(std::string_view path, std::string* bucket, std::string* token) {
+  static constexpr std::string_view prefix = "/__sb/r2/transfer/";
+  if (path.size() < prefix.size() || path.substr(0, prefix.size()) != prefix) return false;
+  const std::string_view rest = path.substr(prefix.size());
+  const size_t slash = rest.find('/');
+  if (slash == std::string_view::npos || slash == 0 || slash > 255 || rest.find('/', slash + 1) != std::string_view::npos) return false;
+  const std::string_view b = rest.substr(0, slash), t = rest.substr(slash + 1);
+  if (t.size() != 24) return false;
+  for (char c : b) if (!((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_')) return false;
+  for (char c : t) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return false;
+  *bucket = std::string(b); *token = std::string(t); return true;
+}
+static std::string sb_r2_transfer_status(int status) {
+  switch (status) {
+    case 201: return "201 Created"; case 404: return "404 Not Found"; case 410: return "410 Gone";
+    case 413: return "413 Payload Too Large"; case 422: return "422 Unprocessable Content";
+    case 503: return "503 Service Unavailable"; case 507: return "507 Insufficient Storage";
+    default: return "500 Internal Server Error";
+  }
+}
+static bool try_handle_r2_transfer(uWS::HttpResponse<false>* res, uWS::HttpRequest* req, std::string_view method) {
+  std::string bucket, token;
+  if (!sb_r2_transfer_path(req->getUrl(), &bucket, &token)) return false;
+  // sb_r2_transfer_download_v1
+  if (method == "GET") {
+    sb_r2_transfer_ctx* download = nullptr;
+    const int open_status = sb_r2_transfer_download_open(bucket.c_str(), token.c_str(), &download);
+    if (open_status != 0 || !download) { respond_with_error(res, sb_r2_transfer_status(open_status), "R2 transfer unavailable", true); return true; }
+    struct Download { sb_r2_transfer_ctx* ctx; uintmax_t size; bool closed = false; };
+    auto state = std::make_shared<Download>(Download{download, sb_r2_transfer_download_size(download)});
+    auto pump = std::make_shared<std::function<bool()>>();
+    *pump = [res, state, pump]() {
+      if (state->closed) return false;
+      const uintmax_t offset = res->getWriteOffset();
+      char chunk[65536];
+      const size_t bytes = sb_r2_transfer_download_read(state->ctx, (size_t)offset, chunk, sizeof(chunk));
+      const auto result = res->tryEnd(std::string_view(chunk, bytes), state->size);
+      if (result.second) { sb_r2_transfer_download_close(state->ctx); state->closed = true; return false; }
+      if (!result.first) res->onWritable([pump](uintmax_t) { return (*pump)(); });
+      return false;
+    };
+    res->writeHeader("Content-Type", "application/octet-stream");
+    res->onAborted([state]() { if (!state->closed) { sb_r2_transfer_download_close(state->ctx); state->closed = true; } });
+    (*pump)();
+    return true;
+  }
+  if (method != "PUT") { respond_with_error(res, "405 Method Not Allowed", "R2 transfer requires GET or PUT", true); return true; }
+  size_t declared = 0;
+  const std::string_view content_length = req->getHeader("content-length");
+  if (!content_length.empty() && !parse_content_length(content_length, &declared)) { respond_with_error(res, "400 Bad Request", "invalid content-length", true); return true; }
+  sb_r2_transfer_ctx* opened = nullptr;
+  const int open_status = sb_r2_transfer_open(bucket.c_str(), token.c_str(), declared, &opened);
+  if (open_status != 0 || !opened) { respond_with_error(res, sb_r2_transfer_status(open_status), "R2 transfer unavailable", true); return true; }
+  auto ctx = std::make_shared<sb_r2_transfer_ctx*>(opened);
+  res->onAborted([ctx]() { if (*ctx) { sb_r2_transfer_abort(*ctx); *ctx = nullptr; } });
+  res->onData([res, ctx](std::string_view chunk, bool is_last) {
+    if (!*ctx) return;
+    const int write_status = sb_r2_transfer_write(*ctx, chunk.data(), chunk.size());
+    if (write_status != 0) { sb_r2_transfer_abort(*ctx); *ctx = nullptr; respond_with_error(res, sb_r2_transfer_status(write_status), "R2 transfer failed", true); return; }
+    if (is_last) {
+      const int finish_status = sb_r2_transfer_finish(*ctx); *ctx = nullptr;
+      if (finish_status == 201) { res->writeStatus("201 Created"); res->end(); }
+      else respond_with_error(res, sb_r2_transfer_status(finish_status), "R2 transfer failed", true);
+    }
+  });
+  return true;
+}
+static void on_request(uWS::HttpResponse<false>* res, uWS::HttpRequest* req) {
+`;
+const R2_TRANSFER_MARKER = "sb_r2_transfer_path(std::string_view path";
+const R2_TRANSFER_CALL_ANCHOR = "  __porffor_js_enter();\n  const i32 url_ptr = alloc_request_url(req);";
+const R2_TRANSFER_CALL_INJECT =
+  "  if (try_handle_r2_transfer(res, req, method)) return;\n" + R2_TRANSFER_CALL_ANCHOR;
+const R2_TRANSFER_CALL_MARKER = "try_handle_r2_transfer(res, req, method)";
+const R2_DOWNLOAD_CACHE_MARKER = "sb_r2_transfer_download_v1";
+const R2_DOWNLOAD_DECL_ANCHOR = "  void sb_r2_transfer_abort(sb_r2_transfer_ctx*);\n}";
+const R2_DOWNLOAD_DECL_INJECT =
+  "  void sb_r2_transfer_abort(sb_r2_transfer_ctx*);\n" +
+  "  int sb_r2_transfer_download_open(const char*, const char*, sb_r2_transfer_ctx**);\n" +
+  "  size_t sb_r2_transfer_download_size(sb_r2_transfer_ctx*);\n" +
+  "  size_t sb_r2_transfer_download_read(sb_r2_transfer_ctx*, size_t, char*, size_t);\n" +
+  "  void sb_r2_transfer_download_close(sb_r2_transfer_ctx*);\n}";
+const R2_DOWNLOAD_BRANCH_ANCHOR = '  if (method != "PUT") { respond_with_error(res, "405 Method Not Allowed", "R2 transfer requires PUT", true); return true; }';
+const R2_DOWNLOAD_BRANCH_INJECT = `  // sb_r2_transfer_download_v1
+  if (method == "GET") {
+    sb_r2_transfer_ctx* download = nullptr;
+    const int status = sb_r2_transfer_download_open(bucket.c_str(), token.c_str(), &download);
+    if (status != 0 || !download) { respond_with_error(res, sb_r2_transfer_status(status), "R2 transfer unavailable", true); return true; }
+    struct Download { sb_r2_transfer_ctx* ctx; uintmax_t size; bool closed = false; };
+    auto state = std::make_shared<Download>(Download{download, sb_r2_transfer_download_size(download)});
+    auto pump = std::make_shared<std::function<bool()>>();
+    *pump = [res, state, pump]() {
+      if (state->closed) return false;
+      const uintmax_t offset = res->getWriteOffset(); char chunk[65536];
+      const size_t bytes = sb_r2_transfer_download_read(state->ctx, (size_t)offset, chunk, sizeof(chunk));
+      const auto result = res->tryEnd(std::string_view(chunk, bytes), state->size);
+      if (result.second) { sb_r2_transfer_download_close(state->ctx); state->closed = true; return false; }
+      if (!result.first) res->onWritable([pump](uintmax_t) { return (*pump)(); });
+      return false;
+    };
+    res->writeHeader("Content-Type", "application/octet-stream");
+    res->onAborted([state]() { if (!state->closed) { sb_r2_transfer_download_close(state->ctx); state->closed = true; } });
+    (*pump)();
+    return true;
+  }
+  if (method != "PUT") { respond_with_error(res, "405 Method Not Allowed", "R2 transfer requires GET or PUT", true); return true; }`;
+
 const done = new Set<string>();
 
 /** Edits to Porffor's uWebSockets shim: `[marker, anchor, inject, what]`. */
@@ -444,6 +566,8 @@ const UWS_EDITS = [
     WRITE_RESPONSE_INJECT,
     "detect x-sb-raw-body, steer the body read (#176)",
   ],
+  [R2_TRANSFER_MARKER, R2_TRANSFER_ANCHOR, R2_TRANSFER_INJECT, "native direct R2 transfer ingress (#195)"],
+  [R2_TRANSFER_CALL_MARKER, R2_TRANSFER_CALL_ANCHOR, R2_TRANSFER_CALL_INJECT, "direct R2 transfer dispatch (#195)"],
 ] as const;
 
 /** The uWebSockets shim source: body limit + the #156 status-line fix. Exported for tests. */
@@ -451,6 +575,16 @@ export async function patchUwebsockets(root: string): Promise<void> {
   const file = resolve(root, "compiler/uwebsockets.js");
   let src = await readFile(file, "utf8");
   let changed = false;
+  // Upgrade already-patched caches from the initial upload-only bridge. New
+  // caches receive the full bridge below; old ones need declarations plus the
+  // GET dispatch grafted onto their existing transfer handler.
+  if (src.includes(R2_TRANSFER_MARKER) && !src.includes(R2_DOWNLOAD_CACHE_MARKER)) {
+    if (!src.includes(R2_DOWNLOAD_DECL_ANCHOR) || !src.includes(R2_DOWNLOAD_BRANCH_ANCHOR))
+      throw new Error(`could not upgrade Porffor's direct R2 transfer bridge: anchor not found in ${file}.`);
+    src = src.replace(R2_DOWNLOAD_DECL_ANCHOR, R2_DOWNLOAD_DECL_INJECT);
+    src = src.replace(R2_DOWNLOAD_BRANCH_ANCHOR, R2_DOWNLOAD_BRANCH_INJECT);
+    changed = true;
+  }
   for (const [marker, anchor, inject, what] of UWS_EDITS) {
     if (src.includes(marker)) continue;
     if (!src.includes(anchor)) {