edgeway 0.1.0 → 0.2.0

data/js/cloudflare.js

@@ -5,20 +5,101 @@
 //   export default createWorker(app);
 //
 // build/edgeway/config.js is written by `edgeway build` from
-// config/database.yml: the environment name and the D1 binding.
+// config/database.yml: the environment name, the D1 binding, and the bindings
+// that use D1 Sessions.
 import { runCgi } from "./runtime.js";
 import config from "./config.js";
 import { jwtImports } from "./jwt.js";
 import { withCors } from "./cors.js";
+import { BOOKMARK_HEADER, D1Sessions } from "./cloudflare-d1-sessions.js";
+
+// D1's transient errors, which are worth a retry by the client
+// (developers.cloudflare.com/d1/observability/debug-d1/). D1 retries read-only
+// queries itself; Edgeway never retries, as a write is not idempotent.
+// "exceeded", "CPU time limit", "memory limit" and "timeout" are not here.
+export const TRANSIENT_D1_ERRORS = [
+  "Network connection lost",
+  "reset because its code was updated",
+  "storage caused object to be reset",
+  "Replica disconnected from primary",
+  "transient issue on remote node",
+  "Can't read from request stream because client disconnected",
+  "D1 DB is overloaded",
+];
+
+export const isTransientD1Error = (message) => TRANSIENT_D1_ERRORS.some((s) => message.includes(s));
+
+// Counts are 32-bit in wasm: a larger one is clamped.
+const MAX_COUNT = 2147483647;
+
+// D1's result.meta in the shape of META above. A local D1 (wrangler dev) may
+// leave fields out.
+export function metaOf(meta) {
+  return {
+    rows_read: Math.min(meta?.rows_read ?? 0, MAX_COUNT),
+    rows_written: Math.min(meta?.rows_written ?? 0, MAX_COUNT),
+    duration: meta?.duration ?? 0,
+    served_by_region: meta?.served_by_region ?? "",
+    served_by_primary: meta?.served_by_primary ?? true,
+  };
+}
+
+export function sumMeta(metas) {
+  const all = metas.map(metaOf);
+  const sum = (field) => all.reduce((total, meta) => total + meta[field], 0);
+  return {
+    ...all[all.length - 1],
+    rows_read: Math.min(sum("rows_read"), MAX_COUNT),
+    rows_written: Math.min(sum("rows_written"), MAX_COUNT),
+    duration: sum("duration"),
+  };
+}
+
+// A BLOB crosses as {"$blob": "<base64>"}. D1 binds an ArrayBuffer as a BLOB and
+// answers a BLOB column as an Array of byte numbers.
+export function bindOf(value) {
+  if (value === null || typeof value !== "object" || typeof value.$blob !== "string") return value;
+  return Uint8Array.from(atob(value.$blob), (char) => char.charCodeAt(0)).buffer;
+}
+
+const isBytes = (value) => Array.isArray(value) || value instanceof ArrayBuffer || ArrayBuffer.isView(value);
+
+function bytesOf(value) {
+  if (Array.isArray(value)) return Uint8Array.from(value);
+  if (ArrayBuffer.isView(value)) return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+  return new Uint8Array(value);
+}
+
+function base64Of(value) {
+  const bytes = bytesOf(value);
+  let binary = "";
+  for (let at = 0; at < bytes.length; at += 8192) binary += String.fromCharCode(...bytes.subarray(at, at + 8192));
+  return btoa(binary);
+}
+
+export function rowOf(row) {
+  for (const column of Object.keys(row)) {
+    const value = row[column];
+    if (value !== null && typeof value === "object" && isBytes(value)) row[column] = { $blob: base64Of(value) };
+  }
+  return row;
+}
 
 // The wasm import module "edgeway_d1" (lib/edgeway/service/cloudflare/d1.rb). Each call
 // answers the byte length of its JSON result; take() copies it out.
-//   query(binding, sql, binds)   -> {"rows": [...]}
-//   execute(binding, sql, binds) -> {"changes": n, "last_insert_id": id}
-//   batch(binding, statements)   -> {"results": [{"changes": n, "last_insert_id": id}, ...]}
+//   query(binding, sql, binds)   -> {"rows": [...], "meta": META}
+//   execute(binding, sql, binds) -> {"changes": n, "last_insert_id": id, "meta": META}
+//   exec(binding, sql)           -> {"count": n, "meta": META}
+//                                   (sql: statements separated by newlines, no binds; an error on a database with sessions)
+//   batch(binding, statements)   -> {"results": [{"changes": n, "last_insert_id": id}, ...], "meta": META}
 //                                   (statements: [[sql, binds], ...], one atomic D1 batch)
-// An error is {"error": "message"}.
-export function d1Imports(env, getInstance) {
+// META is {"rows_read": n, "rows_written": n, "duration": ms, "served_by_region": "...",
+// "served_by_primary": bool}; a batch's sums the statements'. exec's has only
+// rows_read, rows_written and duration.
+// An error is {"error": "message"} (no meta), with "unavailable": true when it is transient.
+const SESSION_EXEC_ERROR = "exec() is not available on a database with sessions: true (a D1 session has no exec)";
+
+export function d1Imports(env, getInstance, sessions = new D1Sessions(env)) {
   const mem = () => new Uint8Array(getInstance().exports.memory.buffer);
   const cstr = (p) => {
     const b = mem();
@@ -29,36 +110,47 @@ export function d1Imports(env, getInstance) {
   let pending = new Uint8Array(0);
 
   // fn(db, ...pointers) answers the result object; a failure is {error}.
-  const suspending = (fn) => new WebAssembly.Suspending(async (bindingPtr, ...ptrs) => {
+  const suspending = (fn, { withoutSessions = false } = {}) => new WebAssembly.Suspending(async (bindingPtr, ...ptrs) => {
     let res;
     try {
       const name = cstr(bindingPtr) || config.d1Binding;
-      const db = env[name];
+      if (withoutSessions && sessions.uses(name)) throw new Error(SESSION_EXEC_ERROR);
+      const db = sessions.database(name);
       if (!db) throw new Error(`no D1 binding ${name}`);
       res = await fn(db, ...ptrs);
     } catch (e) {
-      res = { error: String(e?.message ?? e) };
+      const error = String(e?.message ?? e);
+      res = isTransientD1Error(error) ? { error, unavailable: true } : { error };
     }
     pending = new TextEncoder().encode(JSON.stringify(res));
     return pending.length;
   });
+  const prepare = (db, sql, binds) => db.prepare(sql).bind(...binds.map(bindOf));
   const run = (fn) => suspending((db, sqlPtr, bindsPtr) =>
-    fn(db.prepare(cstr(sqlPtr)).bind(...JSON.parse(cstr(bindsPtr)))));
+    fn(prepare(db, cstr(sqlPtr), JSON.parse(cstr(bindsPtr)))));
 
   return {
     edgeway_d1: {
-      query: run(async (st) => ({ rows: (await st.all()).results })),
-      execute: run(async (st) => {
-        const meta = (await st.run()).meta;
-        return { changes: meta.changes ?? 0, last_insert_id: meta.last_row_id ?? 0 };
+      query: run(async (statement) => {
+        const result = await statement.all();
+        return { rows: result.results.map(rowOf), meta: metaOf(result.meta) };
       }),
+      execute: run(async (statement) => {
+        const meta = (await statement.run()).meta;
+        return { changes: meta.changes ?? 0, last_insert_id: meta.last_row_id ?? 0, meta: metaOf(meta) };
+      }),
+      exec: suspending(async (db, sqlPtr) => {
+        const result = await db.exec(cstr(sqlPtr));
+        return { count: result.count ?? 0, meta: { rows_read: 0, rows_written: 0, duration: result.duration ?? 0 } };
+      }, { withoutSessions: true }),
       // db.batch() is one transaction: a failing statement rejects it and
       // none of the statements stay applied.
       batch: suspending(async (db, statementsPtr) => {
         const statements = JSON.parse(cstr(statementsPtr));
-        const results = await db.batch(statements.map(([sql, binds]) => db.prepare(sql).bind(...binds)));
+        const results = await db.batch(statements.map(([sql, binds]) => prepare(db, sql, binds)));
         return {
-          results: results.map((r) => ({ changes: r.meta.changes ?? 0, last_insert_id: r.meta.last_row_id ?? 0 })),
+          results: results.map((result) => ({ changes: result.meta.changes ?? 0, last_insert_id: result.meta.last_row_id ?? 0 })),
+          meta: sumMeta(results.map((result) => result.meta)),
         };
       }),
       take(buf) {
@@ -90,14 +182,23 @@ export function varsOf(env) {
   return vars;
 }
 
+export function corsOptions(sessionBindings) {
+  const names = sessionBindings.length > 0 ? [BOOKMARK_HEADER] : [];
+  return { exposeHeaders: names, allowHeaders: names };
+}
+
 // appModule: the WebAssembly.Module that `import app from "...app.wasm"` gives.
 export function createWorker(appModule) {
   return {
-    fetch: withCors((request, env) => runCgi(appModule, request, {
-      extraEnv: { EDGEWAY_ENV: config.env, ...varsOf(env) },
-      // Both modules are always supplied, whether or not the app uses them:
-      // the wasm instance needs every import it declares.
-      imports: (getInstance) => ({ ...d1Imports(env, getInstance), ...jwtImports(env, getInstance) }),
-    })),
+    fetch: withCors(async (request, env) => {
+      const sessions = new D1Sessions(env, request, config.sessions ?? []);
+      const response = await runCgi(appModule, request, {
+        extraEnv: { EDGEWAY_ENV: config.env, ...varsOf(env) },
+        // Both modules are always supplied, whether or not the app uses them:
+        // the wasm instance needs every import it declares.
+        imports: (getInstance) => ({ ...d1Imports(env, getInstance, sessions), ...jwtImports(env, getInstance) }),
+      });
+      return sessions.annotate(response);
+    }, corsOptions(config.sessions ?? [])),
   };
 }

data/js/cors.js

@@ -4,6 +4,8 @@
 // status instead of an opaque CORS error. Nothing here is specific to a
 // platform. Cookies are not supported: Access-Control-Allow-Credentials is
 // never sent.
+import { addVary, withHeaders } from "./headers.js";
+
 const DEFAULT_HEADERS = "Authorization, Content-Type";
 const DEFAULT_MAX_AGE = "600";
 const METHODS = "GET, HEAD, POST, PUT, PATCH, DELETE";
@@ -15,17 +17,17 @@ function json(status, body, headers = {}) {
 // The settings from the Worker's variables, or null when CORS is off.
 // A bad CORS_MAX_AGE is { error }: like JWT, a misconfiguration is a 500, not
 // a silent default.
-function settings(env) {
-  const list = (env.CORS_ORIGINS ?? "").split(",").map((s) => s.trim()).filter(Boolean);
+function settings(env, { exposeHeaders, allowHeaders }) {
+  const list = (env.CORS_ORIGINS ?? "").split(",").map((origin) => origin.trim()).filter(Boolean);
   if (list.length === 0) return null;
   const maxAge = (env.CORS_MAX_AGE ?? "").trim() || DEFAULT_MAX_AGE;
   if (!/^\d+$/.test(maxAge)) return { error: `CORS_MAX_AGE must be a non-negative integer, got "${env.CORS_MAX_AGE}"` };
   return {
     any: list.includes("*"),
     origins: new Set(list),
-    headers: (env.CORS_HEADERS ?? "").trim() || DEFAULT_HEADERS,
+    headers: withHeaders((env.CORS_HEADERS ?? "").trim() || DEFAULT_HEADERS, ...allowHeaders),
     maxAge,
-    expose: (env.CORS_EXPOSE_HEADERS ?? "").trim(),
+    expose: withHeaders(env.CORS_EXPOSE_HEADERS, ...exposeHeaders),
   };
 }
 
@@ -36,20 +38,17 @@ function allowed(cfg, origin) {
   return cfg.any || cfg.origins.has(origin);
 }
 
-// Appends Origin to Vary unless it (or *) is already there.
-function addVary(headers, ...names) {
-  const have = (headers.get("Vary") ?? "").split(",").map((s) => s.trim()).filter(Boolean);
-  if (have.includes("*")) return;
-  for (const n of names) {
-    if (!have.some((h) => h.toLowerCase() === n.toLowerCase())) have.push(n);
-  }
-  headers.set("Vary", have.join(", "));
+function exposeOn(headers, expose) {
+  const merged = withHeaders(headers.get("Access-Control-Expose-Headers"), ...expose.split(", ").filter(Boolean));
+  if (merged) headers.set("Access-Control-Expose-Headers", merged);
 }
 
-// Wraps a fetch handler (request, env, ctx) -> Response.
-export function withCors(handler) {
+// Wraps a fetch handler (request, env, ctx) -> Response. exposeHeaders and
+// allowHeaders are names the platform layer always adds to the exposed and the
+// preflight-allowed headers.
+export function withCors(handler, { exposeHeaders = [], allowHeaders = [] } = {}) {
   return async (request, env, ctx) => {
-    const cfg = settings(env);
+    const cfg = settings(env, { exposeHeaders, allowHeaders });
     if (!cfg) return handler(request, env, ctx);
     if (cfg.error) {
       console.error(cfg.error);
@@ -79,7 +78,7 @@ export function withCors(handler) {
     const out = new Response(res.body, res);
     if (ok) {
       out.headers.set("Access-Control-Allow-Origin", origin);
-      if (cfg.expose) out.headers.set("Access-Control-Expose-Headers", cfg.expose);
+      exposeOn(out.headers, cfg.expose);
     }
     addVary(out.headers, "Origin");
     return out;

data/js/headers.js

@@ -0,0 +1,16 @@
+const listOf = (value) => (value ?? "").split(",").map((item) => item.trim()).filter(Boolean);
+
+export function withHeaders(list, ...names) {
+  const have = listOf(list);
+  for (const name of names) {
+    if (!have.some((existing) => existing.toLowerCase() === name.toLowerCase())) have.push(name);
+  }
+  return have.join(", ");
+}
+
+// A Vary of * already covers every header.
+export function addVary(headers, ...names) {
+  const have = listOf(headers.get("Vary"));
+  if (have.includes("*")) return;
+  headers.set("Vary", withHeaders(have.join(", "), ...names));
+}

data/js/jwt.js

@@ -245,7 +245,7 @@ export function jwtVerifier(env, { fetch = globalThis.fetch, now = Date.now, cac
 //   verify(token) -> {"header": {...}, "payload": {...}}
 //                  | {"error": "message"}                      the token is bad
 //                  | {"error": "message", "unavailable": true}  it can't be checked
-// Like D1, a call answers the byte length of the result and take() copies it
+// Like the other imports, a call answers the byte length of the result and take() copies it
 // out; verify is wrapped in WebAssembly.Suspending (the WebCrypto and fetch
 // Promises are awaited with the wasm stack suspended).
 export function jwtImports(env, getInstance, options = {}) {

data/lib/edgeway/application.rb

@@ -38,6 +38,11 @@ module Edgeway
         # The message names the table and columns: the log only.
         Edgeway.logger.info "409 #{e.message}"
         message(409, "record not unique")
+      elsif e.is_a?(DatabaseUnavailable)
+        Edgeway.logger.error "503 #{e.message}"
+        response = message(503, "database unavailable")
+        response.headers["Retry-After"] = "1"
+        response
       elsif e.is_a?(RoutingError) || e.is_a?(RecordNotFound)
         # The message can hold the path or an id from the request: the log only.
         Edgeway.logger.info "404 #{e.message}"
@@ -58,9 +63,7 @@ module Edgeway
     end
 
     def message(status, text)
-      # .dup: since matz/spinel 44119ab30 (Strings shared by reference),
-      # Spinel refuses this String as it is, or emits C that doesn't compile.
-      Controller::Rendering.json_response(status, JSON.generate({ "error" => text.dup }))
+      Controller::Rendering.json_response(status, JSON.generate({ "error" => text }))
     end
   end
 

data/lib/edgeway/cli/build/app_config.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "database_config"
+
+module Edgeway
+  module CLI
+    class Build
+      class AppConfig
+        # adapter: => the file build/main.rb requires for it.
+        ADAPTERS = {
+          "d1" => "edgeway/service/cloudflare/d1",
+          "memory" => "edgeway/db/memory"
+        }.freeze
+
+        attr_reader :env, :adapter, :binding_name, :databases, :sessions
+
+        def self.load(root, env)
+          config = DatabaseConfig.new(root)
+          section = config.section(env)
+          databases = config.databases_in(section, env)
+          sessions = config.session_bindings(section, env)
+          new(env, section["adapter"].to_s, config.primary_binding(section), databases, sessions)
+        end
+
+        def initialize(env, adapter, binding_name, databases = {}, sessions = [])
+          raise Error, "unknown database adapter #{adapter.inspect} (one of #{ADAPTERS.keys.join(", ")})" unless ADAPTERS.key?(adapter)
+
+          @env = env
+          @adapter = adapter
+          @binding_name = binding_name
+          @databases = databases
+          @sessions = adapter == "d1" ? sessions : []
+        end
+
+        def adapter_file = ADAPTERS.fetch(@adapter)
+      end
+    end
+  end
+end

data/lib/edgeway/cli/build/app_files.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Edgeway
+  module CLI
+    class Build
+      class AppFiles
+        def initialize(root)
+          @root = root
+        end
+
+        def require_lines(views: nil)
+          lines = []
+          app = File.join(@root, "app")
+          dirs = Dir.exist?(app) ? Dir.children(app).sort.select { |d| File.directory?(File.join(app, d)) } : []
+          dirs.each do |d|
+            next if %w[models controllers].include?(d)
+
+            lines.concat(requires("app/#{d}"))
+          end
+          lines.concat(requires("app/models"))
+          lines.concat(requires("app/controllers"))
+          lines << "require_relative #{views.dump}" if views
+          lines << 'require_relative "../config/routes"'
+          lines
+        end
+
+        # require_relative lines for the .rb files under dir, application_*.rb first.
+        def requires(dir)
+          full = File.join(@root, dir)
+          return [] unless Dir.exist?(full)
+
+          files = Dir.glob("**/*.rb", base: full).map { |f| File.join(dir, f) }
+          first, rest = files.partition { |f| File.basename(f).start_with?("application_") }
+          # String#dump quotes a path with spaces, quotes or "#{" safely.
+          (first.sort + rest.sort).map { |f| "require_relative #{"../#{f.delete_suffix(".rb")}".dump}" }
+        end
+      end
+    end
+  end
+end

data/lib/edgeway/cli/build/database_config.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Edgeway
+  module CLI
+    class Build
+      class DatabaseConfig
+        DATABASE_NAME = /\A[a-z_][a-z0-9_]*\z/
+
+        def initialize(root)
+          @root = root
+        end
+
+        def section(env)
+          section = sections[env]
+          raise Error, "config/database.yml has no #{env} section" unless section.is_a?(Hash)
+
+          section
+        end
+
+        def databases_in(section, env)
+          entries = section["databases"] || {}
+          raise Error, "config/database.yml: databases in #{env} must be a Hash" unless entries.is_a?(Hash)
+
+          entries.to_h do |name, entry|
+            name = name.to_s
+            [name, database_binding(name, entry, env)]
+          end
+        end
+
+        def primary_binding(section) = (section["binding"] || "DB").to_s
+
+        def session_bindings(section, env)
+          flags = {}
+          record_sessions(flags, "the #{env} section", primary_binding(section), section, "the #{env} section")
+          (section["databases"] || {}).each do |name, entry|
+            record_sessions(flags, "databases.#{name}", entry["binding"].to_s, entry, "databases.#{name} in #{env}")
+          end
+          flags.select { |_binding, (_label, enabled)| enabled }.keys
+        end
+
+        def names
+          names = []
+          return names unless File.exist?(path)
+
+          sections.each do |env, section|
+            next unless section.is_a?(Hash)
+            databases_in(section, env).each_key { |name| names << name unless names.include?(name) }
+          end
+          names
+        end
+
+        private
+
+        def path = File.join(@root, "config", "database.yml")
+
+        def sections
+          raise Error, "no config/database.yml in #{@root} (is this an Edgeway app?)" unless File.exist?(path)
+
+          YAML.safe_load_file(path) || {}
+        end
+
+        def record_sessions(flags, label, binding_name, entry, where)
+          enabled = sessions?(entry, where)
+          known_label, known = flags[binding_name]
+          if known_label && known != enabled
+            raise Error, "config/database.yml: #{known_label} and #{label} share binding #{binding_name} but differ in sessions"
+          end
+
+          flags[binding_name] = [label, enabled]
+        end
+
+        def sessions?(entry, where)
+          return false unless entry.is_a?(Hash) && entry.key?("sessions")
+          return entry["sessions"] if [true, false].include?(entry["sessions"])
+
+          raise Error, "config/database.yml: sessions in #{where} must be true or false"
+        end
+
+        def database_binding(name, entry, env)
+          unless name.match?(DATABASE_NAME) && name != "primary"
+            raise Error, "config/database.yml: bad database name #{name.inspect} in #{env} (lowercase letters, digits and _, and not primary)"
+          end
+
+          binding_name = entry.is_a?(Hash) ? entry["binding"].to_s : ""
+          raise Error, "config/database.yml: databases.#{name} in #{env} has no binding:" if binding_name.empty?
+
+          binding_name
+        end
+      end
+    end
+  end
+end