@voidbase-cloud/voidbase 0.1.0

package/src/server/realtime/index.ts

@@ -0,0 +1,239 @@
+// Realtime (decision d5): PocketBase's SSE protocol on void/sse. Two transports behind the same protocol:
+//
+// - the hub (HUB binding, src/server/hub.ts): every connection holds one hibernatable WebSocket to the instance's
+//   Durable Object, record writes publish to it, subscription changes are relayed through it. Push, no polling,
+//   nothing shared between instances.
+// - the D1 change feed (no binding: the Bun runtime, a deploy without the hub): every connection runs a poll loop
+//   inside its request; the loops in one isolate share the last query result (plain data), about one D1 read per
+//   second per isolate, zero when nobody is connected.
+//
+// Workers bind timers and I/O objects to the request that created them, so both the poll loop and the hub socket
+// live inside the SSE request, and a stream is only ever written from its own request.
+import type { Context } from "hono";
+import { eventStream } from "#platform/sse";
+import type { Collection } from "../collections/model";
+import { loadCollections } from "../collections/model";
+import { one, stmt } from "../db";
+import { badRequest, notFound } from "../errors";
+import { nowString, randomString } from "../ids";
+import { findAuthRecordByToken, isSuperuser } from "../auth";
+import { enrich, fetchRecord, recordMatchesRule, type RecordContext } from "../records/service";
+import { rowToValues } from "../records/values";
+import { trigger } from "../hooks/runtime";
+import type { AppEnv, Row } from "../types";
+import { controlClient, hubActive, openHubSocket, sendFilter, type ChangeEvent, type HubMessage } from "./hub-client";
+
+interface Subscription { topic: string; collection: string; recordId: string | null; query: Record<string, string>; headers: Record<string, string> }
+interface Change { id: number; collection: string; recordId: string; action: string; data: string | null }
+interface Client {
+  id: string;
+  cursor: number;
+  subs: Subscription[];
+  token: string;
+  send: (event: string, data: unknown) => Promise<void>;
+  closed: boolean;
+  hub?: WebSocket | null;
+}
+
+const clients = new Map<string, Client>();
+const POLL_MS = 1000;
+const CHANGES_LIMIT = 500;
+// last change-feed read in this isolate, shared by every connection loop (data only, never I/O objects)
+let lastRead: { at: number; since: number; changes: Change[] } | null = null;
+
+export function parseSubscription(raw: string): Subscription | null {
+  const [topicPart, optionsPart] = raw.split("?options=");
+  const topic = topicPart ?? "";
+  if (topic.startsWith("@")) return { topic: raw, collection: topic, recordId: null, query: {}, headers: {} }; // @oauth2 and friends: messages, not records
+  const m = /^([^/]+)\/(.+)$/.exec(topic);
+  if (!m) return null;
+  let query: Record<string, string> = {}, headers: Record<string, string> = {};
+  if (optionsPart) {
+    try {
+      const o = JSON.parse(decodeURIComponent(optionsPart)) as { query?: Record<string, string>; headers?: Record<string, string> };
+      query = o.query ?? {};
+      headers = Object.fromEntries(Object.entries(o.headers ?? {}).map(([k, v]) => [k.toLowerCase(), String(v)]));
+    } catch { /* ignore malformed options */ }
+  }
+  return { topic: raw, collection: m[1]!, recordId: m[2] === "*" ? null : m[2]!, query, headers };
+}
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+// GET /api/realtime
+export async function connect(c: Context<AppEnv>): Promise<Response> {
+  const env = c.env;
+  const clientId = randomString(40);
+  const token = c.req.header("Authorization")?.replace(/^bearer /i, "") ?? "";
+  const now = nowString();
+  await stmt(env.DB, "INSERT INTO `_realtime_clients` (id, subscriptions, token, created, updated) VALUES (?, '[]', ?, ?, ?)", [clientId, token, now, now]).run();
+  const useHub = hubActive();
+  const max = useHub ? null : await one<{ m: number | null }>(env.DB, "SELECT MAX(id) AS m FROM `_changes`");
+  const cursor = max?.m ?? 0;
+  return eventStream(
+    async (stream) => {
+      const client: Client = {
+        id: clientId, cursor, subs: [], token, closed: false,
+        send: async (event, data) => { await stream.send({ event, data }); },
+      };
+      clients.set(clientId, client);
+      let hubSocket: WebSocket | null = null;
+      if (useHub) {
+        // the socket belongs to this request, like the stream it feeds; if the hub is unreachable the stream ends
+        // and the SDK reconnects, since writers publish there instead of writing feed rows
+        try { hubSocket = await openHubSocket(clientId); client.hub = hubSocket; } catch (err) { console.error("voidbase: realtime hub unreachable", err); stream.close(); return; }
+        hubSocket.addEventListener("message", (e) => { void onHubMessage(env, client, String(e.data)).catch((err) => { if (!client.closed) console.error("voidbase: realtime hub message failed", err); }); });
+        hubSocket.addEventListener("close", () => { if (!client.closed) stream.close(); });
+        hubSocket.addEventListener("error", () => { if (!client.closed) stream.close(); });
+        // a torn-down request may never reach the cleanup below: close the hub socket on abort as well
+        c.req.raw.signal.addEventListener("abort", () => { try { hubSocket?.close(1000, "client gone"); } catch { /* gone */ } });
+      }
+      await stream.send({ id: clientId, event: "PB_CONNECT", data: { clientId } });
+      const loop = useHub ? (async () => {
+        // a keepalive the object answers without waking; the race lets the loop end as soon as the stream does
+        while (!client.closed) { await Promise.race([sleep(60_000), stream.closed]); if (client.closed) break; try { hubSocket?.send("ping"); } catch { stream.close(); } }
+      })() : (async () => {
+        while (!client.closed) {
+          await sleep(POLL_MS);
+          if (client.closed) break;
+          try { await pollOne(env, client); } catch (err) { if (!client.closed) console.error("voidbase: realtime poll failed", err); }
+        }
+      })();
+      await stream.closed;
+      client.closed = true;
+      clients.delete(clientId);
+      try { hubSocket?.close(); } catch { /* already closed */ }
+      await loop.catch(() => {});
+      try { await stmt(env.DB, "DELETE FROM `_realtime_clients` WHERE id = ?", [clientId]).run(); } catch { /* best effort */ }
+    },
+    { signal: c.req.raw.signal, keepAlive: { intervalMs: 15000, comment: "" } },
+  );
+}
+
+// a frame from the hub: changes to deliver, this client's new subscriptions, or a one-off message
+async function onHubMessage(env: AppEnv["Bindings"], cl: Client, raw: string) {
+  if (raw === "pong") return;
+  const msg = JSON.parse(raw) as HubMessage;
+  if (msg.t === "subs") { applySubscriptions(cl, msg.subscriptions, msg.token); await announceFilter(env, cl); return; }
+  if (msg.t === "message") {
+    if (!cl.subs.some((s) => s.topic === msg.event)) return;
+    await cl.send(msg.event, msg.data ?? {});
+    cl.subs = cl.subs.filter((s) => s.topic !== msg.event);
+    await stmt(env.DB, "UPDATE `_realtime_clients` SET subscriptions = ?, updated = ? WHERE id = ?", [JSON.stringify(cl.subs.map((s) => s.topic)), nowString(), cl.id]).run();
+    return;
+  }
+  if (msg.t === "changes") {
+    const collections = await loadCollections(env.DB);
+    for (const ch of msg.changes) { if (cl.closed) return; await dispatch(env, cl, { id: 0, collection: ch.collection, recordId: ch.recordId, action: ch.action, data: ch.data ? JSON.stringify(ch.data) : null }, collections); }
+  }
+}
+function applySubscriptions(cl: Client, subs: string[], token: string) {
+  cl.subs = subs.map(parseSubscription).filter((s): s is Subscription => !!s);
+  cl.token = token;
+}
+// tell the hub which collections this connection wants (names; ids are resolved), so it skips the rest
+async function announceFilter(env: AppEnv["Bindings"], cl: Client) {
+  const ws = cl.hub; if (!ws) return;
+  const collections = await loadCollections(env.DB);
+  const names = new Set<string>();
+  for (const s of cl.subs) { if (s.collection.startsWith("@")) continue; const col = collections.get(s.collection); names.add(col ? col.name : s.collection); }
+  sendFilter(ws, [...names]);
+}
+
+// POST /api/realtime  {clientId, subscriptions: []}
+export async function setSubscriptions(c: Context<AppEnv>, pre?: { clientId?: string; subscriptions?: string[] }): Promise<Response> {
+  let body: { clientId?: string; subscriptions?: string[] } = pre ?? {};
+  if (!pre) {
+    try {
+      const ct = c.req.header("content-type") ?? "";
+      body = ct.includes("json") ? await c.req.json() : (Object.fromEntries((await c.req.formData()).entries()) as unknown as typeof body);
+    } catch { throw badRequest(); }
+  }
+  const clientId = String(body.clientId ?? "");
+  const subs = Array.isArray(body.subscriptions) ? body.subscriptions.map(String) : [];
+  if (!clientId) throw badRequest("An error occurred while validating the submitted data.", { clientId: { code: "validation_required", message: "Cannot be blank." } });
+  const row = await one(c.env.DB, "SELECT id FROM `_realtime_clients` WHERE id = ?", [clientId]);
+  if (!row) throw notFound("Missing or invalid client id.");
+  const token = c.req.header("Authorization")?.replace(/^bearer /i, "") ?? "";
+  await stmt(c.env.DB, "UPDATE `_realtime_clients` SET subscriptions = ?, token = ?, updated = ? WHERE id = ?", [JSON.stringify(subs), token, nowString(), clientId]).run();
+  const local = clients.get(clientId);
+  if (local) { applySubscriptions(local, subs, token); if (local.hub) await announceFilter(c.env, local); }
+  else if (hubActive()) await controlClient(clientId, subs, token); // the stream lives in another isolate
+  return c.body(null, 204);
+}
+
+// One tick for one connection: refresh its subscriptions (they may have been set through another isolate),
+// read the change feed past its cursor (reusing this isolate's last read when it covers the range) and deliver.
+async function pollOne(env: AppEnv["Bindings"], cl: Client) {
+  const db = env.DB;
+  const now = Date.now();
+  const reuse = lastRead && now - lastRead.at < POLL_MS && lastRead.since <= cl.cursor && lastRead.changes.length < CHANGES_LIMIT ? lastRead : null;
+  const statements = [stmt(db, "SELECT subscriptions, token FROM `_realtime_clients` WHERE id = ?", [cl.id])];
+  if (!reuse) statements.push(stmt(db, "SELECT id, collection, recordId, action, data FROM `_changes` WHERE id > ? ORDER BY id ASC LIMIT ?", [cl.cursor, CHANGES_LIMIT]));
+  const results = await db.batch(statements);
+  const me = (results[0]?.results?.[0] ?? null) as { subscriptions: string; token: string } | null;
+  if (me) {
+    cl.subs = (JSON.parse(me.subscriptions || "[]") as string[]).map(parseSubscription).filter((s): s is Subscription => !!s);
+    cl.token = me.token;
+  }
+  let changes: Change[];
+  if (reuse) changes = reuse.changes.filter((ch) => ch.id > cl.cursor);
+  else {
+    changes = (results[1]?.results ?? []) as unknown as Change[];
+    lastRead = { at: now, since: cl.cursor, changes };
+  }
+  if (!changes.length) return;
+  const collections = await loadCollections(db);
+  for (const ch of changes) {
+    if (cl.closed) return;
+    cl.cursor = Math.max(cl.cursor, ch.id);
+    await dispatch(env, cl, ch, collections);
+  }
+}
+
+// one change for one connection: the OAuth2 hand-off, or every matching subscription through the rules
+async function dispatch(env: AppEnv["Bindings"], cl: Client, ch: Change, collections: Map<string, Collection>) {
+  const db = env.DB;
+  if (ch.collection.startsWith("@")) { // one-off message for a single client (OAuth2 redirect handoff)
+    if (ch.recordId === cl.id && cl.subs.some((s) => s.topic === ch.collection)) {
+      await cl.send(ch.collection, ch.data ? JSON.parse(ch.data) : {});
+      cl.subs = cl.subs.filter((s) => s.topic !== ch.collection);
+      await stmt(db, "UPDATE `_realtime_clients` SET subscriptions = ?, updated = ? WHERE id = ?", [JSON.stringify(cl.subs.map((s) => s.topic)), nowString(), cl.id]).run();
+    }
+    return;
+  }
+  const matching = cl.subs.filter((s) => (s.collection === ch.collection || collections.get(s.collection)?.name === ch.collection) && (s.recordId === null || s.recordId === ch.recordId));
+  if (!matching.length) return;
+  const collection = collections.get(ch.collection);
+  if (!collection) return;
+  for (const sub of matching) {
+    try { await deliver(db, env, cl, sub, collection, ch, collections); } catch (err) { if (!cl.closed) console.error("voidbase: realtime deliver failed", err); }
+  }
+}
+
+async function deliver(db: D1Database, bindings: AppEnv["Bindings"], cl: Client, sub: Subscription, collection: Collection, ch: Change, collections: Map<string, Collection>) {
+  const token = sub.headers.authorization?.replace(/^bearer /i, "") || cl.token;
+  const auth = token ? await findAuthRecordByToken(db, token) : null;
+  const ctx: RecordContext = {
+    db, storage: bindings.STORAGE, auth, superuser: isSuperuser(auth),
+    request: { auth: auth ? { collection: auth.collection, row: auth.row } : null, method: "GET", query: sub.query, headers: sub.headers, body: {}, context: "realtime" },
+    collections,
+  };
+  const rule = sub.recordId === null ? collection.listRule : collection.viewRule;
+  let row: Row | null = null;
+  if (ch.action === "delete") {
+    if (!ch.data) return;
+    row = JSON.parse(ch.data) as Row;
+    if (!ctx.superuser) {
+      if (rule === null) return;
+      if (rule.trim() !== "" && !(await recordMatchesRule(ctx, collection, rule, rowToValues(collection, row)))) return;
+    }
+  } else {
+    try { row = await fetchRecord(ctx, collection, ch.recordId, rule); } catch { row = null; }
+    if (!row) return; // gone, or not visible to this subscriber
+  }
+  const [record] = await enrich(ctx, collection, [row], { expand: sub.query.expand, fields: sub.query.fields });
+  if (cl.closed) return;
+  const ev = { app: undefined as unknown, client: { id: cl.id, subscriptions: cl.subs.map((s) => s.topic) }, message: { name: sub.topic, data: { action: ch.action, record } }, next: async () => undefined as unknown };
+  await trigger("onRealtimeMessageSend", ev, null, async () => { if (!cl.closed) await cl.send(ev.message.name, ev.message.data); });
+}

package/src/server/records/expand.ts

@@ -0,0 +1,129 @@
+// Relation expansion (core/record_query_expand.go): direct and nested paths, back-relations `coll_via_field`,
+// batched fetches with the target collection's view rule applied, depth capped at 6.
+import { isMultiple, type Field } from "../collections/fields";
+import type { Collection } from "../collections/model";
+import { all, ident } from "../db";
+import { compileFilter, renderJoin, type RequestInfo } from "../filter/compile";
+import type { AuthRecord, Row } from "../types";
+import { recordToJSON } from "./json";
+import { fromColumn } from "./values";
+
+export interface ExpandContext {
+  db: D1Database;
+  collections: Map<string, Collection>;
+  auth: AuthRecord | null;
+  superuser: boolean;
+  request: RequestInfo;
+}
+
+const MAX_DEPTH = 6;
+const VIA = /^(\w+)_via_(\w+)$/;
+
+// Returns per-record expand objects (keyed by record id). Records without any resolvable expand get no entry
+// unless a valid relation path was requested, in which case they get {} (PocketBase keeps the key).
+export async function expandRecords(ctx: ExpandContext, c: Collection, rows: Row[], expands: string[]): Promise<Map<string, Record<string, unknown>>> {
+  const result = new Map<string, Record<string, unknown>>();
+  const paths = [...new Set(expands.map((s) => s.trim()).filter(Boolean))];
+  for (const path of paths) await expandPath(ctx, c, rows, path, 1, result);
+  return result;
+}
+
+async function expandPath(ctx: ExpandContext, c: Collection, rows: Row[], path: string, depth: number, result: Map<string, Record<string, unknown>>): Promise<void> {
+  if (depth > MAX_DEPTH || rows.length === 0) return;
+  const [head, ...restParts] = path.split(".");
+  const rest = restParts.join(".");
+  const fields = c.fields as Field[];
+  let related: Collection | null = null;
+  let byRow = new Map<string, Row[]>(); // base row id -> related rows
+  let single = false;
+  const relField = fields.find((f) => f.name === head && f.type === "relation");
+  if (relField) {
+    related = ctx.collections.get(String(relField.collectionId)) ?? null;
+    if (!related) return;
+    single = !isMultiple(relField);
+    const idsByRow = new Map<string, string[]>();
+    const allIds = new Set<string>();
+    for (const r of rows) {
+      const v = fromColumn(relField, r[relField.name]);
+      const ids = Array.isArray(v) ? v : v ? [String(v)] : [];
+      idsByRow.set(String(r.id), ids);
+      ids.forEach((id) => allIds.add(id));
+    }
+    const fetched = await fetchAllowed(ctx, related, [...allIds]);
+    for (const [rid, ids] of idsByRow) byRow.set(rid, ids.map((id) => fetched.get(id)).filter((x): x is Row => !!x));
+  } else {
+    const via = VIA.exec(head!);
+    if (!via) return; // unknown expand: ignored, as PocketBase does
+    related = ctx.collections.get(via[1]!) ?? null;
+    const backField = related && (related.fields as Field[]).find((f) => f.name === via[2] && f.type === "relation" && f.collectionId === c.id);
+    if (!related || !backField) return;
+    single = related.indexes.some((idx) => /UNIQUE/i.test(idx) && new RegExp("\\(\\s*[`\"']?" + backField.name + "[`\"']?\\s*\\)").test(idx));
+    const ids = rows.map((r) => String(r.id));
+    const fetched = await fetchBackRelated(ctx, related, backField, ids);
+    for (const r of rows) byRow.set(String(r.id), []);
+    for (const rel of fetched) {
+      const v = fromColumn(backField, rel[backField.name]);
+      const targets = Array.isArray(v) ? v : v ? [String(v)] : [];
+      for (const t of targets) byRow.get(t)?.push(rel);
+    }
+  }
+  // nested expands on the related rows
+  const nested = new Map<string, Record<string, unknown>>();
+  const relatedRows = [...new Map([...byRow.values()].flat().map((r) => [String(r.id), r])).values()];
+  if (rest) await expandPath(ctx, related, relatedRows, rest, depth + 1, nested);
+  for (const r of rows) {
+    const rid = String(r.id);
+    const items = byRow.get(rid) ?? [];
+    const entry = result.get(rid) ?? {};
+    const toJSON = (x: Row) => recordToJSON(related!, x, { auth: ctx.auth, own: ctx.auth?.collection.id === related!.id && ctx.auth.row.id === x.id, expand: nested.get(String(x.id)) });
+    if (single) { if (items[0]) entry[head!] = toJSON(items[0]); }
+    else if (items.length) entry[head!] = items.map(toJSON);
+    result.set(rid, entry);
+  }
+}
+
+// Fetch records by id through the collection's view rule (superusers bypass it).
+async function fetchAllowed(ctx: ExpandContext, c: Collection, ids: string[]): Promise<Map<string, Row>> {
+  const out = new Map<string, Row>();
+  if (ids.length === 0) return out;
+  if (!ctx.superuser && c.viewRule === null) return out;
+  let ruleSql = "";
+  let ruleParams: unknown[] = [];
+  let joins = "";
+  if (!ctx.superuser && c.viewRule) {
+    const compiled = compileFilter(c.viewRule, { base: c, collections: ctx.collections, request: { ...ctx.request, context: "expand" }, allowHiddenFields: true });
+    ruleSql = ` AND (${compiled.where})`;
+    ruleParams = compiled.params;
+    joins = compiled.joins.map(renderJoin).join(" ");
+  }
+  for (let i = 0; i < ids.length; i += 80) {
+    const chunk = ids.slice(i, i + 80);
+    const rows = await all(ctx.db, `SELECT DISTINCT ${ident(c.name)}.* FROM ${ident(c.name)} ${joins} WHERE ${ident(c.name)}.id IN (${chunk.map(() => "?").join(",")})${ruleSql}`, [...chunk, ...ruleParams]);
+    for (const r of rows) out.set(String(r.id), r);
+  }
+  return out;
+}
+
+async function fetchBackRelated(ctx: ExpandContext, c: Collection, backField: Field, ids: string[]): Promise<Row[]> {
+  if (ids.length === 0) return [];
+  if (!ctx.superuser && c.viewRule === null) return [];
+  let ruleSql = "";
+  let ruleParams: unknown[] = [];
+  let joins = "";
+  if (!ctx.superuser && c.viewRule) {
+    const compiled = compileFilter(c.viewRule, { base: c, collections: ctx.collections, request: { ...ctx.request, context: "expand" }, allowHiddenFields: true });
+    ruleSql = ` AND (${compiled.where})`;
+    ruleParams = compiled.params;
+    joins = compiled.joins.map(renderJoin).join(" ");
+  }
+  const out: Row[] = [];
+  for (let i = 0; i < ids.length; i += 80) {
+    const chunk = ids.slice(i, i + 80);
+    const col = `${ident(c.name)}.${ident(backField.name)}`;
+    const match = isMultiple(backField)
+      ? `EXISTS (SELECT 1 FROM json_each(CASE WHEN json_valid(${col}) THEN ${col} ELSE json_array(${col}) END) WHERE value IN (${chunk.map(() => "?").join(",")}))`
+      : `${col} IN (${chunk.map(() => "?").join(",")})`;
+    out.push(...(await all(ctx.db, `SELECT DISTINCT ${ident(c.name)}.* FROM ${ident(c.name)} ${joins} WHERE ${match}${ruleSql}`, [...chunk, ...ruleParams])));
+  }
+  return out;
+}

package/src/server/records/files.ts

@@ -0,0 +1,69 @@
+// File naming, content sniffing, and R2 storage keyed {collectionId}/{recordId}/{filename}.
+import { randomWithAlphabet } from "../ids";
+import type { Upload } from "./values";
+
+const EXT_INVALID = /[^\w.*\-+=#]+/g;
+
+// tools/filesystem.normalizeName: snakecase(name) + "_" + random10 + ext (random part always present)
+export function normalizeFilename(original: string, detectedExt: string): string {
+  let name = original.length > 300 ? original.slice(-300) : original;
+  const dot = name.lastIndexOf(".");
+  const originalExt = dot >= 0 ? name.slice(dot) : "";
+  let ext = "." + originalExt.replace(EXT_INVALID, "").replace(/^\.+|\.+$/g, "");
+  if (ext === ".") ext = detectedExt || "";
+  if (ext.length > 20) ext = "." + ext.slice(-20).replace(/^\.+/, "");
+  let clean = snakecase((originalExt ? name.slice(0, -originalExt.length) : name).replace(/^\.+|\.+$/g, ""));
+  if (clean.length < 3) clean += randomWithAlphabet(10, "abcdefghijklmnopqrstuvwxyz0123456789");
+  else if (clean.length > 100) clean = clean.slice(0, 100);
+  return `${clean}_${randomWithAlphabet(10, "abcdefghijklmnopqrstuvwxyz0123456789")}${ext.toLowerCase()}`;
+}
+
+export function snakecase(s: string): string {
+  return s.split(/[^\p{L}\p{N}]+/u).filter(Boolean).map((w) => w.replace(/([a-z0-9])([A-Z])/g, "$1_$2")).join("_").toLowerCase();
+}
+
+// Minimal content sniffing for the common cases; falls back to the client's declared type.
+export function sniffMime(bytes: Uint8Array, declared: string, name: string): { type: string; ext: string } {
+  const b = bytes;
+  const startsWith = (...sig: number[]) => sig.every((x, i) => b[i] === x);
+  if (startsWith(0x89, 0x50, 0x4e, 0x47)) return { type: "image/png", ext: ".png" };
+  if (startsWith(0xff, 0xd8, 0xff)) return { type: "image/jpeg", ext: ".jpg" };
+  if (startsWith(0x47, 0x49, 0x46, 0x38)) return { type: "image/gif", ext: ".gif" };
+  if (startsWith(0x52, 0x49, 0x46, 0x46) && b[8] === 0x57 && b[9] === 0x45 && b[10] === 0x42 && b[11] === 0x50) return { type: "image/webp", ext: ".webp" };
+  if (startsWith(0x25, 0x50, 0x44, 0x46)) return { type: "application/pdf", ext: ".pdf" };
+  if (startsWith(0x50, 0x4b, 0x03, 0x04)) return { type: "application/zip", ext: ".zip" };
+  const head = new TextDecoder().decode(b.slice(0, 512)).trimStart();
+  if (/^<svg[\s>]/i.test(head) || (/^<\?xml/i.test(head) && /<svg/i.test(head))) return { type: "image/svg+xml", ext: ".svg" };
+  if (/^\s*[{[]/.test(head)) { try { JSON.parse(new TextDecoder().decode(b)); return { type: "application/json", ext: ".json" }; } catch { /* not json */ } }
+  const ext = name.includes(".") ? name.slice(name.lastIndexOf(".")).toLowerCase() : "";
+  // PocketBase detects the type from the bytes (gabriel-vasile/mimetype) and ignores the declared one
+  void declared;
+  const isText = b.slice(0, 512).every((x) => x === 9 || x === 10 || x === 13 || (x >= 32 && x < 127) || x >= 128);
+  return isText ? { type: "text/plain; charset=utf-8", ext: ext || ".txt" } : { type: "application/octet-stream", ext };
+}
+
+export const fileKey = (collectionId: string, recordId: string, filename: string) => `${collectionId}/${recordId}/${filename}`;
+
+export async function putUpload(storage: R2Bucket, collectionId: string, recordId: string, up: Upload): Promise<void> {
+  await storage.put(fileKey(collectionId, recordId, up.name), up.bytes, { httpMetadata: { contentType: up.type } });
+}
+
+export async function deleteFiles(storage: R2Bucket, collectionId: string, recordId: string, names: string[]): Promise<void> {
+  if (names.length === 0) return;
+  await storage.delete(names.map((n) => fileKey(collectionId, recordId, n)));
+  // cached thumbnails live under thumbs_{filename}/
+  for (const n of names) await deletePrefix(storage, `${collectionId}/${recordId}/thumbs_${n}/`);
+}
+
+export async function deleteAllRecordFiles(storage: R2Bucket, collectionId: string, recordId: string): Promise<void> {
+  await deletePrefix(storage, `${collectionId}/${recordId}/`);
+}
+
+export async function deletePrefix(storage: R2Bucket, prefix: string): Promise<void> {
+  let cursor: string | undefined;
+  do {
+    const listed = await storage.list({ prefix, cursor });
+    if (listed.objects.length) await storage.delete(listed.objects.map((o) => o.key));
+    cursor = listed.truncated ? listed.cursor : undefined;
+  } while (cursor);
+}

package/src/server/records/json.ts

@@ -0,0 +1,23 @@
+import { isMultiple, type Field } from "../collections/fields";
+import type { Collection } from "../collections/model";
+import type { AuthRecord, Row } from "../types";
+import { fromColumn } from "./values";
+
+export interface JSONOptions { auth?: AuthRecord | null; own?: boolean; expand?: Record<string, unknown> }
+
+// The record JSON PocketBase returns: hidden fields dropped, password fields always "", collectionId/Name added,
+// keys sorted, email hidden unless emailVisibility, own record, or superuser; `expand` attached when present.
+export function recordToJSON(c: Collection, row: Row, opts: JSONOptions = {}): Record<string, unknown> {
+  const out: Record<string, unknown> = { collectionId: c.id, collectionName: c.name };
+  const superuser = opts.auth?.collection.name === "_superusers";
+  for (const f of c.fields as Field[]) {
+    if (f.hidden) continue;
+    out[f.name] = f.type === "password" ? "" : fromColumn(f, row[f.name]);
+    if (f.type === "relation" || f.type === "file" || f.type === "select") {
+      if (!isMultiple(f) && Array.isArray(out[f.name])) out[f.name] = (out[f.name] as string[])[0] ?? "";
+    }
+  }
+  if (c.type === "auth" && !superuser && !opts.own && !row.emailVisibility) delete out.email;
+  if (opts.expand) out.expand = opts.expand;
+  return Object.fromEntries(Object.entries(out).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
+}

package/src/server/records/picker.ts

@@ -0,0 +1,80 @@
+// The `fields` query param: comma separated paths with optional modifiers (`body:excerpt(200,true)`), `*` wildcard.
+interface Node { children: Map<string, Node>; modifier?: (v: unknown) => unknown; leaf: boolean }
+
+export function parseFields(raw: string): Node | null {
+  const root: Node = { children: new Map(), leaf: false };
+  const parts = splitTopLevel(raw).map((s) => s.trim()).filter(Boolean);
+  if (!parts.length) return null;
+  for (const part of parts) {
+    const m = /^([^:]+)(?::(\w+)\(([^)]*)\))?$/.exec(part);
+    if (!m) throw new Error(`invalid fields expression "${part}"`);
+    const path = m[1]!.split(".").map((s) => s.trim()).filter(Boolean);
+    let node = root;
+    for (const seg of path) {
+      let next = node.children.get(seg);
+      if (!next) { next = { children: new Map(), leaf: false }; node.children.set(seg, next); }
+      node = next;
+    }
+    node.leaf = true;
+    if (m[2] === "excerpt") {
+      const args = (m[3] ?? "").split(",").map((s) => s.trim());
+      const max = Number(args[0]) || 0;
+      const ellipsis = args[1] === "true";
+      node.modifier = (v) => (typeof v === "string" ? excerpt(v, max, ellipsis) : v);
+    } else if (m[2]) throw new Error(`unknown fields modifier "${m[2]}"`);
+  }
+  return root;
+}
+
+export function pick(data: unknown, node: Node | null): unknown {
+  if (!node) return data;
+  if (Array.isArray(data)) return data.map((d) => pick(d, node));
+  if (!data || typeof data !== "object") return data;
+  const src = data as Record<string, unknown>;
+  const out: Record<string, unknown> = {};
+  const star = node.children.get("*");
+  // tools/picker decodes into map[string]any, so Go serializes the picked object with sorted keys
+  for (const [k, v] of Object.entries(src).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
+    const child = node.children.get(k) ?? star;
+    if (!child) continue;
+    const value = child.children.size ? pick(v, child) : sortDeep(v);
+    out[k] = child.modifier ? child.modifier(value) : value;
+  }
+  return out;
+}
+
+// the whole picked document went through map[string]any in Go, so nested objects come back key-sorted too
+function sortDeep(v: unknown): unknown {
+  if (Array.isArray(v)) return v.map(sortDeep);
+  if (!v || typeof v !== "object") return v;
+  return Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([k, x]) => [k, sortDeep(x)]));
+}
+
+const INLINE = new Set(["a", "abbr", "acronym", "b", "bdo", "big", "br", "button", "cite", "code", "dfn", "em", "i", "img", "input", "kbd", "label", "map", "object", "output", "q", "samp", "select", "small", "span", "strong", "sub", "sup", "textarea", "time", "tt", "u", "var", "video"]);
+
+// tools/picker/excerpt_modifier.go: strip tags (block tags become spaces), collapse whitespace, cut to max, optional "...".
+export function excerpt(html: string, max: number, ellipsis: boolean): string {
+  let text = html.replace(/<(script|style|head|title|template)[\s\S]*?<\/\1>/gi, "");
+  text = text.replace(/<\/?([a-zA-Z][a-zA-Z0-9]*)[^>]*>/g, (_m, tag: string) => (INLINE.has(tag.toLowerCase()) ? "" : " "));
+  text = text.replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'");
+  text = text.replace(/\s+/g, " ").trim();
+  if (max <= 0 || [...text].length <= max) return text;
+  let cut = [...text].slice(0, max).join("");
+  const lastSpace = cut.lastIndexOf(" ");
+  if (lastSpace > 0) cut = cut.slice(0, lastSpace);
+  cut = cut.trimEnd();
+  return ellipsis ? cut + "..." : cut;
+}
+
+function splitTopLevel(s: string): string[] {
+  const out: string[] = [];
+  let depth = 0, cur = "";
+  for (const ch of s) {
+    if (ch === "(") depth++;
+    if (ch === ")") depth--;
+    if (ch === "," && depth === 0) { out.push(cur); cur = ""; continue; }
+    cur += ch;
+  }
+  out.push(cur);
+  return out;
+}