@makerclay/core 1.0.0

package/src/util/cookies.js

@@ -0,0 +1,26 @@
+// A cookie header parser, so the core needs no cookie-parser dependency.
+// Express already provides res.cookie(); only reading is missing.
+
+function parseCookies(header) {
+  const out = {};
+  if (typeof header !== 'string' || header.length === 0) return out;
+  for (const part of header.split(';')) {
+    const eq = part.indexOf('=');
+    if (eq === -1) continue;
+    const name = part.slice(0, eq).trim();
+    if (!name || Object.hasOwn(out, name)) continue;
+    let value = part.slice(eq + 1).trim();
+    if (value.startsWith('"') && value.endsWith('"')) value = value.slice(1, -1);
+    try {
+      out[name] = decodeURIComponent(value);
+    } catch {
+      out[name] = value;
+    }
+  }
+  return out;
+}
+
+export function cookieMiddleware(req, res, next) {
+  req.cookies = parseCookies(req.headers.cookie);
+  next();
+}

package/src/util/express.js

@@ -0,0 +1,8 @@
+// An async route handler that rejects must reach Express's error handler, and
+// Express 4 does not await anything, so an unwrapped `async (req, res)` turns a
+// thrown HostError into an unhandled rejection and a request that hangs until the
+// client gives up.
+//
+// This lived in three files as three identical definitions. One is enough, and a
+// shared one is what makes "every route is wrapped" a thing you can check.
+export const wrap = (fn) => (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next);

package/src/util/fsx.js

@@ -0,0 +1,205 @@
+import fs from 'node:fs/promises';
+import path from 'upath';
+
+// Every disk call in the core goes through here. The one rule that separates
+// this module from hyperclay's dx.js is that these THROW: in a files-are-truth
+// system a swallowed write error is silent data loss.
+
+let tmpCounter = 0;
+
+export async function readFileText(filePath) {
+  return await fs.readFile(filePath, 'utf8');
+}
+
+// Returns null only for a definitively absent file. Every other error throws,
+// so an EACCES can never be mistaken for "not there" (identity rule 4 rebinds
+// history on ENOENT alone).
+export async function readFileTextIfExists(filePath) {
+  try {
+    return await fs.readFile(filePath, 'utf8');
+  } catch (error) {
+    if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return null;
+    throw error;
+  }
+}
+
+export async function statIfExists(filePath) {
+  try {
+    return await fs.stat(filePath);
+  } catch (error) {
+    if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return null;
+    throw error;
+  }
+}
+
+// A normalized stat for identity work. `bigint: true` is required because a
+// Windows NTFS file index is 64 bit and two distinct ones collapse into one
+// value above 2^53 as a JS number. It also turns EVERY numeric field into a
+// BigInt, so size and mtime are converted back here rather than at each call
+// site, where `Math.round(bigint)` throws and `bigint === number` is silently
+// false.
+export async function statInfo(filePath) {
+  let stat;
+  try {
+    stat = await fs.stat(filePath, { bigint: true });
+  } catch (error) {
+    if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return null;
+    throw error;
+  }
+  return {
+    size: Number(stat.size),
+    // Sub-millisecond on purpose. Under `bigint: true` mtimeMs is whole
+    // milliseconds, and the scanner compares this against the stored value to
+    // decide a file is unchanged: at millisecond resolution a rewrite landing in
+    // the same millisecond as the host's own last write, with an unchanged byte
+    // count, is invisible to the boot scan. Measured at 480 collisions in 500
+    // back-to-back rewrites. mtimeNs is already in this same stat, so dividing it
+    // keeps the field's name and units and buys about 400ns of resolution.
+    mtimeMs: Number(stat.mtimeNs) / 1e6,
+    birthtimeMs: Number(stat.birthtimeMs),
+    dev: Number(stat.dev),
+    // Null, not 0: FAT32, exFAT and some network mounts report no usable
+    // identity, and null makes every consumer fall back to path matching.
+    ino: stat.ino === 0n ? null : String(stat.ino),
+    isFile: stat.isFile(),
+    isDirectory: stat.isDirectory(),
+  };
+}
+
+export async function exists(filePath) {
+  return (await statIfExists(filePath)) !== null;
+}
+
+export async function mkdirp(dir) {
+  await fs.mkdir(dir, { recursive: true });
+}
+
+export async function readdir(dir, options) {
+  return await fs.readdir(dir, options);
+}
+
+export async function readdirIfExists(dir, options) {
+  try {
+    return await fs.readdir(dir, options);
+  } catch (error) {
+    if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return [];
+    throw error;
+  }
+}
+
+// Durability, and the one switch that turns it off. Only a test suite should
+// ever set MAKERCLAY_FSYNC=0.
+//
+// fsync is what survives a power cut. It is NOT what makes a write atomic:
+// atomicity here comes from writing a temp file and then renaming or linking it
+// into place, and that rename is visible to every other process the instant the
+// kernel records it, flushed or not. So killing a process, which is the only
+// crash a test can stage, observes exactly the same states either way, while the
+// flushes themselves are the single largest cost the suite pays.
+const DURABLE = process.env.MAKERCLAY_FSYNC !== '0';
+
+export async function flush(handle) {
+  if (DURABLE) await handle.sync();
+}
+
+// fsync a directory so a freshly renamed entry survives a crash. Best-effort:
+// some platforms refuse to open a directory for sync, and that must never fail
+// a save that has already landed.
+export async function fsyncDir(dir) {
+  if (!DURABLE) return;
+  let handle;
+  try {
+    handle = await fs.open(dir, 'r');
+    await handle.sync();
+  } catch {
+    return;
+  } finally {
+    if (handle) await handle.close().catch(() => {});
+  }
+}
+
+// Write via a same-directory temp + rename, so a crash, a full disk, or a killed
+// process can never leave partial bytes at `filePath`. The rename replaces the
+// target rather than following it, which is why callers pass an
+// already-canonicalized path. This rename is the durable commit point of the
+// mutation kernel (§9.1 step 11).
+export async function atomicWrite(filePath, content, encoding = 'utf8') {
+  const dir = path.dirname(filePath);
+  await mkdirp(dir);
+
+  tmpCounter = (tmpCounter + 1) % 1e6;
+  const tmpPath = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${tmpCounter}.tmp`);
+
+  let mode = 0o644;
+  const current = await statIfExists(filePath);
+  if (current) mode = current.mode & 0o777;
+
+  let handle = null;
+  try {
+    handle = await fs.open(tmpPath, 'wx', 0o600);
+    await handle.writeFile(content, encoding === null ? undefined : encoding);
+    await flush(handle);
+    await handle.chmod(mode);
+    await handle.close();
+    handle = null;
+    await fs.rename(tmpPath, filePath);
+  } catch (error) {
+    if (handle) await handle.close().catch(() => {});
+    await fs.unlink(tmpPath).catch(() => {});
+    throw error;
+  }
+  await fsyncDir(dir);
+}
+
+// Claim a path and write it in one step, or fail because someone else holds it.
+// `atomicWrite` cannot do this: its rename REPLACES the target, so two writers
+// racing for one name both succeed and one file is silently lost. Here the
+// exclusive open IS the claim, so the loser gets EEXIST and can pick another name.
+// Used by the upload lane, where the host names every file and a taken name must
+// never mean an error and never mean an overwrite.
+export async function createExclusive(filePath, content) {
+  const dir = path.dirname(filePath);
+  await mkdirp(dir);
+  const handle = await fs.open(filePath, 'wx', 0o644);
+  try {
+    await handle.writeFile(content);
+    await flush(handle);
+  } catch (error) {
+    await handle.close().catch(() => {});
+    await fs.unlink(filePath).catch(() => {});
+    throw error;
+  }
+  await handle.close();
+  await fsyncDir(dir);
+}
+
+export async function rename(from, to) {
+  await fs.rename(from, to);
+}
+
+export async function unlink(filePath) {
+  await fs.unlink(filePath);
+}
+
+export async function unlinkIfExists(filePath) {
+  try {
+    await fs.unlink(filePath);
+    return true;
+  } catch (error) {
+    if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return false;
+    throw error;
+  }
+}
+
+export async function rmrf(target) {
+  await fs.rm(target, { recursive: true, force: true });
+}
+
+export async function copyFile(from, to) {
+  await mkdirp(path.dirname(to));
+  await fs.copyFile(from, to);
+}
+
+export async function realpath(target) {
+  return path.resolve(await fs.realpath(target));
+}

package/src/util/id.js

@@ -0,0 +1,37 @@
+import { randomBytes } from 'node:crypto';
+
+const ENCODING = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
+const TIME_LEN = 10;
+const RANDOM_LEN = 16;
+
+function encodeTime(now) {
+  let out = '';
+  let value = now;
+  for (let i = TIME_LEN - 1; i >= 0; i--) {
+    out = ENCODING[value % 32] + out;
+    value = Math.floor(value / 32);
+  }
+  return out;
+}
+
+function encodeRandom() {
+  const bytes = randomBytes(RANDOM_LEN);
+  let out = '';
+  for (let i = 0; i < RANDOM_LEN; i++) out += ENCODING[bytes[i] % 32];
+  return out;
+}
+
+// ULID: 48 bits of millisecond time in Crockford base32, then 80 bits of
+// randomness. Lexically sortable by mint time, never reused, and safe in a URL,
+// a filename, and an HTML attribute without escaping.
+export function ulid(now = Date.now()) {
+  return encodeTime(now) + encodeRandom();
+}
+
+
+// Opaque high-entropy string for share links, save tokens, session cookies and
+// setup keys. base64url of 24 random bytes: 32 chars, URL- and path-safe.
+export function token(bytes = 24) {
+  return randomBytes(bytes).toString('base64url');
+}
+

package/src/util/lockfile.js

@@ -0,0 +1,52 @@
+import fs from 'node:fs';
+
+// Invariant 10: single writer. makerclay refuses to start if <data>/makerclay.lock
+// is held by a live pid. Two processes on one data directory would race the
+// SQLite writer, which has no story for that.
+
+function pidIsAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM';
+  }
+}
+
+export class LockHeldError extends Error {
+  constructor(file, pid) {
+    super(`makerclay is already running on this data directory (pid ${pid}, lock ${file})`);
+    this.name = 'LockHeldError';
+    this.pid = pid;
+  }
+}
+
+export function acquireLock(file) {
+  const claim = () => fs.writeFileSync(file, `${process.pid}\n`, { encoding: 'utf8', flag: 'wx' });
+  try {
+    claim();
+  } catch (error) {
+    if (error.code !== 'EEXIST') throw error;
+    const held = Number(String(fs.readFileSync(file, 'utf8')).trim());
+    if (Number.isInteger(held) && held > 0 && pidIsAlive(held)) {
+      throw new LockHeldError(file, held);
+    }
+    fs.unlinkSync(file);
+    claim();
+  }
+
+  let released = false;
+  return {
+    file,
+    release() {
+      if (released) return;
+      released = true;
+      try {
+        const held = Number(String(fs.readFileSync(file, 'utf8')).trim());
+        if (held === process.pid) fs.unlinkSync(file);
+      } catch {
+        // Already gone: nothing to release.
+      }
+    },
+  };
+}

package/src/util/locks.js

@@ -0,0 +1,35 @@
+// Per-key promise chains. Lifted from hyperclay-local's write-queue.js.
+//
+// The key is ALWAYS a canonical resolved absolute path. Any other key (a raw
+// request path, a pre-realpath join) hands two names for one file two different
+// slots and makes the queue a silent no-op.
+//
+// Callers wrap the ENTIRE read-modify-write region, not just the write:
+// serializing only the write still lets two requests read the same stale base.
+//
+// withFileLock is NOT reentrant. The versions directory is a separate namespace
+// from the live file, deliberately, so publishing a version inside a save's own
+// critical section cannot deadlock.
+
+export function createLockPool() {
+  const chains = new Map();
+
+  function withLock(key, fn) {
+    if (typeof key !== 'string' || key.length === 0) {
+      throw new Error('withLock requires a non-empty key');
+    }
+    const prev = chains.get(key) || Promise.resolve();
+    const run = prev.then(() => fn());
+    const tail = run.then(() => {}, () => {});
+    chains.set(key, tail);
+    tail.then(() => {
+      if (chains.get(key) === tail) chains.delete(key);
+    });
+    return run;
+  }
+
+  return {
+    withLock,
+    pendingKeys: () => [...chains.keys()],
+  };
+}

package/src/util/multipart.js

@@ -0,0 +1,33 @@
+import busboy from 'busboy';
+import { HostError } from '../spec/codes.js';
+
+// One multipart reader for both upload lanes. The dashboard's lane takes several
+// files plus fields; the document lane takes one part named `file` (spec §9). They
+// differ in what they accept afterwards, never in how the body is read.
+
+export function readMultipart(req, { limit, files = 20 } = {}) {
+  return new Promise((resolve, reject) => {
+    const fields = {};
+    const parts = [];
+    const parser = busboy({ headers: req.headers, limits: { fileSize: limit, files } });
+    let failed = null;
+
+    parser.on('field', (name, value) => { fields[name] = value; });
+    parser.on('file', (name, stream, info) => {
+      const chunks = [];
+      stream.on('data', (chunk) => chunks.push(chunk));
+      // busboy truncates at the limit rather than erroring, so a file that trips it
+      // would otherwise be stored silently short.
+      stream.on('limit', () => {
+        failed = new HostError('too-large', `Files are limited to ${limit} bytes.`);
+        stream.resume();
+      });
+      stream.on('end', () => {
+        if (!failed) parts.push({ field: name, filename: info.filename, mimeType: info.mimeType, content: Buffer.concat(chunks) });
+      });
+    });
+    parser.on('error', reject);
+    parser.on('close', () => (failed ? reject(failed) : resolve({ fields, files: parts })));
+    req.pipe(parser);
+  });
+}