@makerclay/core 1.0.0

package/src/nodes/scanner.js

@@ -0,0 +1,321 @@
+import path from 'upath';
+import chokidar from 'chokidar';
+import { documentEtag } from '../spec/wire.js';
+import { isReservedNodePath } from '../paths.js';
+import {
+  readdirIfExists, statIfExists, statInfo, readFileTextIfExists, unlinkIfExists,
+} from '../util/fsx.js';
+
+// The overlay reconciler: a boot walk and a watcher, both of which agree with
+// the save lane rather than racing it.
+//
+// The cost model is the whole design. A row exists only for a path that has
+// state or history, so the walk is one readdir pass plus work proportional to
+// the number of ROWS, never to the number of files. Someone will point makerclay
+// at their home directory in week one; 50k untouched files must cost 50k stat
+// calls and nothing else. Digests are computed lazily, on first serve or save.
+
+const DEBRIS = /^\..+\.(\d+)\.\d+\.tmp$/;
+
+export function createScanner({
+  paths, nodes, locks, events, clock, versions = null, logger = console,
+}) {
+  const missing = new Set();
+  const watchers = [];
+  const key = (owner, relPath) => `${owner}\x1f${relPath}`;
+
+  async function walk(root, prefix = '', depth = 0, out = new Map()) {
+    if (depth > paths.maxFolderDepth + 1) return out;
+    for (const entry of await readdirIfExists(root, { withFileTypes: true })) {
+      if (entry.name.startsWith('.')) continue;
+      const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
+      const full = path.join(root, entry.name);
+      if (entry.isDirectory()) {
+        await walk(full, relPath, depth + 1, out);
+        continue;
+      }
+      // Deliberate: a symlink is neither a directory nor a file here, so it is
+      // skipped rather than followed. Do not "fix" this into stat-following.
+      // The inode matcher below rebinds a row to whatever carries its inode, and
+      // a followed symlink lets anyone who can write one in this tree rebind a
+      // row's identity, history and private flag onto a target of their choosing.
+      if (!entry.isFile()) continue;
+      const info = await statInfo(full);
+      if (info) out.set(relPath, info);
+    }
+    return out;
+  }
+
+  // A kill between the temp write and the rename leaves `.name.pid.n.tmp`
+  // behind. It is already unreachable (paths.js 404s every dotfile), so this is
+  // housekeeping, and it is only safe at boot: the single-writer lock is what
+  // guarantees no other process has one of these open.
+  async function sweepDebris(root, prefix = '', depth = 0) {
+    let swept = 0;
+    if (depth > paths.maxFolderDepth + 1) return swept;
+    for (const entry of await readdirIfExists(root, { withFileTypes: true })) {
+      const full = path.join(root, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name.startsWith('.')) continue;
+        swept += await sweepDebris(full, `${prefix}${entry.name}/`, depth + 1);
+        continue;
+      }
+      const debris = DEBRIS.exec(entry.name);
+      if (!debris) continue;
+      // Our OWN pid is a save in flight, not debris. Only HTTP requests wait on
+      // boot, so an embedder calling host.documents.replace() straight away runs
+      // alongside this sweep, and deleting its temp file fails the save.
+      if (Number(debris[1]) === process.pid) continue;
+      if (await unlinkIfExists(full)) swept += 1;
+    }
+    return swept;
+  }
+
+  async function refreshRow(owner, relPath, row, stat) {
+    const full = path.join(paths.realmRoot(owner), relPath);
+    return await locks.withLock(full, async () => {
+      const bytes = await readFileTextIfExists(full);
+      if (bytes == null) return null;
+      const etag = documentEtag(bytes);
+      const signature = {
+        bytes: Buffer.byteLength(bytes, 'utf8'), mtimeMs: stat.mtimeMs,
+      };
+      // Nothing to announce, but the mtime still moved and it is the key the
+      // unchanged fast path reads. Now that the kernel stamps observedEtag this
+      // branch fires on an ordinary `touch` after a save, and a row left holding
+      // a stale mtime is one this scan and every scan after it re-reads.
+      if (etag === row.observedEtag) {
+        nodes.update(row, signature);
+        return null;
+      }
+      const updated = nodes.update(row, { observedEtag: etag, ...signature });
+      events.emit('node-changed', { node: updated, source: 'scanner' });
+      return updated;
+    });
+  }
+
+  // Only ever called for rows whose file vanished or was replaced, so the cost
+  // is proportional to real moves rather than to folder size. Two signals,
+  // strongest first: an inode is exact and free, and a digest is a proof that
+  // costs a bounded read.
+  async function rebindRenames(owner, gone, seen, inodeAt) {
+    if (!gone.length) return 0;
+    const indexed = new Set(nodes.list({ owner }).map((row) => row.path));
+    const unclaimed = [...seen].filter(([relPath]) => !indexed.has(relPath));
+    let rebound = 0;
+
+    async function claim(row, relPath) {
+      const moved = await reanchor(row, relPath);
+      missing.delete(key(owner, row.path));
+      events.emit('node-renamed', { node: moved, from: row.path, source: 'scanner' });
+      indexed.add(relPath);
+      rebound += 1;
+    }
+
+    function byInode(row) {
+      if (!row.ino) return null;
+      const at = inodeAt.get(`${row.dev}:${row.ino}`);
+      if (!at) return null;
+      // An inode is an identity only while its file is alive. For a row whose
+      // file is gone, which is the only kind this runs for, the number alone
+      // proves nothing: ext4, XFS and btrfs hand a freed inode straight back out.
+      // Birth time is what separates a rename from a reissue, and content cannot,
+      // because a rename that also rewrites the bytes is precisely what this
+      // strategy exists to catch.
+      const info = seen.get(at);
+      if (info?.birthtimeMs && row.updatedAt && info.birthtimeMs > row.updatedAt) return null;
+      return at;
+    }
+
+    // The size filter is free: `seen` already holds every stat. On a tree with
+    // nothing missing this does zero reads, and when something IS missing it
+    // reads only the byte-exact size matches.
+    async function byDigest(row) {
+      const want = row.observedEtag || row.etag;
+      if (!want || row.bytes == null) return null;
+      let hit = null;
+      for (const [relPath, info] of unclaimed) {
+        if (info.size !== row.bytes || indexed.has(relPath)) continue;
+        const bytes = await readFileTextIfExists(path.join(paths.realmRoot(owner), relPath));
+        if (bytes == null || documentEtag(bytes) !== want) continue;
+        // Two byte-identical candidates: there is no way to tell which is the
+        // one that moved, so strand the history rather than guess wrong.
+        if (hit) return null;
+        hit = relPath;
+      }
+      return hit;
+    }
+
+    let pending = gone.slice();
+    for (const strategy of [byInode, byDigest]) {
+      const next = [];
+      for (const row of pending) {
+        const found = await strategy(row);
+        if (found && !indexed.has(found)) {
+          await claim(row, found);
+          continue;
+        }
+        next.push(row);
+      }
+      pending = next;
+    }
+
+    gone.length = 0;
+    gone.push(...pending);
+    return rebound;
+  }
+
+  // Written after the bytes have already moved, so a failure here costs a stale
+  // record and never the rebind. Same deal as ops.js's patchMetaQuietly.
+  async function patchRecordPath(node, relPath) {
+    if (!versions) return;
+    try {
+      await versions.patchMeta(node, { path: relPath });
+    } catch (error) {
+      logger.warn?.('[makerclay] could not update the durable record:', error?.message || error);
+    }
+  }
+
+  // The record's path is patched here for the same reason ops.move patches it:
+  // this is the lane that absorbs a rename made in Finder or a terminal, so
+  // leaving the record behind goes stale on exactly the renames this host is
+  // built for. It also fails OPEN, because a record whose path is stale rebuilds
+  // a row at a path nothing is at, while the real file serves with no row.
+  async function reanchor(row, relPath) {
+    const moved = nodes.update(row, { path: relPath, updatedAt: clock.now() });
+    await patchRecordPath(moved, relPath);
+    return moved;
+  }
+
+  async function scan({ owner = '', sweep = true } = {}) {
+    const root = paths.realmRoot(owner);
+    const swept = sweep ? await sweepDebris(root) : 0;
+    const seen = await walk(root);
+    // Where two live paths share one inode (a hardlink), the entry is null:
+    // "this inode is here" is no longer a single answer, so nothing may guess.
+    const inodeAt = new Map();
+    for (const [relPath, info] of seen) {
+      if (!info.ino) continue;
+      const at = `${info.dev}:${info.ino}`;
+      inodeAt.set(at, inodeAt.has(at) ? null : relPath);
+    }
+    // Reserved rows anchor identity for things that are not files in this tree
+    // (collection records). The walk cannot see them, so without this every one
+    // of them would report as permanently missing.
+    const rows = nodes.list({ owner }).filter((row) => !isReservedNodePath(row.path));
+    const rowPaths = new Set(rows.map((row) => row.path));
+
+    const gone = [];
+    let refreshed = 0;
+    for (const row of rows) {
+      const info = seen.get(row.path);
+      if (!info) {
+        gone.push(row);
+        continue;
+      }
+      // The row's own file is alive at ANOTHER unindexed path, so this path now
+      // holds something else. That is `mv a.html archive.html; cp t.html a.html`,
+      // an ordinary publishing move, and also what a git checkout across a
+      // rename looks like. Binding to the path would hand this row's identity,
+      // history and private flag to a stranger's file.
+      const elsewhere = row.ino ? inodeAt.get(`${row.dev}:${row.ino}`) : null;
+      if (elsewhere && elsewhere !== row.path && !rowPaths.has(elsewhere)) {
+        gone.push(row);
+        continue;
+      }
+      missing.delete(key(owner, row.path));
+      // Recorded HERE, not in refreshRow, which returns early when the bytes are
+      // unchanged. A replace-in-place with identical content (git checkout,
+      // rsync, cp over the top) changes the inode without changing a byte, and
+      // `rsync -a` preserves mtime and size too, so the unchanged fast path
+      // below would skip it entirely.
+      if (row.dev !== info.dev || row.ino !== info.ino) {
+        nodes.update(row, { dev: info.dev, ino: info.ino });
+      }
+      const unchanged = row.mtimeMs === info.mtimeMs && row.bytes === info.size;
+      if (unchanged) continue;
+      if (await refreshRow(owner, row.path, row, info)) refreshed += 1;
+    }
+
+    const rebound = await rebindRenames(owner, gone, seen, inodeAt);
+    // A row whose file is gone is marked, never tombstoned. An unmounted volume
+    // and a deliberate delete look identical from here, and only one of them
+    // should cost someone their history; the API delete lane tombstones.
+    for (const row of gone) missing.add(key(owner, row.path));
+
+    return { owner, files: seen.size, rows: rows.length, refreshed, rebound, missing: gone.length, swept };
+  }
+
+  // The boot scan is not the only rename lane. chokidar reports a rename as an
+  // unlink followed by an add, and until this existed the add was announced as a
+  // brand new file: the old row kept a path that no longer resolved, and the new
+  // path had no row at all, which means it served PUBLIC until the next restart.
+  //
+  // Ordering is not a coincidence. `awaitWriteFinish` holds the add back by at
+  // least its stability threshold while unlink fires immediately, so by the time
+  // this runs the vacated row is already marked missing.
+  async function rebindOne(owner, relPath) {
+    const gone = nodes.list({ owner }).filter((row) => missing.has(key(owner, row.path)));
+    if (!gone.length) return null;
+    const info = await statInfo(path.join(paths.realmRoot(owner), relPath));
+    if (!info?.isFile) return null;
+    const seen = new Map([[relPath, info]]);
+    const inodeAt = new Map(info.ino ? [[`${info.dev}:${info.ino}`, relPath]] : []);
+    return (await rebindRenames(owner, gone, seen, inodeAt)) ? nodes.get(owner, relPath) : null;
+  }
+
+  async function onDiskChange(owner, relPath) {
+    const row = nodes.get(owner, relPath);
+    if (!row) {
+      // rebindRenames emits node-renamed through claim, so a rebind announces
+      // itself and this lane must not announce it a second time.
+      if (await rebindOne(owner, relPath)) return;
+      events.emit('node-appeared', { owner, path: relPath, source: 'watcher' });
+      return;
+    }
+    const full = path.join(paths.realmRoot(owner), relPath);
+    const stat = await statIfExists(full);
+    if (!stat) return;
+    missing.delete(key(owner, relPath));
+    await refreshRow(owner, relPath, row, stat);
+  }
+
+  function watch({ owner = '' } = {}) {
+    const root = paths.realmRoot(owner);
+    const watcher = chokidar.watch(root, {
+      ignoreInitial: true,
+      ignored: (target) => path.basename(target).startsWith('.'),
+      awaitWriteFinish: { stabilityThreshold: 200, pollInterval: 50 },
+      depth: paths.maxFolderDepth + 1,
+    });
+
+    const relative = (full) => path.relative(root, path.resolve(full));
+    const guard = (fn) => (full) => Promise.resolve(fn(relative(full)))
+      .catch((error) => logger.error?.('[makerclay] watcher:', error?.message || error));
+
+    watcher.on('add', guard((relPath) => onDiskChange(owner, relPath)));
+    watcher.on('change', guard((relPath) => onDiskChange(owner, relPath)));
+    watcher.on('unlink', guard((relPath) => {
+      const row = nodes.get(owner, relPath);
+      if (!row) return;
+      missing.add(key(owner, relPath));
+      events.emit('node-missing', { node: row, source: 'watcher' });
+    }));
+
+    watchers.push(watcher);
+    return watcher;
+  }
+
+  return {
+    scan,
+    watch,
+    isMissing: (owner, relPath) => missing.has(key(owner, relPath)),
+    missingPaths: () => [...missing].map((entry) => {
+      const [owner, relPath] = entry.split('\x1f');
+      return { owner, path: relPath };
+    }),
+    async close() {
+      await Promise.all(watchers.splice(0).map((watcher) => watcher.close()));
+    },
+  };
+}

package/src/nodes/store.js

@@ -0,0 +1,111 @@
+import { ulid } from '../util/id.js';
+import { stableId } from './identity.js';
+import { isDocumentPath } from '../paths.js';
+
+// Node CRUD over the store driver, with a provisional path when there is no
+// driver at all. Every caller in the core resolves a node through here rather
+// than touching the driver, so `store: null` degrades in exactly one place.
+
+export function kindFor(relPath, isDir = false) {
+  if (isDir) return 'dir';
+  return isDocumentPath(relPath) ? 'doc' : 'file';
+}
+
+export function createNodes({ store, clock, ids = { ulid } }) {
+  function provisional(owner, relPath, kind) {
+    const now = clock.now();
+    return {
+      id: stableId(owner, relPath),
+      owner: owner ?? '',
+      path: relPath,
+      kind: kind || kindFor(relPath),
+      etag: null,
+      observedEtag: null,
+      bytes: null,
+      mtimeMs: null,
+      dev: null,
+      ino: null,
+      seq: 0,
+      createdAt: now,
+      updatedAt: now,
+      deletedAt: null,
+      private: false,
+      isolated: null,
+      signups: false,
+      sourceId: null,
+      provisional: true,
+    };
+  }
+
+  function get(owner, relPath) {
+    if (!store) return null;
+    return store.getNode(owner ?? '', relPath);
+  }
+
+  function byId(id) {
+    if (!store) return null;
+    return store.getNodeById(id);
+  }
+
+  // A node object for (owner, path) whether or not a row exists. Nothing is
+  // written: rows are materialized lazily on first save or first version, so a
+  // 50k-file folder costs no rows until someone actually edits something.
+  function resolve(owner, relPath, { kind = null } = {}) {
+    return get(owner, relPath) || provisional(owner, relPath, kind);
+  }
+
+  function materialize(node, patch = {}) {
+    if (!store) return { ...node, ...patch };
+    return store.tx(() => {
+      const existing = store.getNodeById(node.id) || store.getNode(node.owner, node.path);
+      if (existing) {
+        return store.updateNode(existing.id, { updatedAt: clock.now(), seq: store.nextSeq(), ...patch });
+      }
+      const now = clock.now();
+      return store.insertNode({
+        ...node,
+        id: node.provisional ? ids.ulid() : node.id,
+        seq: store.nextSeq(),
+        createdAt: now,
+        updatedAt: now,
+        ...patch,
+      });
+    });
+  }
+
+  function update(node, patch) {
+    if (!store) return { ...node, ...patch };
+    const row = store.getNodeById(node.id);
+    if (!row) return materialize(node, patch);
+    return store.updateNode(row.id, { updatedAt: clock.now(), ...patch });
+  }
+
+  function list(options) {
+    if (!store) return [];
+    return store.listNodes(options);
+  }
+
+  function tombstone(node, at = clock.now()) {
+    if (!store) return { ...node, deletedAt: at };
+    const row = store.getNodeById(node.id) || materialize(node);
+    return store.tombstone(row.id, at);
+  }
+
+  function restore(node) {
+    if (!store) return { ...node, deletedAt: null };
+    return store.updateNode(node.id, { deletedAt: null, updatedAt: clock.now() });
+  }
+
+  return {
+    provisional,
+    get,
+    byId,
+    resolve,
+    materialize,
+    update,
+    list,
+    tombstone,
+    restore,
+    kindFor,
+  };
+}

package/src/pages.js

@@ -0,0 +1,166 @@
+import path from 'upath';
+import { createRequire } from 'node:module';
+import { can } from './auth/can.js';
+import { jsonError } from './json-errors.js';
+import { statIfExists, readFileText } from './util/fsx.js';
+import { documentEtag } from './spec/wire.js';
+
+// Every HTML page the host serves for itself.
+//
+// The naming rule keeps the surface unambiguous: PAGES end in .html, APIs do
+// not. `/_/versions` returns JSON, `/_/versions.html` is the page that renders
+// it. The dashboard at `/_/` is the one exception, because a bare host root
+// should open something.
+//
+// The editor is the first conditional-save client in the ecosystem: it sends
+// Document-URL and If-Match, which is exactly where two-tab clobbers actually
+// happen and the one place with no snapshot pipeline in the way.
+
+const CM_FILES = new Set([
+  'lib/codemirror.js',
+  'lib/codemirror.css',
+  'mode/xml/xml.js',
+  'mode/javascript/javascript.js',
+  'mode/css/css.js',
+  'mode/htmlmixed/htmlmixed.js',
+  'addon/edit/closetag.js',
+  'addon/edit/matchbrackets.js',
+  'addon/selection/active-line.js',
+  'addon/search/search.js',
+  'addon/search/searchcursor.js',
+  'addon/search/jump-to-line.js',
+  'addon/dialog/dialog.js',
+  'addon/dialog/dialog.css',
+]);
+
+function locateCodeMirror() {
+  try {
+    const require_ = createRequire(import.meta.url);
+    return path.dirname(path.dirname(require_.resolve('codemirror/lib/codemirror.js')));
+  } catch {
+    return null;
+  }
+}
+
+export function mountPageRoutes(ctx) {
+  const {
+    router, config, ui, 
+    gateCtx, wrap, resolveTarget, serve, saveTokens, 
+  } = ctx;
+
+  const codemirror = locateCodeMirror();
+
+  function requireOwner(req, res, next) {
+    if (can(req.actor, 'admin', null, gateCtx)) return next();
+    // A person, not a script: send them to sign in rather than to a JSON blob.
+    // A tenant is already signed in, though, so sending them to the sign-in page
+    // is a redirect loop; they get their own document instead.
+    if (req.method === 'GET' && req.accepts(['html', 'json']) !== 'json') {
+      if (req.actor?.kind === 'tenant') return res.redirect(303, '/_/tenant/home');
+      return res.redirect(303, `/_/login?next=${encodeURIComponent(req.originalUrl)}`);
+    }
+    return jsonError(res, 'not-found', 'Not found');
+  }
+
+  // Not owner-gated: the login and setup pages need it before anyone is signed
+  // in, and it is a stylesheet.
+  router.get('/ui.css', wrap(async (req, res) => {
+    const file = await ui.resolveFile('app.css');
+    if (!file) return jsonError(res, 'not-found', 'Not found');
+    res.setHeader('Content-Type', 'text/css; charset=utf-8');
+    res.setHeader('Cache-Control', 'public, max-age=0, must-revalidate');
+    return res.sendFile(file);
+  }));
+
+  // ---- the dashboard -----------------------------------------------------
+
+  router.get('/', requireOwner, wrap(async (req, res) => {
+    if (config.features?.dashboard === false) return jsonError(res, 'not-found', 'Not found');
+    return ui.send(res, 'dashboard', {
+      title: 'makerclay',
+      host: config.publicUrl || '',
+      tenants: config.features?.tenants ? 'true' : 'false',
+      uploads: config.features?.uploads === false ? 'false' : 'true',
+      shares: config.features?.share === false ? 'false' : 'true',
+      collections: config.features?.collections === false ? 'false' : 'true',
+    });
+  }));
+
+  router.get('/trash.html', requireOwner, wrap(async (req, res) => {
+    return ui.send(res, 'trash', { title: 'Trash' });
+  }));
+
+  router.get('/attic.html', requireOwner, wrap(async (req, res) => {
+    return ui.send(res, 'attic', { title: 'The attic' });
+  }));
+
+  router.get('/records.html', requireOwner, wrap(async (req, res) => {
+    if (config.features?.collections === false) return jsonError(res, 'not-found', 'Not found');
+    const slug = String(req.query.slug || '').replace(/[^a-z0-9_]/gi, '');
+    return ui.send(res, 'records', { title: 'Submissions', slug });
+  }));
+
+  router.get('/versions.html', requireOwner, wrap(async (req, res) => {
+    const relPath = String(req.query.path || '').replace(/^\/+/, '');
+    return ui.send(res, 'versions', {
+      title: relPath ? `History of ${path.basename(relPath)}` : 'History',
+      path: relPath,
+      name: relPath ? path.basename(relPath) : '',
+    });
+  }));
+
+  // ---- the source editor -------------------------------------------------
+
+  router.get(/^\/edit\/(.*)$/, requireOwner, wrap(async (req, res) => {
+    if (config.features?.editor === false) return jsonError(res, 'not-found', 'Not found');
+    const relPath = decodeURIComponent(req.params[0] || '').replace(/^\/+/, '');
+    if (!relPath) return jsonError(res, 'bad-request', 'Which file?');
+
+    const found = await resolveTarget(req.actor, `/${relPath}`, 'read');
+    if (!found) return jsonError(res, 'not-found', 'Not found');
+    if (!can(req.actor, 'edit-source', found.node, gateCtx)) {
+      return jsonError(res, 'not-found', 'Not found');
+    }
+    const stat = await statIfExists(found.real);
+    if (!stat?.isFile()) return jsonError(res, 'not-found', 'Not found');
+
+    const source = await readFileText(found.real);
+
+    // A sandboxed document's save lane is token-only, and the editor is a real
+    // client of it. Without this, turning on the "run this file in a sandbox"
+    // toggle would quietly break the file's own editor.
+    let saveUrl = '/_/save';
+    if (serve?.isolationFor(found.node)) {
+      const token = await saveTokens?.mintFor({ actor: req.actor, node: found.node });
+      if (token) saveUrl = `/_/save/${token}`;
+    }
+
+    return ui.send(res, 'editor', {
+      title: `Edit ${path.basename(found.node.path)}`,
+      path: found.node.path,
+      name: path.basename(found.node.path),
+      saveUrl,
+      // The editor holds the etag it loaded and sends it back as If-Match, so a
+      // second tab cannot silently overwrite the first.
+      etag: documentEtag(source),
+      source,
+      mode: /\.(css)$/i.test(found.node.path) ? 'css'
+        : /\.(js|mjs)$/i.test(found.node.path) ? 'javascript'
+          : /\.(json)$/i.test(found.node.path) ? 'application/json'
+            : 'htmlmixed',
+    });
+  }));
+
+  // The vendored editor tree, allowlisted file by file. No CDN: an offline box
+  // has to be able to edit its own source.
+  router.get(/^\/editor\/(.+)$/, (req, res) => {
+    if (!codemirror) return jsonError(res, 'not-found', 'This host has no bundled editor.');
+    const rest = req.params[0];
+    if (!CM_FILES.has(rest)) return jsonError(res, 'not-found', 'Not found');
+    res.setHeader('Cache-Control', 'public, max-age=0, must-revalidate');
+    res.setHeader('X-Content-Type-Options', 'nosniff');
+    return res.sendFile(path.join(codemirror, rest));
+  });
+
+  return { codemirror };
+}