@3sln/trove 0.0.2

package/packages/server/src/cachePolicy.js

@@ -0,0 +1,34 @@
+// What may be cached, and for how long.
+//
+// Its own module, with no imports at all, because it is needed in two places that
+// cannot import each other: the Node/Bun static file server (which needs `node:path`
+// and so cannot load on Workers) and the runtime-agnostic request path in index.js
+// (which must load on Workers, where the assets come from a binding instead).
+//
+// The rule the build enforces and this describes: everything under /assets/ is
+// content-addressed, so the filename changes whenever the bytes do and the URL can be
+// kept forever. Everything else — index.html, sw.js, the manifest, the icon — keeps a
+// stable name and must be revalidated on every load.
+//
+// Getting this backwards is what turns a deploy into a blank page. Marking a mutable,
+// stable-named URL `immutable` hands a browser a cached entry point it will not check
+// again, so it goes on importing hashed modules from a build that no longer exists.
+// Leaving the content-addressed tree unmarked is the cheaper mistake and was the one in
+// force: the responses carried no Cache-Control at all, so the hashing bought nothing.
+
+/** Kept in step with the `naming` patterns in packages/web/build.mjs. */
+export const IMMUTABLE_PREFIX = '/assets/';
+
+/** A year, and never revalidated — safe only because the name changes with the bytes. */
+export const IMMUTABLE = 'public, max-age=31536000, immutable';
+
+/** Cacheable, but check with the server before every use. */
+export const REVALIDATE = 'no-cache';
+
+/**
+ * @param {string} pathname request path, already decoded
+ * @returns {string} a Cache-Control value
+ */
+export function cacheControlFor(pathname) {
+  return pathname.startsWith(IMMUTABLE_PREFIX) ? IMMUTABLE : REVALIDATE;
+}

package/packages/server/src/engine/README.md

@@ -0,0 +1,88 @@
+# Spike: the backend as an ngin engine
+
+One route — the collection scan — rewritten as a provider graph and an action,
+to find out whether the backend reads better this way before forty of them are
+touched. **Not merged into the architecture yet**: `createServer` still builds
+everything else by hand.
+
+## The check that makes it worth attempting
+
+`beginScan` keeps the signature it always had, so `createServer` stays a facade
+and every existing test drives HTTP exactly as before. 409 tests passed before
+the rewrite and 409 passed after, which is the only useful definition of "this
+changed no behaviour". A migration that cannot be verified that way should not
+be started.
+
+## What it replaced
+
+78 lines inside `createServer`, which had accreted three things:
+
+| Hand-rolled | What it actually was |
+| --- | --- |
+| `{ task, alreadyRunning, done }` | a feed — `started` now, `result` later |
+| `task.progress()` + clients polling `/api/tasks` | a feed, with a second mechanism bolted on |
+| a `try/finally` releasing the scan claim | a resource lifetime — and it released too early the first time it was written |
+
+## What came out better
+
+**Dependencies are declared.** `ScanCollection.deps` is `['vfs', 'tasks',
+'lifecycle']` plus a `claim` carrying its collection. Today every route receives
+one 18-key context object, which is a service locator: nothing records what a
+route uses, so nothing stops it reaching for more. A declared list is checkable,
+and `engine-scan.test.js` checks it.
+
+**The claim is a resource, not a discipline.** `ScanClaimProvider.obtain` takes
+the lease, `release` gives it back, and the container calls `release` whether the
+action returns, throws, or is aborted. Two of those paths are tested; before,
+they were a `finally` block that had already been written wrong once.
+
+**Cancellation has one shape.** `signal.aborted` (the caller gave up),
+`handle.cancelled` (someone clicked Cancel) and `lifecycle.closing` (the server
+is going down) are three different events that all mean stop, and they now read
+as three lines in one place. The caller-gave-up case did not exist before.
+
+**Shutdown stopped being a closure variable.** `lifecycle` is a dependency, so an
+action that must stop when the server is going down has to say so.
+
+## What it cost, and what to watch
+
+**A dispatch costs a turn of the event loop.** `dispatch()` returns
+synchronously and the action runs on a later macrotask, so `beginScan` awaits a
+`started` event rather than having the task record in hand. Microseconds — but
+it moved a real boundary. The `alreadyRunning` answer used to be decided before
+`beginScan` returned; now the claim is taken one turn later. Two scan requests
+arriving in the same tick can therefore both run, back to back, instead of the
+second being turned away. That is wasteful, not incorrect — the claim still
+guarantees they never overlap, which is the property that protects the resume
+cursor — but two tests were asserting the old timing and had to be given a
+storage backend that genuinely blocks so the overlap is real.
+
+**`next()` semantics need care at the seam.** `done` was first written as
+`feed.next(['result'])`, which the abort contract correctly pre-empts — aborting
+rejects anything waiting. But `done` is the work's own completion, not a caller
+who gave up, and an aborted scan still produces a partial result with
+`stopped: true` that a resumable scan needs to report. Naming `abort` in the wait
+opts out of the pre-emption. Worth knowing before forty routes are written
+against it.
+
+**Half the graph is still singletons.** `vfs`, `tasks` and `kv` are wrapped
+instances rather than providers that build their own resources. That is the
+honest first step — the graph moves under the container a piece at a time — but
+it means this spike does not yet demonstrate the container's teardown ordering
+or lazy initialisation, which is where the remaining value of `createServer`'s
+474 lines actually is.
+
+## Not done here
+
+- The two-domain split (request vs background) as a first-class idea rather than
+  a Workers concession.
+- `backgroundWork` as a provider with local and Durable Object implementations.
+  The seam exists already as `config.background`; it has not been renamed.
+- Anything about queries. One-shot HTTP reads through a boot/kill subscription
+  store is the wrong shape; inside the Durable Object it is the right one.
+
+## Running it
+
+The spike needs ngin with `DispatchFeed.next`/`abort`
+([3sln/ngin#2](https://github.com/3sln/ngin/pull/2)), which is unpublished. Until
+it lands, `node_modules/@3sln/ngin` must point at a checkout of that branch.

package/packages/server/src/engine/actions/scanCollection.js

@@ -0,0 +1,114 @@
+// Reconcile a collection against the bytes actually in its store.
+//
+// SPIKE. The same work `beginScan` did by hand, as an ngin action — which is
+// the interesting comparison, because the hand-rolled version had grown three
+// things an action gets for free:
+//
+//   • a `{ task, alreadyRunning, done }` return shape, invented because a route
+//     must answer immediately with something watchable while the work runs on.
+//     That is a feed: `started` now, `result` later.
+//   • progress reported by poking a TaskRegistry, which clients then poll. That
+//     is a feed too — and one an SSE endpoint could subscribe to directly.
+//   • a try/finally releasing the claim, which released too early the first
+//     time it was written. The container does it, and cannot get it wrong.
+
+import { Action } from '@3sln/ngin';
+
+export class ScanCollection extends Action {
+  static deps = ['vfs', 'tasks', 'lifecycle'];
+
+  /**
+   * @param {object} opts
+   * @param {string} [opts.collectionId]
+   * @param {string} [opts.reason]     shown under the task title
+   * @param {number|null} [opts.deadlineMs] budget for one slice; null runs to the end
+   */
+  constructor({ collectionId = 'default', reason = null, deadlineMs = null } = {}) {
+    super();
+    this.collectionId = collectionId;
+    this.reason = reason;
+    this.deadlineMs = deadlineMs;
+    // Instance deps carry options to the provider — this is how the claim knows
+    // which collection it is claiming.
+    this.deps = { claim: { collectionId } };
+  }
+
+  async execute({ vfs, tasks, lifecycle, claim }, { dispatchFeed, signal }) {
+    if (!claim.held) {
+      // Someone else is scanning this collection. Say so and stop; this is an
+      // ordinary answer, not a failure, so the dispatch completes normally.
+      emit(dispatchFeed, 'started', { task: null, alreadyRunning: true });
+      return;
+    }
+
+    const { task, done } = tasks.begin(
+      {
+        kind: 'scan',
+        title: `Scanning “${this.collectionId}” for outside changes`,
+        detail: this.reason,
+        unit: 'objects',
+        collectionId: this.collectionId,
+        cancellable: true,
+      },
+      (handle) => this.#scan({ vfs, lifecycle, claim, handle, dispatchFeed, signal }),
+    );
+
+    // The route is waiting on this: it has the record to answer with long before
+    // there is a result to report.
+    emit(dispatchFeed, 'started', { task, alreadyRunning: false });
+    emit(dispatchFeed, 'result', { result: await done });
+  }
+
+  async #scan({ vfs, lifecycle, claim, handle, dispatchFeed, signal }) {
+    // A budget, when the caller has one. On Workers a request has a CPU ceiling
+    // measured in seconds and a bucket has none at all, so "run until done" is
+    // not a thing that can be promised — this runs a slice and remembers where
+    // it got to.
+    const until = this.deadlineMs ? Date.now() + this.deadlineMs : null;
+    const cursor = await claim.readCursor();
+
+    // A scan with no deadline runs for minutes, well past the claim. Renew as it
+    // goes, and treat a failed renewal as a stop: something else owns this
+    // collection now, and carrying on would put us back in the race the claim
+    // exists to prevent.
+    let lost = false;
+    let renewedAt = Date.now();
+
+    const result = await vfs.scanCollection(this.collectionId, {
+      cursor,
+      shouldStop: () =>
+        lifecycle.closing
+        || signal.aborted // the caller gave up — see DispatchFeed.abort
+        || handle.cancelled // …or clicked Cancel on the task
+        || lost
+        || (until != null && Date.now() > until),
+      // The store cannot say how many objects it holds without listing them, so
+      // this is honestly indeterminate: a count that rises, with no total to
+      // divide it by.
+      onProgress: ({ scanned, adopted, refreshed }) => {
+        handle.progress({
+          done: scanned,
+          detail: adopted || refreshed ? `${adopted} new, ${refreshed} changed` : null,
+        });
+        // The same numbers on the feed. Nothing consumes these yet — the task
+        // registry is still what /api/tasks reports from — but this is the shape
+        // that makes streaming progress possible without a second mechanism.
+        emit(dispatchFeed, 'progress', { scanned, adopted, refreshed });
+        if (Date.now() - renewedAt < 20_000) return;
+        renewedAt = Date.now();
+        claim.renew().then((ok) => { if (!ok) lost = true; }).catch(() => {});
+      },
+    });
+
+    // Record the resume point, or clear it once a pass completes — but only
+    // while we still hold the claim. Writing a cursor we no longer own is
+    // exactly the clobber the claim is here to stop.
+    if (lost || !(await claim.renew())) return { ...result, lostLease: true };
+    await claim.writeCursor(result.nextCursor);
+    return result;
+  }
+}
+
+function emit(feed, type, detail) {
+  feed.dispatchEvent(Object.assign(new Event(type), detail));
+}

package/packages/server/src/engine/index.js

@@ -0,0 +1,95 @@
+// The drive as an ngin engine.
+//
+// `createServer` is a facade over this: it builds the container, obtains the
+// backbone, and wires the HTTP surface on top. That layering is what makes the
+// migration checkable — every existing test calls `createServer` and drives
+// HTTP, so if they still pass, the rewrite changed no behaviour.
+
+import { Engine } from '@3sln/ngin';
+import { coreProviders } from './providers/core.js';
+import { ScanClaimProvider } from './providers/scan.js';
+import {
+  NodeAccessProvider, CollectionAccessProvider, UploadAccessProvider,
+  SystemNodeProvider, SystemCollectionProvider,
+} from './providers/access.js';
+import { ScanCollection } from './actions/scanCollection.js';
+
+export { ScanCollection, ScanClaimProvider };
+export { buildStorage } from './providers/core.js';
+
+/**
+ * Every provider the drive has.
+ *
+ * @param {object} config the same config object `createServer` takes
+ * @param {{closing: boolean}} lifecycleState mutable; `close()` flips it
+ */
+export function driveProviders(config, lifecycleState) {
+  return {
+    ...coreProviders(config, lifecycleState),
+    claim: ScanClaimProvider,
+    // Authorization as something you hold rather than something you checked —
+    // see providers/access.js. Denial happens during the lease, so an action
+    // that may not act never runs at all.
+    node: NodeAccessProvider,
+    collection: CollectionAccessProvider,
+    upload: UploadAccessProvider,
+    // The background domain's grant, named separately so declaring it is a
+    // visible decision rather than an option someone passed.
+    systemNode: SystemNodeProvider,
+    systemCollection: SystemCollectionProvider,
+  };
+}
+
+/** @returns {Engine} */
+export function createDriveEngine(config = {}, lifecycleState = { closing: false }) {
+  return new Engine({ providers: driveProviders(config, lifecycleState) });
+}
+
+/** The names `createServer` hands back, and therefore obtains up front. */
+export const BACKBONE = [
+  'storage', 'sqlite', 'metadata', 'kv', 'tasks', 'issues', 'notifications',
+  'sidecar', 'collections', 'identity', 'auth', 'search', 'vfs', 'plugins',
+  'lifecycle',
+];
+
+/** The shape `beginScan` has always returned, so no caller has to change. */
+const NOTHING_SCANNED = {
+  scanned: 0, adopted: 0, refreshed: 0, orphaned: 0, skipped: 0,
+  failed: 0, unaddressable: 0, stopped: false, nextCursor: null, resumed: false,
+};
+
+/**
+ * `beginScan(collectionId, opts) -> { task, alreadyRunning, done, feed }`,
+ * backed by a dispatch.
+ *
+ * @param {Engine} engine
+ */
+export function scanStarter(engine) {
+  return async function beginScan(collectionId = 'default', { reason, deadlineMs = null } = {}) {
+    const feed = engine.dispatch(new ScanCollection({ collectionId, reason, deadlineMs }));
+
+    // `started` carries the task record. Awaiting it costs one turn of the event
+    // loop and is what lets the route answer with something watchable.
+    const started = await feed.next(['started']);
+    if (started.alreadyRunning) {
+      return {
+        task: null,
+        alreadyRunning: true,
+        done: Promise.resolve({ alreadyRunning: true, ...NOTHING_SCANNED }),
+        feed,
+      };
+    }
+
+    // `abort` is named so the wait is not pre-empted by aborting. That rule is
+    // for a caller who gave up; this promise is the work's own completion, and
+    // an aborted scan still produces a result — a partial one, with
+    // `stopped: true` — which is exactly what a resumable scan needs to report.
+    const done = feed.next(['result', 'abort']).then((event) => {
+      if (event.type === 'result') return event.result;
+      // No result means the action never reached the end of a scan: it threw.
+      throw event.error ?? event.reason;
+    });
+    done.catch(() => {});
+    return { task: started.task, alreadyRunning: false, done, feed };
+  };
+}

package/packages/server/src/engine/lazy.js

@@ -0,0 +1,25 @@
+// Obtaining several backbone resources at once.
+//
+// The lazy-singleton provider that used to live here is `Provider.fromLazySingleton`
+// in ngin now (3sln/ngin#3) — it was the one shape the library was missing, and
+// every resource in this drive's graph is that shape.
+//
+// What is left is Trove's own convenience, and it is only safe because of what
+// that shape guarantees.
+
+/**
+ * Obtain several dependency providers at once.
+ *
+ * NOT released, deliberately: everything in the core graph is a lazy singleton
+ * whose `release` is a no-op, and a resource built from these holds them for its
+ * own lifetime — releasing would be a lie about a lease nobody is keeping.
+ * Anything pooled or ref-counted must be leased properly instead, which is why
+ * ScanClaimProvider does its own obtain/release rather than coming through here.
+ */
+export async function need(providers, names) {
+  const out = {};
+  for (const name of names) {
+    out[name] = await providers[name].obtain();
+  }
+  return out;
+}