@hachej/boring-files 0.2.0 → 0.2.2

package/dist/workspace.js

@@ -2,7 +2,10 @@ import { createWorkspaceCatalog } from './workspace-catalog.js';
 import { committedChanges } from './journal.js';
 import { randomUUID, sha1 } from './platform.js';
 import { accessSnapshot, identifier, locator, publicationDigest, publicationSnapshot } from './publication-input.js';
+import { assertLimit, fileTooLarge } from './file-limits.js';
 import { journalConnectionOf, sqliteBatchOf } from './sqlite-batch.js';
+import { isTemporary, mediaTypeOf, temporaryFileName } from './file-types.js';
+export { isTemporary } from './file-types.js';
 const sameBytes = (left, right) => left.length === right.length && left.every((byte, index) => byte === right[index]);
 const queues = new WeakMap();
 /** One queue per file system object: two providers over the same `fs` share it. */
@@ -30,16 +33,6 @@ export async function blobRevision(bytes) {
     whole.set(bytes, header.length);
     return hex(await sha1(whole));
 }
-const mediaTypes = {
-    md: 'text/markdown', markdown: 'text/markdown', html: 'text/html', htm: 'text/html', json: 'application/json', svg: 'image/svg+xml', txt: 'text/plain', tldraw: 'application/vnd.tldraw+json',
-};
-function mediaTypeOf(path) {
-    return mediaTypes[path.slice(path.lastIndexOf('.') + 1).toLowerCase()] ?? 'application/octet-stream';
-}
-/** The provider's own temporary files. They are removed when the provider opens and are never a resource. */
-const temporaryPattern = /^\.boring-[0-9a-f-]{36}\.tmp$/;
-/** Whether a path is one of the provider's temporary files (a lister hides them). */
-export const isTemporary = (path) => temporaryPattern.test(path.slice(path.lastIndexOf('/') + 1));
 export function createWorkspaceProvider(options) {
     const { fs, journal } = options;
     const identity = options.identity;
@@ -58,6 +51,11 @@ export function createWorkspaceProvider(options) {
     const queue = queueFor(fs);
     /** Present when `fs` is the SQLite file system (`@hachej/boring-files/sqlite-filesystem`): multi-file batches then commit in one transaction. */
     const batch = sqliteBatchOf(fs);
+    if (options.maxFileBytes !== undefined) {
+        assertLimit(options.maxFileBytes);
+        batch?.setMaxFileBytes(options.maxFileBytes);
+    }
+    const limitOf = () => batch ? batch.maxFileBytes() : options.maxFileBytes;
     const historyScope = JSON.stringify([key.providerId, key.instanceId, key.incarnation, key.viewId]);
     const listeners = new Set();
     const known = new Map();
@@ -71,14 +69,38 @@ export function createWorkspaceProvider(options) {
         principal: access.principalId, initiator: access.initiatorId, operation: operationId,
     });
     const ownEvidence = (evidence) => evidence.startsWith(`${key.incarnation}:`);
-    function emit(change) {
-        known.set(change.path, change.revision);
+    // ---- The change feed: a bounded, seq-ordered buffer of invalidation hints ----------------------------------------------------
+    const bufferLimit = options.changeBuffer ?? 1024;
+    if (!Number.isSafeInteger(bufferLimit) || bufferLimit < 1)
+        throw new RangeError('The change buffer must be a positive integer');
+    const lingerMs = options.watchLingerMs ?? 30_000;
+    // Sequence numbers start from the clock (microseconds), so a cursor from an earlier provider lifetime is older than this one's
+    // first event and reads as a gap, never as a position in this lifetime.
+    const firstSeq = Date.now() * 1000;
+    let head = firstSeq;
+    // Names this provider lifetime. A cursor from another lifetime is refused by this, not only by the clock: the clock may go back.
+    const incarnationOfFeed = randomUUID();
+    const lifetime = new AbortController();
+    const buffer = [];
+    const waiting = new Set();
+    function append(event) {
+        head += 1;
+        buffer.push(Object.freeze({ ...event, seq: head, at: Date.now() }));
+        if (buffer.length > bufferLimit)
+            buffer.shift();
+        for (const wake of [...waiting])
+            wake();
+    }
+    function emit(change, extra = {}) {
+        if (!extra.directory)
+            known.set(change.path, change.revision);
         for (const listener of [...listeners]) {
             try {
                 listener(change);
             }
             catch { /* a listener failure cannot undo a write */ }
         }
+        append({ kind: 'change', ...change, ...extra });
     }
     const dirname = (path) => path.slice(0, path.lastIndexOf('/')) || '/';
     const inside = (base, path) => path === base || path.startsWith(base === '/' ? '/' : `${base}/`);
@@ -182,7 +204,7 @@ export function createWorkspaceProvider(options) {
         const result = await read({ target: { resource: { providerId, path }, view: { kind: 'published' } }, revision: { kind: 'latest' } }, access);
         if (result.kind === 'available') {
             const path = result.snapshot.ref.resource.path;
-            journal.remember(historyScope, path, { revision: result.snapshot.ref.revision, bytes: result.snapshot.bytes, mediaType: result.snapshot.mediaType, savedAt: await savedAtOf(path) });
+            journal.remember(historyScope, path, { revision: result.snapshot.ref.revision, bytes: result.snapshot.bytes, mediaType: result.snapshot.mediaType, savedAt: await savedAtOf(path), source: 'observed' });
         }
         return result;
     });
@@ -228,6 +250,15 @@ export function createWorkspaceProvider(options) {
             }
             if (journal.intent(operation))
                 return { kind: 'unknown', operationId: request.operationId, reason: 'An earlier attempt of this operation began and has no recorded outcome' };
+            // After the receipt and intent checks, so a replay of a committed operation keeps its result under a lower limit.
+            const limit = limitOf();
+            if (limit !== undefined) {
+                const oversized = changes.find(change => change.bytes.length > limit);
+                if (oversized) {
+                    const size = oversized.bytes.length, path = oversized.target.resource.path;
+                    return { kind: 'file_too_large', path, size, limit, reason: fileTooLarge(path, size, limit).message };
+                }
+            }
             journal.begin(operation, digest);
             let applying = false;
             const refuse = (reason) => { journal.cancel(operation); return { kind: 'unavailable', reason }; };
@@ -276,11 +307,11 @@ export function createWorkspaceProvider(options) {
                 }
                 const evidence = `${key.incarnation}:${randomUUID()}`;
                 const versions = plans.flatMap(({ change, path, before, after, unchanged, beforeSavedAt }) => {
-                    const kept = (revision, bytes, mediaType, savedAt) => ({ scope: historyScope, path, version: { revision, bytes, mediaType, savedAt } });
+                    const kept = (revision, bytes, mediaType, savedAt, source) => ({ scope: historyScope, path, version: { revision, bytes, mediaType, savedAt, source } });
                     return unchanged ? [] : [
                         ...(change.kind === 'replace' && before?.kind === 'file'
-                            ? [kept(before.revision, before.bytes, journal.version(historyScope, path, before.revision)?.mediaType ?? mediaTypeOf(path), beforeSavedAt)] : []),
-                        kept(after, change.bytes, change.mediaType, Date.now()),
+                            ? [kept(before.revision, before.bytes, journal.version(historyScope, path, before.revision)?.mediaType ?? mediaTypeOf(path), beforeSavedAt, 'observed')] : []),
+                        kept(after, change.bytes, change.mediaType, Date.now(), 'publish'),
                     ];
                 });
                 const record = {
@@ -321,7 +352,7 @@ export function createWorkspaceProvider(options) {
                     if (!unchanged) {
                         applying = true;
                         const target = absolute(path);
-                        const temporary = `${target.slice(0, target.lastIndexOf('/'))}/.boring-${randomUUID()}.tmp`;
+                        const temporary = `${target.slice(0, target.lastIndexOf('/'))}/${temporaryFileName(randomUUID())}`;
                         const ctx = fsContext(access.signal);
                         const failure = async (reason) => { await fs.remove(temporary, { force: true }, fsContext()); return refuse(reason); };
                         if (change.kind === 'create') {
@@ -343,7 +374,7 @@ export function createWorkspaceProvider(options) {
                 }
                 for (const { path, after, unchanged } of plans)
                     if (!unchanged)
-                        emit({ path, revision: after, source: 'provider' });
+                        emit({ path, revision: after, source: 'publish' });
                 return { kind: 'committed', receipt: receipt(request.operationId, access, record) };
             }
             catch (error) {
@@ -369,20 +400,63 @@ export function createWorkspaceProvider(options) {
             ? { kind: 'unknown', operationId: id, reason: 'The operation began and has no recorded outcome' }
             : { kind: 'not-found' };
     };
+    /** Every file of the workspace, relative to the root (not `.git`, `node_modules` or the provider's temporary files). */
+    async function everyFile(signal) {
+        const found = [];
+        let scanned = 0;
+        const walk = async (directory) => {
+            const listing = await fs.listDir(directory ? absolute(directory) : root, fsContext(signal));
+            if (!listing.ok)
+                return true;
+            for (const entry of listing.value) {
+                if (++scanned > 100_000)
+                    return false;
+                const path = directory ? `${directory}/${entry.name}` : entry.name;
+                if (entry.kind === 'directory') {
+                    if (entry.name !== '.git' && entry.name !== 'node_modules' && !await walk(path))
+                        return false;
+                }
+                else if (entry.kind === 'file' && !isTemporary(path))
+                    found.push(path);
+            }
+            return true;
+        };
+        return await walk('') ? found : undefined;
+    }
+    let baselined = false;
     const poll = async (paths, signal) => {
         const changed = [];
         await sweep();
-        for (const input of paths) {
+        let inputs;
+        let report = (path) => known.has(path);
+        if (paths === undefined) {
+            const files = await everyFile(signal);
+            // Too many files to hash on each turn: say so to every listener instead of reporting a partial picture.
+            if (files === undefined) {
+                append({ kind: 'resnapshot' });
+                return changed;
+            }
+            const present = new Set(files);
+            const removed = [...known].filter(([path, revision]) => revision !== null && !present.has(path)).map(([path]) => path);
+            inputs = [...files, ...removed];
+            if (baselined)
+                report = () => true;
+            baselined = true;
+        }
+        else
+            inputs = paths;
+        for (const input of inputs) {
             const path = locator({ resource: { providerId, path: input }, view: { kind: 'published' } }).resource.path;
             const now = await observe(path, signal);
             if (now.kind === 'error' || now.kind === 'outside' || isTemporary(path))
                 continue;
             const revision = now.kind === 'file' ? now.revision : null;
-            const first = !known.has(path);
-            if (!first && known.get(path) === revision)
+            if (known.has(path) && known.get(path) === revision)
+                continue;
+            if (!known.has(path) && revision === null && paths === undefined)
                 continue;
             const change = { path, revision, source: 'poll' };
-            if (first)
+            if (!report(path))
                 known.set(path, revision);
             else {
                 emit(change);
@@ -391,10 +465,181 @@ export function createWorkspaceProvider(options) {
         }
         return changed;
     };
+    const record = async (write, context) => {
+        const access = accessSnapshot(context);
+        if (write.source !== 'agent')
+            throw new TypeError('Only agent writes are recorded');
+        const path = locator({ resource: { providerId, path: write.path }, view: { kind: 'published' } }).resource.path;
+        const after = identifier(write.after);
+        const conversationId = write.conversationId === undefined ? undefined : identifier(write.conversationId);
+        // Known before the await: a watcher event for this same write, handled meanwhile, then finds nothing new and emits nothing twice.
+        if (!isTemporary(path))
+            known.set(path, after);
+        // Deliberately not `queue.run`: the file guard calls this from inside the queue, and the queue runs one piece at a time.
+        const now = await observe(path, access.signal);
+        if (now.kind === 'outside' || now.kind === 'error')
+            return;
+        if (now.kind === 'file' && now.revision === after) {
+            const mediaType = journal.version(historyScope, path, after)?.mediaType ?? mediaTypeOf(path);
+            const before = write.before ?? undefined;
+            // The replaced bytes are kept only when they really are that revision.
+            if (before && before.revision !== after && before.bytes instanceof Uint8Array && await blobRevision(before.bytes) === identifier(before.revision)) {
+                journal.remember(historyScope, path, { revision: before.revision, bytes: Uint8Array.from(before.bytes), mediaType: journal.version(historyScope, path, before.revision)?.mediaType ?? mediaType, savedAt: 0, source: 'observed' });
+            }
+            journal.remember(historyScope, path, { revision: after, bytes: now.bytes, mediaType, savedAt: Date.now(), source: 'agent' });
+        }
+        emit({ path, revision: now.kind === 'file' ? now.revision : null, source: 'agent' }, conversationId === undefined ? {} : { conversationId });
+    };
+    // ---- The watcher: open while someone listens to the feed (and `lingerMs` after), when the file system can watch ----------------
+    let listening = 0;
+    let watcher;
+    let watchUnsupported = false, watchedBefore = false;
+    let linger;
+    let watchWork = Promise.resolve();
+    const relativeTo = (absolutePath, base) => {
+        base = base.replace(/\/+$/, '') || '/';
+        if (absolutePath === base)
+            return '';
+        const prefix = base === '/' ? '/' : `${base}/`;
+        return absolutePath.startsWith(prefix) ? absolutePath.slice(prefix.length) : undefined;
+    };
+    async function watched(input) {
+        // The watcher may name the root as given or as resolved (a symbolic link on the way to the workspace).
+        const real = await realRoot();
+        const path = relativeTo(input, root) ?? (real === undefined ? undefined : relativeTo(input, real));
+        if (path === undefined || isTemporary(path) || path.split('/').some(name => name === '.git' || name === 'node_modules'))
+            return;
+        if (path === '') {
+            emit({ path, revision: null, source: 'watch' }, { directory: true });
+            return;
+        }
+        const info = await fs.fileInfo(absolute(path), fsContext());
+        if (info.ok && info.value.kind === 'directory') {
+            emit({ path, revision: null, source: 'watch' }, { directory: true });
+            return;
+        }
+        const now = await observe(path);
+        if (now.kind === 'error' || now.kind === 'outside')
+            return;
+        const revision = now.kind === 'file' ? now.revision : null;
+        if (known.has(path) && known.get(path) === revision)
+            return;
+        emit({ path, revision, source: 'watch' });
+    }
+    function onWatch(change) {
+        if ('paths' in change) {
+            const paths = [...change.paths];
+            watchWork = watchWork.then(async () => { for (const path of paths)
+                await watched(path).catch(() => undefined); });
+            return;
+        }
+        // Overflow or a failed watcher: what changed is unknown, so every listener reloads. A failed watcher is not reopened (it would
+        // fail again, for example on a tree past the environment's limit); the host's `poll()` at turn end remains.
+        if ('error' in change) {
+            watcher = undefined;
+            watchUnsupported = true;
+        }
+        append({ kind: 'resnapshot' });
+    }
+    function startWatching() {
+        if (lifetime.signal.aborted)
+            return;
+        if (linger !== undefined) {
+            clearTimeout(linger);
+            linger = undefined;
+        }
+        if (watcher || watchUnsupported)
+            return;
+        // Changes made while nobody watched were not seen: a listener resuming from before that reloads.
+        if (watchedBefore)
+            append({ kind: 'resnapshot' });
+        watchedBefore = true;
+        watcher = fs.watch([{ path: root, recursive: true, exclude: { names: ['.git', 'node_modules'] } }], onWatch, fsContext()).then(result => {
+            if (result.ok)
+                return result.value;
+            watchUnsupported = true;
+            return undefined;
+        }, () => { watchUnsupported = true; return undefined; });
+    }
+    function stopWatching() {
+        const current = watcher;
+        watcher = undefined;
+        void current?.then(value => value?.close(fsContext()));
+    }
+    async function* changes(request, context) {
+        const access = accessSnapshot(context);
+        identifier(access.scopeId);
+        const since = request.since;
+        if (since !== undefined && (!Number.isSafeInteger(since) || since < 0))
+            throw new TypeError('Invalid change cursor');
+        if (lifetime.signal.aborted)
+            return;
+        const signals = [request.signal, access.signal, lifetime.signal].filter((value) => value !== undefined);
+        const signal = signals.length ? AbortSignal.any(signals) : new AbortController().signal;
+        listening += 1;
+        startWatching();
+        try {
+            let cursor = head;
+            if (since !== undefined) {
+                const oldest = buffer[0]?.seq ?? head + 1;
+                const foreign = request.incarnation !== undefined && request.incarnation !== incarnationOfFeed;
+                if (foreign || since > head || since < firstSeq || since < oldest - 1)
+                    yield { kind: 'resnapshot', seq: head, at: Date.now() };
+                else
+                    cursor = since;
+            }
+            while (!signal.aborted) {
+                const oldest = buffer[0];
+                // This listener fell behind the retained window: it reloads, then continues with what is retained.
+                if (oldest && oldest.seq > cursor + 1) {
+                    cursor = oldest.seq - 1;
+                    yield { kind: 'resnapshot', seq: cursor, at: Date.now() };
+                    continue;
+                }
+                const next = buffer.find(event => event.seq > cursor);
+                if (next) {
+                    cursor = next.seq;
+                    // The access projection: nothing after revocation, never the provider's temporary files.
+                    if (signal.aborted)
+                        return;
+                    if (next.kind === 'change' && isTemporary(next.path))
+                        continue;
+                    yield next;
+                    continue;
+                }
+                await new Promise(resolve => {
+                    const wake = () => { waiting.delete(wake); signal.removeEventListener('abort', wake); resolve(); };
+                    waiting.add(wake);
+                    signal.addEventListener('abort', wake, { once: true });
+                });
+            }
+        }
+        finally {
+            listening -= 1;
+            if (listening === 0 && watcher && !lifetime.signal.aborted) {
+                linger = setTimeout(() => { linger = undefined; if (listening === 0)
+                    stopWatching(); }, lingerMs);
+                linger.unref?.();
+            }
+        }
+    }
     return {
         providerId, read, queue, keep, identity: Object.freeze(key),
         catalog: createWorkspaceCatalog({ fs, ...(options.catalog ? { config: options.catalog } : {}), saves: path => journal.saves(historyScope, path) }),
-        publication: { publish }, reconciliation: { lookup }, poll,
+        publication: { publish }, reconciliation: { lookup }, poll, record, changes, changeHead: () => head, changeIncarnation: incarnationOfFeed,
+        lifetime: lifetime.signal,
+        close: () => {
+            // Every change stream ends, the watcher closes and the listeners are dropped: nothing of this provider keeps running.
+            lifetime.abort();
+            if (linger !== undefined) {
+                clearTimeout(linger);
+                linger = undefined;
+            }
+            stopWatching();
+            listeners.clear();
+            for (const wake of [...waiting])
+                wake();
+        },
         history: path => journal.revisions(historyScope, locator({ resource: { providerId, path }, view: { kind: 'published' } }).resource.path),
         saves: path => journal.saves(historyScope, locator({ resource: { providerId, path }, view: { kind: 'published' } }).resource.path),
         onChange: listener => { listeners.add(listener); return () => { listeners.delete(listener); }; },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hachej/boring-files",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "license": "MIT",
   "private": false,
   "type": "module",
@@ -81,12 +81,16 @@
     "./revision-handler": {
       "types": "./dist/revision-handler.d.ts",
       "import": "./dist/revision-handler.js"
+    },
+    "./file-types": {
+      "types": "./dist/file-types.d.ts",
+      "import": "./dist/file-types.js"
     }
   },
   "peerDependencies": {
-    "isomorphic-git": "1.42.6",
-    "@earendil-works/pi-durable": "1.1.0",
-    "@earendil-works/chord": "1.1.0"
+    "isomorphic-git": "^1.42.6",
+    "@earendil-works/pi-durable": "^1.1.0",
+    "@earendil-works/chord": "^1.1.0"
   },
   "peerDependenciesMeta": {
     "isomorphic-git": {