@hachej/boring-files 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -1
- package/dist/contracts.d.ts +8 -0
- package/dist/file-limits.d.ts +10 -0
- package/dist/file-limits.js +16 -0
- package/dist/file-types.d.ts +30 -0
- package/dist/file-types.js +61 -0
- package/dist/journal.d.ts +7 -0
- package/dist/journal.js +11 -3
- package/dist/publication-input.js +6 -0
- package/dist/revision-contracts.d.ts +41 -0
- package/dist/revision-handler.d.ts +3 -0
- package/dist/revision-handler.js +109 -1
- package/dist/revision-protocol.d.ts +8 -1
- package/dist/revision-protocol.js +65 -1
- package/dist/revision.js +89 -1
- package/dist/sqlite-batch.d.ts +3 -0
- package/dist/sqlite-filesystem.d.ts +2 -0
- package/dist/sqlite-filesystem.js +75 -12
- package/dist/workspace-catalog.js +22 -9
- package/dist/workspace.d.ts +66 -8
- package/dist/workspace.js +268 -23
- package/package.json +8 -4
package/dist/revision.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createResourceClient } from './remote.js';
|
|
2
|
-
import { binding, bindingHeader, bindingValue, catalogHeader, filePage, record, savedRevisions } from './revision-protocol.js';
|
|
2
|
+
import { binding, bindingHeader, bindingValue, catalogHeader, changeEvent, filePage, record, savedRevisions, serverEvents } from './revision-protocol.js';
|
|
3
3
|
import { readJsonBody } from './request-guard.js';
|
|
4
4
|
import { RevisionError } from './revision-contracts.js';
|
|
5
5
|
import { observe } from './remote-protocol.js';
|
|
@@ -45,6 +45,93 @@ export async function connectRevisionProvider(options) {
|
|
|
45
45
|
throw new RevisionError('denied', 'The response belongs to another workspace');
|
|
46
46
|
return result.value;
|
|
47
47
|
}
|
|
48
|
+
/** One feed across reconnections: each resumes after the last seq delivered; a refused binding ends it. */
|
|
49
|
+
async function* changes(request = {}) {
|
|
50
|
+
const signal = request.signal;
|
|
51
|
+
let cursor = request.since, incarnation = request.incarnation, staged, delay = 500;
|
|
52
|
+
const pause = () => new Promise(resolve => {
|
|
53
|
+
const timer = setTimeout(done, delay);
|
|
54
|
+
function done() { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); }
|
|
55
|
+
signal?.addEventListener('abort', done, { once: true });
|
|
56
|
+
delay = Math.min(delay * 2, 30_000);
|
|
57
|
+
});
|
|
58
|
+
while (!signal?.aborted) {
|
|
59
|
+
const url = new URL(endpoint);
|
|
60
|
+
url.searchParams.set('op', 'changes');
|
|
61
|
+
if (cursor !== undefined) {
|
|
62
|
+
url.searchParams.set('since', String(cursor));
|
|
63
|
+
if (incarnation !== undefined)
|
|
64
|
+
url.searchParams.set('incarnation', incarnation);
|
|
65
|
+
}
|
|
66
|
+
const connection = new AbortController();
|
|
67
|
+
const stop = () => connection.abort();
|
|
68
|
+
signal?.addEventListener('abort', stop, { once: true });
|
|
69
|
+
try {
|
|
70
|
+
let response;
|
|
71
|
+
try {
|
|
72
|
+
response = await fetch(new Request(url, { method: 'GET', redirect: 'error', headers: { accept: 'text/event-stream', [bindingHeader]: bound }, signal: connection.signal }));
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
if (signal?.aborted)
|
|
76
|
+
return;
|
|
77
|
+
await pause();
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
if (response.redirected || !response.ok || !response.body) {
|
|
81
|
+
void response.body?.cancel().catch(() => { });
|
|
82
|
+
if (response.status === 403 || response.status === 409)
|
|
83
|
+
throw new RevisionError('denied', 'The selected workspace is unavailable or no longer authorized');
|
|
84
|
+
if (response.status === 400)
|
|
85
|
+
throw new RevisionError('invalid', 'The change feed request was refused');
|
|
86
|
+
await pause();
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
for await (const message of serverEvents(response.body)) {
|
|
91
|
+
if (message.event === 'ready') {
|
|
92
|
+
const ready = record(JSON.parse(message.data)), seq = ready.seq;
|
|
93
|
+
// A new provider lifetime is adopted with its resnapshot, not with `ready`: a disconnect in between must reconnect with the
|
|
94
|
+
// old cursor and incarnation, which the server answers with a resnapshot again.
|
|
95
|
+
if (typeof ready.incarnation === 'string') {
|
|
96
|
+
if (cursor === undefined || incarnation === undefined)
|
|
97
|
+
incarnation = ready.incarnation;
|
|
98
|
+
else
|
|
99
|
+
staged = ready.incarnation;
|
|
100
|
+
}
|
|
101
|
+
if (cursor === undefined && typeof seq === 'number' && Number.isSafeInteger(seq))
|
|
102
|
+
cursor = seq;
|
|
103
|
+
delay = 500;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (message.event !== 'change' && message.event !== 'resnapshot')
|
|
107
|
+
continue;
|
|
108
|
+
const event = changeEvent(JSON.parse(message.data));
|
|
109
|
+
if (event.kind === 'change' && cursor !== undefined && event.seq <= cursor)
|
|
110
|
+
continue;
|
|
111
|
+
if (event.kind === 'resnapshot' && staged !== undefined) {
|
|
112
|
+
incarnation = staged;
|
|
113
|
+
staged = undefined;
|
|
114
|
+
}
|
|
115
|
+
cursor = event.seq;
|
|
116
|
+
yield event;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
if (signal?.aborted)
|
|
121
|
+
return;
|
|
122
|
+
if (error instanceof RevisionError && error.kind === 'invalid')
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
signal?.removeEventListener('abort', stop);
|
|
128
|
+
connection.abort();
|
|
129
|
+
}
|
|
130
|
+
if (signal?.aborted)
|
|
131
|
+
return;
|
|
132
|
+
await pause();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
48
135
|
if (!client.publish || !client.lookup)
|
|
49
136
|
throw new Error('Publishing client lacks reconciliation');
|
|
50
137
|
return {
|
|
@@ -53,5 +140,6 @@ export async function connectRevisionProvider(options) {
|
|
|
53
140
|
list: async (request, signal) => filePage(await invoke('list', request, signal)),
|
|
54
141
|
search: async (request, signal) => filePage(await invoke('search', request, signal)),
|
|
55
142
|
history: async (path, signal) => savedRevisions(await invoke('history', path, signal)),
|
|
143
|
+
changes: request => changes(request),
|
|
56
144
|
};
|
|
57
145
|
}
|
package/dist/sqlite-batch.d.ts
CHANGED
|
@@ -10,6 +10,9 @@ export interface SqliteBatch {
|
|
|
10
10
|
readonly bytes: (path: string) => Uint8Array | null | undefined;
|
|
11
11
|
/** Write a regular file at an absolute path, creating its missing parent directories. Throws a Pi `FileError` on failure. */
|
|
12
12
|
readonly write: (path: string, bytes: Uint8Array) => void;
|
|
13
|
+
/** The one byte limit of this file system: native writes and publications both enforce it. */
|
|
14
|
+
readonly maxFileBytes: () => number;
|
|
15
|
+
readonly setMaxFileBytes: (limit: number) => void;
|
|
13
16
|
}
|
|
14
17
|
export declare const registerSqliteBatch: (fs: object, batch: SqliteBatch) => void;
|
|
15
18
|
export declare const sqliteBatchOf: (fs: object) => SqliteBatch | undefined;
|
|
@@ -7,6 +7,8 @@ export interface SqliteFileSystemOptions {
|
|
|
7
7
|
readonly workspace: string;
|
|
8
8
|
/** The absolute working directory, created when missing (for example `/workspace`). Relative paths resolve against `cwd`. */
|
|
9
9
|
readonly cwd: string;
|
|
10
|
+
/** The largest file a write may store (default `SQLITE_MAX_FILE_BYTES`, the row limit of a Durable Object). A larger write fails with `EFBIG`. */
|
|
11
|
+
readonly maxFileBytes?: number;
|
|
10
12
|
}
|
|
11
13
|
/** A Pi `FileSystem` plus the incarnation of its rows: new when the workspace was first created in this database. */
|
|
12
14
|
export interface SqliteFileSystem extends FileSystem {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { FileError, LineScanner, err, ok } from '@earendil-works/pi-durable/env';
|
|
2
|
+
import { SQLITE_MAX_FILE_BYTES, assertLimit, fileTooLarge } from './file-limits.js';
|
|
2
3
|
import { randomUUID } from './platform.js';
|
|
3
4
|
import { registerSqliteBatch } from './sqlite-batch.js';
|
|
4
5
|
function normalize(path) {
|
|
@@ -20,7 +21,10 @@ const within = (base, path) => path === base || path.startsWith(base === '/' ? '
|
|
|
20
21
|
// system answers exactly like Pi's environments.
|
|
21
22
|
const fileError = (code, message, path) => new FileError(code, message, path);
|
|
22
23
|
const isFileError = (error) => error instanceof FileError;
|
|
23
|
-
/**
|
|
24
|
+
/**
|
|
25
|
+
* How often a watcher compares snapshots. A write through this file system (this connection) is compared at once, right after
|
|
26
|
+
* its transaction; polling remains for writes through another connection to the same database (`mode: 'polling'`).
|
|
27
|
+
*/
|
|
24
28
|
const WATCH_INTERVAL_MS = 1000;
|
|
25
29
|
/** A FileError whose message starts with the POSIX code, as shells and Git read it. */
|
|
26
30
|
const failure = (code, errno, path, what) => fileError(code, `${errno}: ${what}, '${path}'`, path);
|
|
@@ -32,6 +36,8 @@ export function openSqliteFileSystem(options) {
|
|
|
32
36
|
if (typeof options.cwd !== 'string' || !options.cwd.startsWith('/'))
|
|
33
37
|
throw new TypeError('An absolute working directory is required');
|
|
34
38
|
const now = () => Date.now();
|
|
39
|
+
let maxFileBytes = options.maxFileBytes ?? SQLITE_MAX_FILE_BYTES;
|
|
40
|
+
assertLimit(maxFileBytes);
|
|
35
41
|
function row(path) {
|
|
36
42
|
if (path === '/')
|
|
37
43
|
return { kind: 'directory', size: 0, mtime: 0 };
|
|
@@ -65,20 +71,29 @@ export function openSqliteFileSystem(options) {
|
|
|
65
71
|
if (found.kind !== 'directory')
|
|
66
72
|
throw failure('not_directory', 'ENOTDIR', path, 'not a directory');
|
|
67
73
|
}
|
|
74
|
+
/** Paths written, renamed or removed by this file system since the last comparison: reported as changed even when size and time look alike. */
|
|
75
|
+
const touched = new Set();
|
|
68
76
|
function bytes(path) {
|
|
69
|
-
|
|
77
|
+
const entry = existing(path);
|
|
78
|
+
if (entry.kind === 'directory')
|
|
70
79
|
throw failure('is_directory', 'EISDIR', path, 'illegal operation on a directory');
|
|
71
80
|
const found = db.get('SELECT bytes FROM boring_workspace_files WHERE workspace = ? AND path = ?', workspace, path);
|
|
81
|
+
// A zero-length BLOB can come back as no bytes (a NULL pointer through some engines): a file row with no bytes is an empty file.
|
|
82
|
+
if (found?.bytes === null && entry.kind === 'file')
|
|
83
|
+
return new Uint8Array();
|
|
72
84
|
if (!(found?.bytes instanceof Uint8Array))
|
|
73
85
|
throw fileError('unknown', 'Invalid stored bytes', path);
|
|
74
86
|
return Uint8Array.from(found.bytes);
|
|
75
87
|
}
|
|
76
88
|
function put(path, kind, content) {
|
|
89
|
+
touched.add(path);
|
|
77
90
|
db.run('INSERT INTO boring_workspace_files (workspace, path, parent, kind, bytes, mtime_ms) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(workspace, path) DO UPDATE SET kind = excluded.kind, bytes = excluded.bytes, mtime_ms = excluded.mtime_ms', workspace, path, dirname(path), kind, content, now());
|
|
78
91
|
}
|
|
79
92
|
function write(path, content) {
|
|
80
93
|
if (path === '/')
|
|
81
94
|
throw failure('is_directory', 'EISDIR', path, 'illegal operation on a directory');
|
|
95
|
+
if (content.length > maxFileBytes)
|
|
96
|
+
throw fileTooLarge(path, content.length, maxFileBytes);
|
|
82
97
|
const found = row(path);
|
|
83
98
|
if (found?.kind === 'directory')
|
|
84
99
|
throw failure('is_directory', 'EISDIR', path, 'illegal operation on a directory');
|
|
@@ -125,13 +140,31 @@ export function openSqliteFileSystem(options) {
|
|
|
125
140
|
const incarnation = db.get('SELECT incarnation FROM boring_workspaces WHERE workspace = ?', workspace)?.incarnation;
|
|
126
141
|
if (typeof incarnation !== 'string')
|
|
127
142
|
throw new Error('The workspace has no incarnation');
|
|
143
|
+
/** Each open watcher's comparison, run right after a write through this file system commits. */
|
|
144
|
+
const watchers = new Set();
|
|
145
|
+
let notifying = false;
|
|
146
|
+
function notify() {
|
|
147
|
+
if (watchers.size === 0) {
|
|
148
|
+
touched.clear();
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
if (notifying)
|
|
152
|
+
return;
|
|
153
|
+
notifying = true;
|
|
154
|
+
// After the current synchronous work: a batch write inside a caller's transaction is compared once it has committed.
|
|
155
|
+
queueMicrotask(() => { notifying = false; const paths = [...touched]; touched.clear(); for (const check of [...watchers])
|
|
156
|
+
check(paths); });
|
|
157
|
+
}
|
|
128
158
|
async function run(path, context, mode, work) {
|
|
129
159
|
if (context.abortSignal?.aborted)
|
|
130
160
|
return err(fileError('aborted', 'Operation aborted', path));
|
|
131
161
|
if (typeof path !== 'string' || path.includes('\0'))
|
|
132
162
|
return err(fileError('invalid', 'Invalid path', String(path)));
|
|
133
163
|
try {
|
|
134
|
-
|
|
164
|
+
const value = db.transaction(mode, () => work(normalize(path.startsWith('/') ? path : `${fs.cwd}/${path}`)));
|
|
165
|
+
if (mode === 'write')
|
|
166
|
+
notify();
|
|
167
|
+
return ok(value);
|
|
135
168
|
}
|
|
136
169
|
catch (error) {
|
|
137
170
|
return err(isFileError(error) ? error : fileError('unknown', error instanceof Error ? error.message : String(error), path));
|
|
@@ -244,6 +277,9 @@ export function openSqliteFileSystem(options) {
|
|
|
244
277
|
const added = encode(content);
|
|
245
278
|
if (!row(absolute))
|
|
246
279
|
return write(absolute, added);
|
|
280
|
+
const grown = existing(absolute).size + added.length;
|
|
281
|
+
if (grown > maxFileBytes)
|
|
282
|
+
throw fileTooLarge(absolute, grown, maxFileBytes);
|
|
247
283
|
const before = bytes(absolute), joined = new Uint8Array(before.length + added.length);
|
|
248
284
|
joined.set(before);
|
|
249
285
|
joined.set(added, before.length);
|
|
@@ -252,6 +288,9 @@ export function openSqliteFileSystem(options) {
|
|
|
252
288
|
truncateFile: (path, size, context) => run(path, context, 'write', absolute => {
|
|
253
289
|
if (!Number.isSafeInteger(size) || size < 0)
|
|
254
290
|
throw fileError('invalid', 'File size must be a non-negative safe integer', absolute);
|
|
291
|
+
// Checked before the buffer is allocated: a huge size is a size refusal, not an allocation failure.
|
|
292
|
+
if (size > maxFileBytes)
|
|
293
|
+
throw fileTooLarge(absolute, size, maxFileBytes);
|
|
255
294
|
const before = bytes(absolute), after = new Uint8Array(size);
|
|
256
295
|
after.set(before.subarray(0, size));
|
|
257
296
|
write(absolute, after);
|
|
@@ -264,6 +303,12 @@ export function openSqliteFileSystem(options) {
|
|
|
264
303
|
const moved = existing(from);
|
|
265
304
|
if (from === to)
|
|
266
305
|
return;
|
|
306
|
+
// A moved file keeps its bytes, so a file (or a file below a moved directory) over the limit cannot replace another file by renaming.
|
|
307
|
+
const oversized = db.get(`SELECT path, length(bytes) AS size FROM boring_workspace_files WHERE ${subtree(from)} AND kind = 'file' AND length(bytes) > ? LIMIT 1`, ...subtreeArgs(from), maxFileBytes);
|
|
308
|
+
if (oversized)
|
|
309
|
+
throw fileTooLarge(oversized.path, Number(oversized.size), maxFileBytes);
|
|
310
|
+
touched.add(from);
|
|
311
|
+
touched.add(to);
|
|
267
312
|
if (from === '/' || within(from, to))
|
|
268
313
|
throw failure('invalid', 'EINVAL', to, 'cannot move a directory into itself');
|
|
269
314
|
const target = row(to);
|
|
@@ -306,8 +351,9 @@ export function openSqliteFileSystem(options) {
|
|
|
306
351
|
};
|
|
307
352
|
return reader;
|
|
308
353
|
}),
|
|
309
|
-
// SQLite reports no changes, so the watcher compares snapshots of the watched rows
|
|
310
|
-
// watcher is returned, so a change made
|
|
354
|
+
// SQLite reports no changes, so the watcher compares snapshots of the watched rows: at once after each write through this file
|
|
355
|
+
// system, and every second for other connections. The first snapshot is taken before the watcher is returned, so a change made
|
|
356
|
+
// after `watch` resolves is always seen.
|
|
311
357
|
watch: async (targets, onChange, context) => {
|
|
312
358
|
if (context.abortSignal?.aborted)
|
|
313
359
|
return err(fileError('aborted', 'Operation aborted'));
|
|
@@ -326,7 +372,18 @@ export function openSqliteFileSystem(options) {
|
|
|
326
372
|
onChange(change);
|
|
327
373
|
}
|
|
328
374
|
catch { /* the host's callback cannot stop the watcher */ } };
|
|
329
|
-
const
|
|
375
|
+
const hidden = (name, target) => (target.exclude?.hidden && name.startsWith('.')) || target.exclude?.names?.includes(name);
|
|
376
|
+
/** Whether a path this file system just wrote lies under a watched target, as the snapshot would have listed it. */
|
|
377
|
+
const covered = (path) => targets.some(target => {
|
|
378
|
+
const root = normalize(target.path.startsWith('/') ? target.path : `${fs.cwd}/${target.path}`);
|
|
379
|
+
if (path === root)
|
|
380
|
+
return true;
|
|
381
|
+
if (!within(root, path))
|
|
382
|
+
return false;
|
|
383
|
+
const names = path.slice(root.length + (root === '/' ? 0 : 1)).split('/');
|
|
384
|
+
return (target.recursive || names.length === 1) && !names.some(name => hidden(name, target));
|
|
385
|
+
});
|
|
386
|
+
const check = (direct = []) => {
|
|
330
387
|
if (stopped)
|
|
331
388
|
return;
|
|
332
389
|
let next;
|
|
@@ -335,16 +392,19 @@ export function openSqliteFileSystem(options) {
|
|
|
335
392
|
}
|
|
336
393
|
catch (error) {
|
|
337
394
|
report({ error: isFileError(error) ? error : fileError('unknown', String(error)) });
|
|
338
|
-
|
|
339
|
-
clearInterval(timer);
|
|
395
|
+
stop();
|
|
340
396
|
return;
|
|
341
397
|
}
|
|
342
|
-
const
|
|
398
|
+
const written = new Set(direct.filter(covered));
|
|
399
|
+
const changed = [...new Set([...previous.keys(), ...next.keys(), ...written])].filter(path => written.has(path) || previous.get(path) !== next.get(path));
|
|
343
400
|
previous = next;
|
|
344
401
|
if (changed.length)
|
|
345
402
|
report({ paths: changed });
|
|
346
|
-
}
|
|
347
|
-
const
|
|
403
|
+
};
|
|
404
|
+
const timer = setInterval(() => check(), WATCH_INTERVAL_MS);
|
|
405
|
+
const stop = () => { stopped = true; clearInterval(timer); watchers.delete(check); };
|
|
406
|
+
watchers.add(check);
|
|
407
|
+
const watcher = { mode: 'polling', close: async () => { stop(); } };
|
|
348
408
|
return ok(watcher);
|
|
349
409
|
},
|
|
350
410
|
canonicalPath: (path, context) => run(path, context, 'read', absolute => { existing(absolute); return absolute; }),
|
|
@@ -366,6 +426,7 @@ export function openSqliteFileSystem(options) {
|
|
|
366
426
|
throw failure('permission_denied', 'EPERM', absolute, 'the workspace root cannot be removed');
|
|
367
427
|
if (found.kind === 'directory' && !options?.recursive && children(absolute).length)
|
|
368
428
|
throw failure('invalid', 'ENOTEMPTY', absolute, 'directory not empty');
|
|
429
|
+
touched.add(absolute);
|
|
369
430
|
db.run(`DELETE FROM boring_workspace_files WHERE ${subtree(absolute)}`, ...subtreeArgs(absolute));
|
|
370
431
|
}),
|
|
371
432
|
createTempDir: (prefix, context) => temporary(prefix ?? 'tmp-', '', 'directory', context),
|
|
@@ -374,8 +435,10 @@ export function openSqliteFileSystem(options) {
|
|
|
374
435
|
};
|
|
375
436
|
registerSqliteBatch(fs, {
|
|
376
437
|
connection: db,
|
|
438
|
+
maxFileBytes: () => maxFileBytes,
|
|
439
|
+
setMaxFileBytes: limit => { assertLimit(limit); maxFileBytes = limit; },
|
|
377
440
|
bytes: path => { const found = row(normalize(path)); return !found ? null : found.kind === 'directory' ? undefined : bytes(normalize(path)); },
|
|
378
|
-
write: (path, content) => { const absolute = normalize(path); makeDirectory(dirname(absolute), true); write(absolute, content); },
|
|
441
|
+
write: (path, content) => { const absolute = normalize(path); makeDirectory(dirname(absolute), true); write(absolute, content); notify(); },
|
|
379
442
|
});
|
|
380
443
|
return fs;
|
|
381
444
|
}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import ignore from 'ignore';
|
|
2
2
|
import { identifier, locator } from './publication-input.js';
|
|
3
|
+
import { HIDDEN_NAMES, isTemporary } from './file-types.js';
|
|
3
4
|
import { RevisionError } from './revision-contracts.js';
|
|
4
|
-
const hidden = new Set(['.git', '.boring', 'node_modules', 'dist', 'build', '.cache', '.next', 'coverage', '__pycache__']);
|
|
5
|
-
const temporary = /^\.boring-[0-9a-f-]{36}\.tmp$/;
|
|
6
5
|
const inside = (root, path) => path === root || path.startsWith(root === '/' ? '/' : `${root}/`);
|
|
6
|
+
/** Basename first: a name that starts with the query, then one that contains it, then a match in the directories only. */
|
|
7
|
+
const rank = (path, query) => {
|
|
8
|
+
const name = path.slice(path.lastIndexOf('/') + 1).toLowerCase();
|
|
9
|
+
return name.startsWith(query) ? '0' : name.includes(query) ? '1' : '2';
|
|
10
|
+
};
|
|
7
11
|
export function catalogPath(path, allowRoot = false) {
|
|
8
12
|
if (allowRoot && path === '')
|
|
9
13
|
return path;
|
|
@@ -47,10 +51,10 @@ export function createWorkspaceCatalog(options) {
|
|
|
47
51
|
}
|
|
48
52
|
function visible(entry, rules) {
|
|
49
53
|
const name = entry.path.slice(entry.path.lastIndexOf('/') + 1);
|
|
50
|
-
if (
|
|
54
|
+
if (isTemporary(name))
|
|
51
55
|
return false;
|
|
52
56
|
if (config.defaults !== false) {
|
|
53
|
-
if (
|
|
57
|
+
if (HIDDEN_NAMES.has(name))
|
|
54
58
|
return false;
|
|
55
59
|
let ignored = false;
|
|
56
60
|
for (const rule of rules) {
|
|
@@ -111,7 +115,15 @@ export function createWorkspaceCatalog(options) {
|
|
|
111
115
|
return entries;
|
|
112
116
|
return entries.filter(entry => entry.path.slice(0, entry.path.lastIndexOf('/') + 1) === (directory ? `${directory}/` : ''));
|
|
113
117
|
}
|
|
114
|
-
|
|
118
|
+
// One traversal at a time, each started after its search was called: a search never reuses a walk that began before it, so its
|
|
119
|
+
// result reflects every change made before the call, and no two walks overlap.
|
|
120
|
+
let walking = Promise.resolve();
|
|
121
|
+
function files(access) {
|
|
122
|
+
const run = walking.then(() => collect('', true, access));
|
|
123
|
+
walking = run.then(() => undefined, () => undefined);
|
|
124
|
+
return run.then(entries => entries.filter(entry => entry.kind === 'file'));
|
|
125
|
+
}
|
|
126
|
+
function page(entries, key, cursor, limit, order) {
|
|
115
127
|
const count = limit ?? 100;
|
|
116
128
|
if (!Number.isSafeInteger(count) || count < 1 || count > 500)
|
|
117
129
|
throw new RevisionError('invalid', 'Page limit must be between 1 and 500');
|
|
@@ -121,15 +133,15 @@ export function createWorkspaceCatalog(options) {
|
|
|
121
133
|
const value = JSON.parse(decodeURIComponent(cursor));
|
|
122
134
|
if (!Array.isArray(value) || value.length !== 2 || value[0] !== key || typeof value[1] !== 'string')
|
|
123
135
|
throw new Error();
|
|
124
|
-
after = catalogPath(value[1]);
|
|
136
|
+
after = order ? value[1] : catalogPath(value[1]);
|
|
125
137
|
}
|
|
126
138
|
catch {
|
|
127
139
|
throw new RevisionError('invalid', 'Invalid catalog cursor');
|
|
128
140
|
}
|
|
129
141
|
}
|
|
130
|
-
const sorted = entries.sort((a, b) => a.
|
|
142
|
+
const sorted = entries.map(entry => ({ entry, sort: order ? order(entry) : entry.path })).sort((a, b) => a.sort < b.sort ? -1 : a.sort > b.sort ? 1 : 0).filter(item => item.sort > after);
|
|
131
143
|
const selected = sorted.slice(0, count), last = selected.at(-1);
|
|
132
|
-
return { entries: selected, ...(last && sorted.length > count ? { cursor: encodeURIComponent(JSON.stringify([key, last.
|
|
144
|
+
return { entries: selected.map(item => item.entry), ...(last && sorted.length > count ? { cursor: encodeURIComponent(JSON.stringify([key, last.sort])) } : {}) };
|
|
133
145
|
}
|
|
134
146
|
return {
|
|
135
147
|
list: async (request, access) => {
|
|
@@ -140,7 +152,8 @@ export function createWorkspaceCatalog(options) {
|
|
|
140
152
|
if (typeof request.query !== 'string' || request.query.length > 1024)
|
|
141
153
|
throw new RevisionError('invalid', 'Invalid filename query');
|
|
142
154
|
const query = request.query.toLowerCase();
|
|
143
|
-
|
|
155
|
+
const matches = (await files(access)).filter(entry => entry.path.toLowerCase().includes(query));
|
|
156
|
+
return page(matches, `search:${query}`, request.cursor, request.limit, entry => rank(entry.path, query) + entry.path);
|
|
144
157
|
},
|
|
145
158
|
history: async (path, access) => { check(access); identifier(access.scopeId); return options.saves(catalogPath(path)); },
|
|
146
159
|
};
|
package/dist/workspace.d.ts
CHANGED
|
@@ -1,16 +1,33 @@
|
|
|
1
1
|
import type { FileSystem } from '@earendil-works/pi-durable/env';
|
|
2
2
|
import type { PublicationLookupResult, PublicationResult, ReadResult, ResourceAccess, ResourceProvider } from './contracts.js';
|
|
3
3
|
import type { WorkspaceCatalogOptions } from './workspace-catalog.js';
|
|
4
|
-
import type { WorkspaceCatalog } from './revision-contracts.js';
|
|
4
|
+
import type { ChangeFeedRequest, WorkspaceCatalog, WorkspaceChangeEvent } from './revision-contracts.js';
|
|
5
|
+
export type { ChangeFeedRequest, WorkspaceChangeEvent } from './revision-contracts.js';
|
|
5
6
|
export type { WorkspaceCatalogOptions } from './workspace-catalog.js';
|
|
6
|
-
import type { WorkspaceJournal } from './journal.js';
|
|
7
|
+
import type { HistorySource, WorkspaceJournal } from './journal.js';
|
|
8
|
+
export { isTemporary } from './file-types.js';
|
|
7
9
|
import type { WorkspaceKey } from './workspace-identity.js';
|
|
8
10
|
export type { WorkspaceKey } from './workspace-identity.js';
|
|
9
11
|
export interface WorkspaceChange {
|
|
10
12
|
readonly path: string;
|
|
11
|
-
/** `null` when the file is absent. */
|
|
13
|
+
/** `null` when the file is absent (or, from a watcher, when the path is a directory). */
|
|
12
14
|
readonly revision: string | null;
|
|
13
|
-
|
|
15
|
+
/** `publish`: a conditional write. `agent`: `record` (a Pi `write` or `edit`). `poll`: found by hashing. `watch`: the file system reported it. */
|
|
16
|
+
readonly source: 'publish' | 'agent' | 'poll' | 'watch';
|
|
17
|
+
}
|
|
18
|
+
/** What `record` is told about an agent write (ruling 2: observed, never receipted). */
|
|
19
|
+
export interface AgentWrite {
|
|
20
|
+
readonly path: string;
|
|
21
|
+
/** The file before the write, when it existed: its revision and the bytes of that revision (kept in history as the replaced version). */
|
|
22
|
+
readonly before?: {
|
|
23
|
+
readonly revision: string;
|
|
24
|
+
readonly bytes: Uint8Array;
|
|
25
|
+
} | null | undefined;
|
|
26
|
+
/** The revision the write produced, as the caller observed it right after. */
|
|
27
|
+
readonly after: string;
|
|
28
|
+
readonly source: 'agent';
|
|
29
|
+
/** The Pi conversation that wrote it, carried on the change event. */
|
|
30
|
+
readonly conversationId?: string | undefined;
|
|
14
31
|
}
|
|
15
32
|
/** Runs one piece of work at a time, in order. A failed piece does not stop the ones behind it. */
|
|
16
33
|
export interface WorkspaceQueue {
|
|
@@ -27,29 +44,70 @@ export interface WorkspaceResourceProvider extends ResourceProvider {
|
|
|
27
44
|
};
|
|
28
45
|
/** The workspace's mutation queue. Native tool calls that must not interleave with a conditional write run through it. */
|
|
29
46
|
readonly queue: WorkspaceQueue;
|
|
47
|
+
/**
|
|
48
|
+
* Record a write the agent already made through the native tools (Pi `write`/`edit`, called by the file guard right after the
|
|
49
|
+
* tool succeeded): the file's history gains the written revision (source `agent`, and the replaced one when `before` carries its
|
|
50
|
+
* bytes) and the change feed an `agent` event. It writes history rows only: no publish, no receipt, no operation or intent rows,
|
|
51
|
+
* so agent writes are observed, not published. It does not enter `queue` (the guard already runs inside it), so it is safe to call from queued work.
|
|
52
|
+
* When the bytes there no longer have the `after` revision (a shell wrote in between), only the change event is recorded.
|
|
53
|
+
*/
|
|
54
|
+
readonly record: (write: AgentWrite, access: ResourceAccess) => Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* The change feed: every write this provider makes or records, every change `poll` finds and, while someone is listening, every
|
|
57
|
+
* change the file system's `watch` reports (the SQLite file system reports its own connection's writes at once). Ordered by
|
|
58
|
+
* `seq` and replayable from `since` within a bounded window; a gap yields a `resnapshot` event. The iterator ends when
|
|
59
|
+
* `request.signal` or `access.signal` aborts, or the provider closes. Events are invalidation hints: read the bytes again.
|
|
60
|
+
*
|
|
61
|
+
* After the last listener leaves, the watcher stays open for `watchLingerMs` and then closes. Changes made while no watcher was
|
|
62
|
+
* open are unknown, so the first listener afterwards makes the feed append a `resnapshot`: a client that reconnects after a
|
|
63
|
+
* longer absence reloads everything once, and one that stays connected is never affected.
|
|
64
|
+
*/
|
|
65
|
+
readonly changes: (request: ChangeFeedRequest, access: ResourceAccess) => AsyncIterable<WorkspaceChangeEvent>;
|
|
66
|
+
/** The seq of the newest change event (the cursor of "now" for `changes({ since })`). */
|
|
67
|
+
readonly changeHead: () => number;
|
|
68
|
+
/** Names this provider's lifetime; a cursor from another one (`ChangeFeedRequest.incarnation`) is answered with a `resnapshot`. */
|
|
69
|
+
readonly changeIncarnation: string;
|
|
70
|
+
/** Ends every change stream, closes the file watcher and drops the change listeners. Idempotent; the provider still reads and publishes. */
|
|
71
|
+
readonly close: () => void;
|
|
72
|
+
/** Aborts when the provider closes, so a change stream nobody is reading still ends and releases its lease. */
|
|
73
|
+
readonly lifetime: AbortSignal;
|
|
30
74
|
/** Revisions still retained for a file, newest first. */
|
|
31
75
|
readonly history: (path: string) => string[];
|
|
32
76
|
/** The same revisions with the time each was saved (milliseconds since the epoch; 0 when unknown), newest first. */
|
|
33
77
|
readonly saves: (path: string) => {
|
|
34
78
|
readonly revision: string;
|
|
35
79
|
readonly savedAt: number;
|
|
80
|
+
readonly source: HistorySource;
|
|
36
81
|
}[];
|
|
37
82
|
/** Read the file as it is now and retain that revision in its history (what `present` shows). Serialised with writes. */
|
|
38
83
|
readonly keep: (path: string, access: ResourceAccess) => Promise<ReadResult>;
|
|
39
84
|
/** Provider writes and polled detections. The listener must not throw. */
|
|
40
85
|
readonly onChange: (listener: (change: WorkspaceChange) => void) => () => void;
|
|
41
|
-
/**
|
|
42
|
-
|
|
86
|
+
/**
|
|
87
|
+
* Hash each path now and report those that differ from the last revision this provider wrote or polled. Finds shell writes.
|
|
88
|
+
* Without `paths`, every file of the workspace (except `.git` and `node_modules`): the first such call takes the baseline and
|
|
89
|
+
* reports nothing; later calls also report new and removed files. A host whose file system cannot `watch` calls it at the end of
|
|
90
|
+
* each agent turn (after `bash`), so the change feed still sees shell writes.
|
|
91
|
+
*/
|
|
92
|
+
readonly poll: (paths?: readonly string[], signal?: AbortSignal) => Promise<WorkspaceChange[]>;
|
|
43
93
|
}
|
|
44
94
|
export interface WorkspaceProviderOptions {
|
|
45
95
|
readonly identity: WorkspaceKey;
|
|
46
96
|
readonly fs: FileSystem;
|
|
47
97
|
readonly journal: WorkspaceJournal;
|
|
48
98
|
readonly catalog?: WorkspaceCatalogOptions;
|
|
99
|
+
/** How many change events the feed retains for replay (default 1024). An older cursor gets a `resnapshot`. */
|
|
100
|
+
readonly changeBuffer?: number;
|
|
101
|
+
/** How long the watcher stays open after the last change-feed listener leaves, in milliseconds (default 30 000). */
|
|
102
|
+
readonly watchLingerMs?: number;
|
|
103
|
+
/**
|
|
104
|
+
* The largest file this workspace may store, in bytes. Over a SQLite file system (the SQLite and Durable Object providers) it sets
|
|
105
|
+
* that file system's one limit, so native writes and publications agree; default: the file system's own limit (`SQLITE_MAX_FILE_BYTES`
|
|
106
|
+
* unless opened otherwise). Over other file systems it applies to publications only, with no default.
|
|
107
|
+
*/
|
|
108
|
+
readonly maxFileBytes?: number;
|
|
49
109
|
}
|
|
50
110
|
/** The Git blob id of `bytes` (`sha1("blob <length>\0" + bytes)`), computed without Git. */
|
|
51
111
|
export declare function blobRevision(bytes: Uint8Array): Promise<string>;
|
|
52
|
-
/** Whether a path is one of the provider's temporary files (a lister hides them). */
|
|
53
|
-
export declare const isTemporary: (path: string) => boolean;
|
|
54
112
|
export declare function createWorkspaceProvider(options: WorkspaceProviderOptions): WorkspaceResourceProvider;
|
|
55
113
|
//# sourceMappingURL=workspace.d.ts.map
|