@hachej/boring-files 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/contracts.d.ts +8 -0
- package/dist/file-limits.d.ts +10 -0
- package/dist/file-limits.js +16 -0
- package/dist/file-types.d.ts +30 -0
- package/dist/file-types.js +61 -0
- package/dist/journal.js +3 -0
- package/dist/publication-input.js +6 -0
- package/dist/sqlite-batch.d.ts +3 -0
- package/dist/sqlite-filesystem.d.ts +2 -0
- package/dist/sqlite-filesystem.js +22 -1
- package/dist/workspace-catalog.js +22 -9
- package/dist/workspace.d.ts +7 -2
- package/dist/workspace.js +18 -11
- package/package.json +8 -4
package/dist/contracts.d.ts
CHANGED
|
@@ -132,6 +132,14 @@ export type PublicationRefusal = {
|
|
|
132
132
|
} | {
|
|
133
133
|
readonly kind: 'unavailable';
|
|
134
134
|
readonly reason: string;
|
|
135
|
+
}
|
|
136
|
+
/** A change above the provider's byte limit: permanent for these bytes, nothing was written. */
|
|
137
|
+
| {
|
|
138
|
+
readonly kind: 'file_too_large';
|
|
139
|
+
readonly path: string;
|
|
140
|
+
readonly size: number;
|
|
141
|
+
readonly limit: number;
|
|
142
|
+
readonly reason: string;
|
|
135
143
|
};
|
|
136
144
|
export type PublicationOutcome = {
|
|
137
145
|
readonly kind: 'committed';
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { FileError } from '@earendil-works/pi-durable/env';
|
|
2
|
+
export declare const SQLITE_MAX_FILE_BYTES = 1900000;
|
|
3
|
+
/** A limit is a non-negative safe integer of bytes. */
|
|
4
|
+
export declare function assertLimit(value: unknown): asserts value is number;
|
|
5
|
+
/**
|
|
6
|
+
* A write above the limit. Pi's `FileErrorCode` has no size code and Pi is not patched, so this is an `invalid` error whose message
|
|
7
|
+
* starts with the POSIX `EFBIG` name, like the other errors of the file system.
|
|
8
|
+
*/
|
|
9
|
+
export declare function fileTooLarge(path: string, size: number, limit: number): FileError;
|
|
10
|
+
//# sourceMappingURL=file-limits.d.ts.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// The largest file a SQLite-backed workspace stores by default: a Durable Object's SQL storage caps one row (and so one file's bytes)
|
|
2
|
+
// at 2 MB, so the default leaves headroom for the row's other columns. Disk and remote providers have no default limit.
|
|
3
|
+
import { FileError } from '@earendil-works/pi-durable/env';
|
|
4
|
+
export const SQLITE_MAX_FILE_BYTES = 1_900_000;
|
|
5
|
+
/** A limit is a non-negative safe integer of bytes. */
|
|
6
|
+
export function assertLimit(value) {
|
|
7
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0)
|
|
8
|
+
throw new TypeError('maxFileBytes must be a non-negative safe integer');
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* A write above the limit. Pi's `FileErrorCode` has no size code and Pi is not patched, so this is an `invalid` error whose message
|
|
12
|
+
* starts with the POSIX `EFBIG` name, like the other errors of the file system.
|
|
13
|
+
*/
|
|
14
|
+
export function fileTooLarge(path, size, limit) {
|
|
15
|
+
return new FileError('invalid', `EFBIG: file_too_large: ${size} bytes exceeds the ${limit} byte limit, '${path}'`, path);
|
|
16
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one place boring-files says what a file is: its extension, its media type (by name and by leading bytes), the viewer that
|
|
3
|
+
* opens it, the provider's temporary and hidden names, the workspace path a user or agent typed, and the name of a new upload.
|
|
4
|
+
* Callers import these; none keeps a copy. Pure: no Node, no DOM.
|
|
5
|
+
*/
|
|
6
|
+
/** The lower-case extension of the last path segment, or ''. */
|
|
7
|
+
export declare const extensionOf: (path: string) => string;
|
|
8
|
+
/** The IANA media type by extension. Anything not listed is `application/octet-stream` (see `mediaTypeOf`). Null-prototype: `constructor` or `__proto__` is not an entry. */
|
|
9
|
+
export declare const MEDIA_TYPES: Readonly<Record<string, string>>;
|
|
10
|
+
/** The IANA media type of a path, `application/octet-stream` when unknown. */
|
|
11
|
+
export declare const mediaTypeOf: (path: string) => string;
|
|
12
|
+
/** The media type a file's leading bytes name (a PDF, PNG, JPEG, GIF or WebP signature), or undefined. */
|
|
13
|
+
export declare function sniffMediaType(head: Uint8Array): string | undefined;
|
|
14
|
+
/** Which viewer opens a file. Anything not listed is shown as text when it decodes. */
|
|
15
|
+
export type ViewerKind = 'markdown' | 'html' | 'canvas' | 'image' | 'pdf' | 'text';
|
|
16
|
+
/** The viewer kind of a path: `markdown`, `html`, `canvas`, `image`, `pdf` or `text`. */
|
|
17
|
+
export declare const viewerKindOf: (path: string) => ViewerKind;
|
|
18
|
+
/** Binary viewers never decode a file as text, whatever its bytes look like (an SVG is text). */
|
|
19
|
+
export declare const isBinaryKind: (path: string) => boolean;
|
|
20
|
+
/** The basename of a new temporary file, for a provider's `uuid`. */
|
|
21
|
+
export declare const temporaryFileName: (uuid: string) => string;
|
|
22
|
+
/** Whether a path's last segment is one of the provider's temporary files (a lister hides them). */
|
|
23
|
+
export declare const isTemporary: (path: string) => boolean;
|
|
24
|
+
/** Names a presentation listing hides by default (dependency, build and internal folders, and the provider's own). Not an access policy. */
|
|
25
|
+
export declare const HIDDEN_NAMES: ReadonlySet<string>;
|
|
26
|
+
/** A path as a person or agent typed it, as a workspace-relative resource path: leading `./` segments removed. A leading `/` stays: it is not a workspace path, and the provider refuses it rather than reading it as a relative one. */
|
|
27
|
+
export declare const workspacePath: (path: string) => string;
|
|
28
|
+
/** A new upload's workspace path: `uploads/<stem>-<8 hex>.<ext>`, so a second `image.png` (every pasted screenshot has that name) is a new file beside the first. Whitespace becomes `_`, because a chat mention ends at a space. */
|
|
29
|
+
export declare function uploadPath(name: string, randomSuffix: string): string;
|
|
30
|
+
//# sourceMappingURL=file-types.d.ts.map
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one place boring-files says what a file is: its extension, its media type (by name and by leading bytes), the viewer that
|
|
3
|
+
* opens it, the provider's temporary and hidden names, the workspace path a user or agent typed, and the name of a new upload.
|
|
4
|
+
* Callers import these; none keeps a copy. Pure: no Node, no DOM.
|
|
5
|
+
*/
|
|
6
|
+
/** The lower-case extension of the last path segment, or ''. */
|
|
7
|
+
export const extensionOf = (path) => { const dot = path.lastIndexOf('.'); return dot > path.lastIndexOf('/') ? path.slice(dot + 1).toLowerCase() : ''; };
|
|
8
|
+
/** The IANA media type by extension. Anything not listed is `application/octet-stream` (see `mediaTypeOf`). Null-prototype: `constructor` or `__proto__` is not an entry. */
|
|
9
|
+
export const MEDIA_TYPES = Object.assign(Object.create(null), {
|
|
10
|
+
md: 'text/markdown', markdown: 'text/markdown', html: 'text/html', htm: 'text/html', json: 'application/json', svg: 'image/svg+xml',
|
|
11
|
+
txt: 'text/plain', csv: 'text/csv', js: 'text/javascript', mjs: 'text/javascript', css: 'text/css', tldraw: 'application/vnd.tldraw+json',
|
|
12
|
+
png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', bmp: 'image/bmp', tif: 'image/tiff', tiff: 'image/tiff',
|
|
13
|
+
pdf: 'application/pdf', doc: 'application/msword', docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
14
|
+
xls: 'application/vnd.ms-excel', xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
15
|
+
ppt: 'application/vnd.ms-powerpoint', pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
16
|
+
odt: 'application/vnd.oasis.opendocument.text', ods: 'application/vnd.oasis.opendocument.spreadsheet', odp: 'application/vnd.oasis.opendocument.presentation',
|
|
17
|
+
rtf: 'application/rtf', epub: 'application/epub+zip',
|
|
18
|
+
});
|
|
19
|
+
/** The IANA media type of a path, `application/octet-stream` when unknown. */
|
|
20
|
+
export const mediaTypeOf = (path) => MEDIA_TYPES[extensionOf(path)] ?? 'application/octet-stream';
|
|
21
|
+
/** The media type a file's leading bytes name (a PDF, PNG, JPEG, GIF or WebP signature), or undefined. */
|
|
22
|
+
export function sniffMediaType(head) {
|
|
23
|
+
const at = (signature, offset = 0) => signature.every((byte, index) => head[offset + index] === byte);
|
|
24
|
+
if (at([0x25, 0x50, 0x44, 0x46, 0x2d]))
|
|
25
|
+
return 'application/pdf';
|
|
26
|
+
if (at([0x89, 0x50, 0x4e, 0x47]))
|
|
27
|
+
return 'image/png';
|
|
28
|
+
if (at([0xff, 0xd8, 0xff]))
|
|
29
|
+
return 'image/jpeg';
|
|
30
|
+
if (at([0x47, 0x49, 0x46, 0x38]))
|
|
31
|
+
return 'image/gif';
|
|
32
|
+
if (at([0x52, 0x49, 0x46, 0x46]) && at([0x57, 0x45, 0x42, 0x50], 8))
|
|
33
|
+
return 'image/webp';
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
const VIEWER_KINDS = Object.assign(Object.create(null), {
|
|
37
|
+
md: 'markdown', markdown: 'markdown', html: 'html', htm: 'html', tldraw: 'canvas',
|
|
38
|
+
png: 'image', jpg: 'image', jpeg: 'image', gif: 'image', webp: 'image', svg: 'image', pdf: 'pdf',
|
|
39
|
+
txt: 'text', json: 'text', csv: 'text', js: 'text', mjs: 'text', css: 'text',
|
|
40
|
+
});
|
|
41
|
+
/** The viewer kind of a path: `markdown`, `html`, `canvas`, `image`, `pdf` or `text`. */
|
|
42
|
+
export const viewerKindOf = (path) => VIEWER_KINDS[extensionOf(path)] ?? 'text';
|
|
43
|
+
/** Binary viewers never decode a file as text, whatever its bytes look like (an SVG is text). */
|
|
44
|
+
export const isBinaryKind = (path) => ['image', 'pdf'].includes(viewerKindOf(path)) && extensionOf(path) !== 'svg';
|
|
45
|
+
/** The provider's temporary files are named `.boring-<uuid>.tmp` beside their target. They are removed when the provider opens and are never a resource. */
|
|
46
|
+
const temporaryPattern = /^\.boring-[0-9a-f-]{36}\.tmp$/;
|
|
47
|
+
/** The basename of a new temporary file, for a provider's `uuid`. */
|
|
48
|
+
export const temporaryFileName = (uuid) => `.boring-${uuid}.tmp`;
|
|
49
|
+
/** Whether a path's last segment is one of the provider's temporary files (a lister hides them). */
|
|
50
|
+
export const isTemporary = (path) => temporaryPattern.test(path.slice(path.lastIndexOf('/') + 1));
|
|
51
|
+
/** Names a presentation listing hides by default (dependency, build and internal folders, and the provider's own). Not an access policy. */
|
|
52
|
+
export const HIDDEN_NAMES = new Set(['.git', '.boring', 'node_modules', 'dist', 'build', '.cache', '.next', 'coverage', '__pycache__']);
|
|
53
|
+
/** A path as a person or agent typed it, as a workspace-relative resource path: leading `./` segments removed. A leading `/` stays: it is not a workspace path, and the provider refuses it rather than reading it as a relative one. */
|
|
54
|
+
export const workspacePath = (path) => path.replace(/^(\.\/)+/, '');
|
|
55
|
+
/** A new upload's workspace path: `uploads/<stem>-<8 hex>.<ext>`, so a second `image.png` (every pasted screenshot has that name) is a new file beside the first. Whitespace becomes `_`, because a chat mention ends at a space. */
|
|
56
|
+
export function uploadPath(name, randomSuffix) {
|
|
57
|
+
const dot = name.lastIndexOf('.');
|
|
58
|
+
const [stem, extension] = dot > 0 ? [name.slice(0, dot), name.slice(dot)] : [name, ''];
|
|
59
|
+
const safe = (part) => part.replace(/\s+/g, '_');
|
|
60
|
+
return `uploads/${safe(stem)}-${randomSuffix}${safe(extension)}`;
|
|
61
|
+
}
|
package/dist/journal.js
CHANGED
|
@@ -112,6 +112,9 @@ export function createWorkspaceJournal(db, options = {}) {
|
|
|
112
112
|
const row = db.get('SELECT bytes, media_type FROM boring_history WHERE scope = ? AND path = ? AND revision = ?', scope, path, revision);
|
|
113
113
|
if (!row)
|
|
114
114
|
return undefined;
|
|
115
|
+
// A zero-length BLOB can come back as no bytes (see sqlite-filesystem.ts): a saved empty file is an empty version.
|
|
116
|
+
if (row.bytes === null)
|
|
117
|
+
return { revision, bytes: new Uint8Array(), mediaType: text(row.media_type) };
|
|
115
118
|
if (!(row.bytes instanceof Uint8Array))
|
|
116
119
|
throw new Error('Invalid stored bytes');
|
|
117
120
|
return { revision, bytes: Uint8Array.from(row.bytes), mediaType: text(row.media_type) };
|
|
@@ -135,6 +135,12 @@ function publicationOutcome(value) {
|
|
|
135
135
|
case 'committed': return { kind: 'committed', receipt: publicationReceipt(input.receipt) };
|
|
136
136
|
case 'denied':
|
|
137
137
|
case 'unavailable': return { kind: input.kind, reason: reason(input.reason) };
|
|
138
|
+
case 'file_too_large': {
|
|
139
|
+
const { path, size, limit } = input;
|
|
140
|
+
if (typeof path !== 'string' || typeof size !== 'number' || !Number.isSafeInteger(size) || typeof limit !== 'number' || !Number.isSafeInteger(limit))
|
|
141
|
+
throw new TypeError('Invalid size refusal');
|
|
142
|
+
return { kind: 'file_too_large', path, size, limit, reason: reason(input.reason) };
|
|
143
|
+
}
|
|
138
144
|
case 'unknown': return { kind: 'unknown', operationId: identifier(input.operationId), reason: reason(input.reason) };
|
|
139
145
|
case 'conflict': {
|
|
140
146
|
if (!Array.isArray(input.current))
|
package/dist/sqlite-batch.d.ts
CHANGED
|
@@ -10,6 +10,9 @@ export interface SqliteBatch {
|
|
|
10
10
|
readonly bytes: (path: string) => Uint8Array | null | undefined;
|
|
11
11
|
/** Write a regular file at an absolute path, creating its missing parent directories. Throws a Pi `FileError` on failure. */
|
|
12
12
|
readonly write: (path: string, bytes: Uint8Array) => void;
|
|
13
|
+
/** The one byte limit of this file system: native writes and publications both enforce it. */
|
|
14
|
+
readonly maxFileBytes: () => number;
|
|
15
|
+
readonly setMaxFileBytes: (limit: number) => void;
|
|
13
16
|
}
|
|
14
17
|
export declare const registerSqliteBatch: (fs: object, batch: SqliteBatch) => void;
|
|
15
18
|
export declare const sqliteBatchOf: (fs: object) => SqliteBatch | undefined;
|
|
@@ -7,6 +7,8 @@ export interface SqliteFileSystemOptions {
|
|
|
7
7
|
readonly workspace: string;
|
|
8
8
|
/** The absolute working directory, created when missing (for example `/workspace`). Relative paths resolve against `cwd`. */
|
|
9
9
|
readonly cwd: string;
|
|
10
|
+
/** The largest file a write may store (default `SQLITE_MAX_FILE_BYTES`, the row limit of a Durable Object). A larger write fails with `EFBIG`. */
|
|
11
|
+
readonly maxFileBytes?: number;
|
|
10
12
|
}
|
|
11
13
|
/** A Pi `FileSystem` plus the incarnation of its rows: new when the workspace was first created in this database. */
|
|
12
14
|
export interface SqliteFileSystem extends FileSystem {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { FileError, LineScanner, err, ok } from '@earendil-works/pi-durable/env';
|
|
2
|
+
import { SQLITE_MAX_FILE_BYTES, assertLimit, fileTooLarge } from './file-limits.js';
|
|
2
3
|
import { randomUUID } from './platform.js';
|
|
3
4
|
import { registerSqliteBatch } from './sqlite-batch.js';
|
|
4
5
|
function normalize(path) {
|
|
@@ -35,6 +36,8 @@ export function openSqliteFileSystem(options) {
|
|
|
35
36
|
if (typeof options.cwd !== 'string' || !options.cwd.startsWith('/'))
|
|
36
37
|
throw new TypeError('An absolute working directory is required');
|
|
37
38
|
const now = () => Date.now();
|
|
39
|
+
let maxFileBytes = options.maxFileBytes ?? SQLITE_MAX_FILE_BYTES;
|
|
40
|
+
assertLimit(maxFileBytes);
|
|
38
41
|
function row(path) {
|
|
39
42
|
if (path === '/')
|
|
40
43
|
return { kind: 'directory', size: 0, mtime: 0 };
|
|
@@ -71,9 +74,13 @@ export function openSqliteFileSystem(options) {
|
|
|
71
74
|
/** Paths written, renamed or removed by this file system since the last comparison: reported as changed even when size and time look alike. */
|
|
72
75
|
const touched = new Set();
|
|
73
76
|
function bytes(path) {
|
|
74
|
-
|
|
77
|
+
const entry = existing(path);
|
|
78
|
+
if (entry.kind === 'directory')
|
|
75
79
|
throw failure('is_directory', 'EISDIR', path, 'illegal operation on a directory');
|
|
76
80
|
const found = db.get('SELECT bytes FROM boring_workspace_files WHERE workspace = ? AND path = ?', workspace, path);
|
|
81
|
+
// A zero-length BLOB can come back as no bytes (a NULL pointer through some engines): a file row with no bytes is an empty file.
|
|
82
|
+
if (found?.bytes === null && entry.kind === 'file')
|
|
83
|
+
return new Uint8Array();
|
|
77
84
|
if (!(found?.bytes instanceof Uint8Array))
|
|
78
85
|
throw fileError('unknown', 'Invalid stored bytes', path);
|
|
79
86
|
return Uint8Array.from(found.bytes);
|
|
@@ -85,6 +92,8 @@ export function openSqliteFileSystem(options) {
|
|
|
85
92
|
function write(path, content) {
|
|
86
93
|
if (path === '/')
|
|
87
94
|
throw failure('is_directory', 'EISDIR', path, 'illegal operation on a directory');
|
|
95
|
+
if (content.length > maxFileBytes)
|
|
96
|
+
throw fileTooLarge(path, content.length, maxFileBytes);
|
|
88
97
|
const found = row(path);
|
|
89
98
|
if (found?.kind === 'directory')
|
|
90
99
|
throw failure('is_directory', 'EISDIR', path, 'illegal operation on a directory');
|
|
@@ -268,6 +277,9 @@ export function openSqliteFileSystem(options) {
|
|
|
268
277
|
const added = encode(content);
|
|
269
278
|
if (!row(absolute))
|
|
270
279
|
return write(absolute, added);
|
|
280
|
+
const grown = existing(absolute).size + added.length;
|
|
281
|
+
if (grown > maxFileBytes)
|
|
282
|
+
throw fileTooLarge(absolute, grown, maxFileBytes);
|
|
271
283
|
const before = bytes(absolute), joined = new Uint8Array(before.length + added.length);
|
|
272
284
|
joined.set(before);
|
|
273
285
|
joined.set(added, before.length);
|
|
@@ -276,6 +288,9 @@ export function openSqliteFileSystem(options) {
|
|
|
276
288
|
truncateFile: (path, size, context) => run(path, context, 'write', absolute => {
|
|
277
289
|
if (!Number.isSafeInteger(size) || size < 0)
|
|
278
290
|
throw fileError('invalid', 'File size must be a non-negative safe integer', absolute);
|
|
291
|
+
// Checked before the buffer is allocated: a huge size is a size refusal, not an allocation failure.
|
|
292
|
+
if (size > maxFileBytes)
|
|
293
|
+
throw fileTooLarge(absolute, size, maxFileBytes);
|
|
279
294
|
const before = bytes(absolute), after = new Uint8Array(size);
|
|
280
295
|
after.set(before.subarray(0, size));
|
|
281
296
|
write(absolute, after);
|
|
@@ -288,6 +303,10 @@ export function openSqliteFileSystem(options) {
|
|
|
288
303
|
const moved = existing(from);
|
|
289
304
|
if (from === to)
|
|
290
305
|
return;
|
|
306
|
+
// A moved file keeps its bytes, so a file (or a file below a moved directory) over the limit cannot replace another file by renaming.
|
|
307
|
+
const oversized = db.get(`SELECT path, length(bytes) AS size FROM boring_workspace_files WHERE ${subtree(from)} AND kind = 'file' AND length(bytes) > ? LIMIT 1`, ...subtreeArgs(from), maxFileBytes);
|
|
308
|
+
if (oversized)
|
|
309
|
+
throw fileTooLarge(oversized.path, Number(oversized.size), maxFileBytes);
|
|
291
310
|
touched.add(from);
|
|
292
311
|
touched.add(to);
|
|
293
312
|
if (from === '/' || within(from, to))
|
|
@@ -416,6 +435,8 @@ export function openSqliteFileSystem(options) {
|
|
|
416
435
|
};
|
|
417
436
|
registerSqliteBatch(fs, {
|
|
418
437
|
connection: db,
|
|
438
|
+
maxFileBytes: () => maxFileBytes,
|
|
439
|
+
setMaxFileBytes: limit => { assertLimit(limit); maxFileBytes = limit; },
|
|
419
440
|
bytes: path => { const found = row(normalize(path)); return !found ? null : found.kind === 'directory' ? undefined : bytes(normalize(path)); },
|
|
420
441
|
write: (path, content) => { const absolute = normalize(path); makeDirectory(dirname(absolute), true); write(absolute, content); notify(); },
|
|
421
442
|
});
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import ignore from 'ignore';
|
|
2
2
|
import { identifier, locator } from './publication-input.js';
|
|
3
|
+
import { HIDDEN_NAMES, isTemporary } from './file-types.js';
|
|
3
4
|
import { RevisionError } from './revision-contracts.js';
|
|
4
|
-
const hidden = new Set(['.git', '.boring', 'node_modules', 'dist', 'build', '.cache', '.next', 'coverage', '__pycache__']);
|
|
5
|
-
const temporary = /^\.boring-[0-9a-f-]{36}\.tmp$/;
|
|
6
5
|
const inside = (root, path) => path === root || path.startsWith(root === '/' ? '/' : `${root}/`);
|
|
6
|
+
/** Basename first: a name that starts with the query, then one that contains it, then a match in the directories only. */
|
|
7
|
+
const rank = (path, query) => {
|
|
8
|
+
const name = path.slice(path.lastIndexOf('/') + 1).toLowerCase();
|
|
9
|
+
return name.startsWith(query) ? '0' : name.includes(query) ? '1' : '2';
|
|
10
|
+
};
|
|
7
11
|
export function catalogPath(path, allowRoot = false) {
|
|
8
12
|
if (allowRoot && path === '')
|
|
9
13
|
return path;
|
|
@@ -47,10 +51,10 @@ export function createWorkspaceCatalog(options) {
|
|
|
47
51
|
}
|
|
48
52
|
function visible(entry, rules) {
|
|
49
53
|
const name = entry.path.slice(entry.path.lastIndexOf('/') + 1);
|
|
50
|
-
if (
|
|
54
|
+
if (isTemporary(name))
|
|
51
55
|
return false;
|
|
52
56
|
if (config.defaults !== false) {
|
|
53
|
-
if (
|
|
57
|
+
if (HIDDEN_NAMES.has(name))
|
|
54
58
|
return false;
|
|
55
59
|
let ignored = false;
|
|
56
60
|
for (const rule of rules) {
|
|
@@ -111,7 +115,15 @@ export function createWorkspaceCatalog(options) {
|
|
|
111
115
|
return entries;
|
|
112
116
|
return entries.filter(entry => entry.path.slice(0, entry.path.lastIndexOf('/') + 1) === (directory ? `${directory}/` : ''));
|
|
113
117
|
}
|
|
114
|
-
|
|
118
|
+
// One traversal at a time, each started after its search was called: a search never reuses a walk that began before it, so its
|
|
119
|
+
// result reflects every change made before the call, and no two walks overlap.
|
|
120
|
+
let walking = Promise.resolve();
|
|
121
|
+
function files(access) {
|
|
122
|
+
const run = walking.then(() => collect('', true, access));
|
|
123
|
+
walking = run.then(() => undefined, () => undefined);
|
|
124
|
+
return run.then(entries => entries.filter(entry => entry.kind === 'file'));
|
|
125
|
+
}
|
|
126
|
+
function page(entries, key, cursor, limit, order) {
|
|
115
127
|
const count = limit ?? 100;
|
|
116
128
|
if (!Number.isSafeInteger(count) || count < 1 || count > 500)
|
|
117
129
|
throw new RevisionError('invalid', 'Page limit must be between 1 and 500');
|
|
@@ -121,15 +133,15 @@ export function createWorkspaceCatalog(options) {
|
|
|
121
133
|
const value = JSON.parse(decodeURIComponent(cursor));
|
|
122
134
|
if (!Array.isArray(value) || value.length !== 2 || value[0] !== key || typeof value[1] !== 'string')
|
|
123
135
|
throw new Error();
|
|
124
|
-
after = catalogPath(value[1]);
|
|
136
|
+
after = order ? value[1] : catalogPath(value[1]);
|
|
125
137
|
}
|
|
126
138
|
catch {
|
|
127
139
|
throw new RevisionError('invalid', 'Invalid catalog cursor');
|
|
128
140
|
}
|
|
129
141
|
}
|
|
130
|
-
const sorted = entries.sort((a, b) => a.
|
|
142
|
+
const sorted = entries.map(entry => ({ entry, sort: order ? order(entry) : entry.path })).sort((a, b) => a.sort < b.sort ? -1 : a.sort > b.sort ? 1 : 0).filter(item => item.sort > after);
|
|
131
143
|
const selected = sorted.slice(0, count), last = selected.at(-1);
|
|
132
|
-
return { entries: selected, ...(last && sorted.length > count ? { cursor: encodeURIComponent(JSON.stringify([key, last.
|
|
144
|
+
return { entries: selected.map(item => item.entry), ...(last && sorted.length > count ? { cursor: encodeURIComponent(JSON.stringify([key, last.sort])) } : {}) };
|
|
133
145
|
}
|
|
134
146
|
return {
|
|
135
147
|
list: async (request, access) => {
|
|
@@ -140,7 +152,8 @@ export function createWorkspaceCatalog(options) {
|
|
|
140
152
|
if (typeof request.query !== 'string' || request.query.length > 1024)
|
|
141
153
|
throw new RevisionError('invalid', 'Invalid filename query');
|
|
142
154
|
const query = request.query.toLowerCase();
|
|
143
|
-
|
|
155
|
+
const matches = (await files(access)).filter(entry => entry.path.toLowerCase().includes(query));
|
|
156
|
+
return page(matches, `search:${query}`, request.cursor, request.limit, entry => rank(entry.path, query) + entry.path);
|
|
144
157
|
},
|
|
145
158
|
history: async (path, access) => { check(access); identifier(access.scopeId); return options.saves(catalogPath(path)); },
|
|
146
159
|
};
|
package/dist/workspace.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type { ChangeFeedRequest, WorkspaceCatalog, WorkspaceChangeEvent } from '
|
|
|
5
5
|
export type { ChangeFeedRequest, WorkspaceChangeEvent } from './revision-contracts.js';
|
|
6
6
|
export type { WorkspaceCatalogOptions } from './workspace-catalog.js';
|
|
7
7
|
import type { HistorySource, WorkspaceJournal } from './journal.js';
|
|
8
|
+
export { isTemporary } from './file-types.js';
|
|
8
9
|
import type { WorkspaceKey } from './workspace-identity.js';
|
|
9
10
|
export type { WorkspaceKey } from './workspace-identity.js';
|
|
10
11
|
export interface WorkspaceChange {
|
|
@@ -99,10 +100,14 @@ export interface WorkspaceProviderOptions {
|
|
|
99
100
|
readonly changeBuffer?: number;
|
|
100
101
|
/** How long the watcher stays open after the last change-feed listener leaves, in milliseconds (default 30 000). */
|
|
101
102
|
readonly watchLingerMs?: number;
|
|
103
|
+
/**
|
|
104
|
+
* The largest file this workspace may store, in bytes. Over a SQLite file system (the SQLite and Durable Object providers) it sets
|
|
105
|
+
* that file system's one limit, so native writes and publications agree; default: the file system's own limit (`SQLITE_MAX_FILE_BYTES`
|
|
106
|
+
* unless opened otherwise). Over other file systems it applies to publications only, with no default.
|
|
107
|
+
*/
|
|
108
|
+
readonly maxFileBytes?: number;
|
|
102
109
|
}
|
|
103
110
|
/** The Git blob id of `bytes` (`sha1("blob <length>\0" + bytes)`), computed without Git. */
|
|
104
111
|
export declare function blobRevision(bytes: Uint8Array): Promise<string>;
|
|
105
|
-
/** Whether a path is one of the provider's temporary files (a lister hides them). */
|
|
106
|
-
export declare const isTemporary: (path: string) => boolean;
|
|
107
112
|
export declare function createWorkspaceProvider(options: WorkspaceProviderOptions): WorkspaceResourceProvider;
|
|
108
113
|
//# sourceMappingURL=workspace.d.ts.map
|
package/dist/workspace.js
CHANGED
|
@@ -2,7 +2,10 @@ import { createWorkspaceCatalog } from './workspace-catalog.js';
|
|
|
2
2
|
import { committedChanges } from './journal.js';
|
|
3
3
|
import { randomUUID, sha1 } from './platform.js';
|
|
4
4
|
import { accessSnapshot, identifier, locator, publicationDigest, publicationSnapshot } from './publication-input.js';
|
|
5
|
+
import { assertLimit, fileTooLarge } from './file-limits.js';
|
|
5
6
|
import { journalConnectionOf, sqliteBatchOf } from './sqlite-batch.js';
|
|
7
|
+
import { isTemporary, mediaTypeOf, temporaryFileName } from './file-types.js';
|
|
8
|
+
export { isTemporary } from './file-types.js';
|
|
6
9
|
const sameBytes = (left, right) => left.length === right.length && left.every((byte, index) => byte === right[index]);
|
|
7
10
|
const queues = new WeakMap();
|
|
8
11
|
/** One queue per file system object: two providers over the same `fs` share it. */
|
|
@@ -30,16 +33,6 @@ export async function blobRevision(bytes) {
|
|
|
30
33
|
whole.set(bytes, header.length);
|
|
31
34
|
return hex(await sha1(whole));
|
|
32
35
|
}
|
|
33
|
-
const mediaTypes = {
|
|
34
|
-
md: 'text/markdown', markdown: 'text/markdown', html: 'text/html', htm: 'text/html', json: 'application/json', svg: 'image/svg+xml', txt: 'text/plain', tldraw: 'application/vnd.tldraw+json',
|
|
35
|
-
};
|
|
36
|
-
function mediaTypeOf(path) {
|
|
37
|
-
return mediaTypes[path.slice(path.lastIndexOf('.') + 1).toLowerCase()] ?? 'application/octet-stream';
|
|
38
|
-
}
|
|
39
|
-
/** The provider's own temporary files. They are removed when the provider opens and are never a resource. */
|
|
40
|
-
const temporaryPattern = /^\.boring-[0-9a-f-]{36}\.tmp$/;
|
|
41
|
-
/** Whether a path is one of the provider's temporary files (a lister hides them). */
|
|
42
|
-
export const isTemporary = (path) => temporaryPattern.test(path.slice(path.lastIndexOf('/') + 1));
|
|
43
36
|
export function createWorkspaceProvider(options) {
|
|
44
37
|
const { fs, journal } = options;
|
|
45
38
|
const identity = options.identity;
|
|
@@ -58,6 +51,11 @@ export function createWorkspaceProvider(options) {
|
|
|
58
51
|
const queue = queueFor(fs);
|
|
59
52
|
/** Present when `fs` is the SQLite file system (`@hachej/boring-files/sqlite-filesystem`): multi-file batches then commit in one transaction. */
|
|
60
53
|
const batch = sqliteBatchOf(fs);
|
|
54
|
+
if (options.maxFileBytes !== undefined) {
|
|
55
|
+
assertLimit(options.maxFileBytes);
|
|
56
|
+
batch?.setMaxFileBytes(options.maxFileBytes);
|
|
57
|
+
}
|
|
58
|
+
const limitOf = () => batch ? batch.maxFileBytes() : options.maxFileBytes;
|
|
61
59
|
const historyScope = JSON.stringify([key.providerId, key.instanceId, key.incarnation, key.viewId]);
|
|
62
60
|
const listeners = new Set();
|
|
63
61
|
const known = new Map();
|
|
@@ -252,6 +250,15 @@ export function createWorkspaceProvider(options) {
|
|
|
252
250
|
}
|
|
253
251
|
if (journal.intent(operation))
|
|
254
252
|
return { kind: 'unknown', operationId: request.operationId, reason: 'An earlier attempt of this operation began and has no recorded outcome' };
|
|
253
|
+
// After the receipt and intent checks, so a replay of a committed operation keeps its result under a lower limit.
|
|
254
|
+
const limit = limitOf();
|
|
255
|
+
if (limit !== undefined) {
|
|
256
|
+
const oversized = changes.find(change => change.bytes.length > limit);
|
|
257
|
+
if (oversized) {
|
|
258
|
+
const size = oversized.bytes.length, path = oversized.target.resource.path;
|
|
259
|
+
return { kind: 'file_too_large', path, size, limit, reason: fileTooLarge(path, size, limit).message };
|
|
260
|
+
}
|
|
261
|
+
}
|
|
255
262
|
journal.begin(operation, digest);
|
|
256
263
|
let applying = false;
|
|
257
264
|
const refuse = (reason) => { journal.cancel(operation); return { kind: 'unavailable', reason }; };
|
|
@@ -345,7 +352,7 @@ export function createWorkspaceProvider(options) {
|
|
|
345
352
|
if (!unchanged) {
|
|
346
353
|
applying = true;
|
|
347
354
|
const target = absolute(path);
|
|
348
|
-
const temporary = `${target.slice(0, target.lastIndexOf('/'))}
|
|
355
|
+
const temporary = `${target.slice(0, target.lastIndexOf('/'))}/${temporaryFileName(randomUUID())}`;
|
|
349
356
|
const ctx = fsContext(access.signal);
|
|
350
357
|
const failure = async (reason) => { await fs.remove(temporary, { force: true }, fsContext()); return refuse(reason); };
|
|
351
358
|
if (change.kind === 'create') {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hachej/boring-files",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"private": false,
|
|
6
6
|
"type": "module",
|
|
@@ -81,12 +81,16 @@
|
|
|
81
81
|
"./revision-handler": {
|
|
82
82
|
"types": "./dist/revision-handler.d.ts",
|
|
83
83
|
"import": "./dist/revision-handler.js"
|
|
84
|
+
},
|
|
85
|
+
"./file-types": {
|
|
86
|
+
"types": "./dist/file-types.d.ts",
|
|
87
|
+
"import": "./dist/file-types.js"
|
|
84
88
|
}
|
|
85
89
|
},
|
|
86
90
|
"peerDependencies": {
|
|
87
|
-
"isomorphic-git": "1.42.6",
|
|
88
|
-
"@earendil-works/pi-durable": "1.1.0",
|
|
89
|
-
"@earendil-works/chord": "1.1.0"
|
|
91
|
+
"isomorphic-git": "^1.42.6",
|
|
92
|
+
"@earendil-works/pi-durable": "~1.1.0",
|
|
93
|
+
"@earendil-works/chord": "~1.1.0"
|
|
90
94
|
},
|
|
91
95
|
"peerDependenciesMeta": {
|
|
92
96
|
"isomorphic-git": {
|