@statewalker/webrun-files-sqlite 0.10.3 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -55
- package/dist/blocks/sqlite-files-api.d.ts +3 -3
- package/dist/blocks/sqlite-files-api.d.ts.map +1 -1
- package/dist/bytes.d.ts +1 -1
- package/dist/bytes.d.ts.map +1 -1
- package/dist/cjs/index.cjs +81 -112
- package/dist/esm/index.js +82 -110
- package/dist/index.d.ts +0 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/run-statements.d.ts +15 -0
- package/dist/run-statements.d.ts.map +1 -0
- package/dist/sqlar-files-api.d.ts +2 -2
- package/dist/sqlar-files-api.d.ts.map +1 -1
- package/package.json +7 -3
- package/src/blocks/sqlite-files-api.ts +40 -31
- package/src/bytes.ts +1 -3
- package/src/index.ts +0 -9
- package/src/run-statements.ts +24 -0
- package/src/sqlar-files-api.ts +20 -25
- package/dist/drivers.d.ts +0 -73
- package/dist/drivers.d.ts.map +0 -1
- package/dist/sql.types.d.ts +0 -25
- package/dist/sql.types.d.ts.map +0 -1
- package/src/drivers.ts +0 -133
- package/src/sql.types.ts +0 -25
package/README.md
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
## What it is
|
|
4
4
|
|
|
5
5
|
Two implementations of `FilesApi` from `@statewalker/webrun-files` that store files in a SQLite
|
|
6
|
-
database,
|
|
7
|
-
|
|
6
|
+
database, through `@statewalker/db-api`'s `Db`: any `@statewalker/db-sqlite-*` driver — `node:sqlite`,
|
|
7
|
+
the browser (sqlite-wasm), or a Durable Object's storage:
|
|
8
8
|
|
|
9
9
|
- `SqliteFilesApi` streams content as fixed-size blocks (1 MiB by default, optionally deflated per
|
|
10
10
|
block), shares identical content between paths, and makes every metadata change one transaction.
|
|
@@ -14,9 +14,9 @@ Object storage and Cloudflare D1:
|
|
|
14
14
|
|
|
15
15
|
## Why it exists
|
|
16
16
|
|
|
17
|
-
Durable Objects
|
|
17
|
+
Durable Objects offer SQLite and nothing like a file system, and a single SQLite file is a
|
|
18
18
|
convenient, transactional store elsewhere too. Storing a file as one blob breaks there: Durable
|
|
19
|
-
Objects
|
|
19
|
+
Objects cap a row at 2 MB, and a blob is read and written whole. `SqliteFilesApi` cuts
|
|
20
20
|
content into blocks that stay under the cap and can be streamed and range-read. `SqlarFilesApi`
|
|
21
21
|
exists for the opposite need: an archive other SQLAR tools can open, in runtimes without a row
|
|
22
22
|
limit.
|
|
@@ -25,7 +25,7 @@ limit.
|
|
|
25
25
|
| --- | --- | --- |
|
|
26
26
|
| Storage | paths -> contents -> blocks of at most 1.5 MiB | one row per file, SQLite Archive format |
|
|
27
27
|
| File size | unbounded; streamed block by block | one blob, read and written whole |
|
|
28
|
-
| Durable Objects
|
|
28
|
+
| Durable Objects | yes, every row stays under the 2 MB limit | only files whose stored row is under 2 MB |
|
|
29
29
|
| Readable by `sqlite3 -A` | no | yes |
|
|
30
30
|
| Identical content | stored once, shared by every path | stored per path |
|
|
31
31
|
|
|
@@ -34,23 +34,28 @@ Use `SqliteFilesApi` by default.
|
|
|
34
34
|
## How to use
|
|
35
35
|
|
|
36
36
|
```bash
|
|
37
|
-
pnpm add @statewalker/webrun-files-sqlite @statewalker/webrun-files
|
|
37
|
+
pnpm add @statewalker/webrun-files-sqlite @statewalker/webrun-files @statewalker/db-sqlite-node
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
Pick the driver for your runtime: `@statewalker/db-sqlite-node` on Node, `@statewalker/db-sqlite-do`
|
|
41
|
+
in a Durable Object, `@statewalker/db-sqlite-browser` in the browser.
|
|
42
|
+
|
|
40
43
|
`pako` is not a dependency: pass the module to `pakoCodec` / `pakoDeflateCodec` if you use them.
|
|
41
44
|
|
|
42
45
|
One entry point, `@statewalker/webrun-files-sqlite`: ESM (`dist/esm/index.js`), CommonJS
|
|
43
46
|
(`dist/cjs/index.cjs`), types (`dist/index.d.ts`); sources in `src/`. The package imports no
|
|
44
|
-
database
|
|
47
|
+
database driver itself; you pass a `Db` in, so it bundles for any runtime.
|
|
45
48
|
|
|
46
49
|
```typescript
|
|
47
|
-
import {
|
|
50
|
+
import { newNodeSqliteDb } from '@statewalker/db-sqlite-node';
|
|
51
|
+
import { SqliteFilesApi } from '@statewalker/webrun-files-sqlite';
|
|
48
52
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
53
|
+
const db = await newNodeSqliteDb({ path: 'files.db' }); // node:sqlite
|
|
54
|
+
// or, in a Durable Object (ctx.storage, not ctx.storage.sql):
|
|
55
|
+
// import { newDoSqliteDb } from '@statewalker/db-sqlite-do';
|
|
56
|
+
// const db = newDoSqliteDb(this.ctx.storage);
|
|
52
57
|
|
|
53
|
-
const files = new SqliteFilesApi(
|
|
58
|
+
const files = new SqliteFilesApi(db, { blockSize: 1024 * 1024 });
|
|
54
59
|
await files.init(); // creates tables and indexes if missing; await it before any other call
|
|
55
60
|
```
|
|
56
61
|
|
|
@@ -76,10 +81,10 @@ await files.init(); // creates tables and indexes if missing; await it before an
|
|
|
76
81
|
### Stream a large file through `SqliteFilesApi`
|
|
77
82
|
|
|
78
83
|
```typescript
|
|
79
|
-
import {
|
|
80
|
-
import {
|
|
84
|
+
import { newNodeSqliteDb } from '@statewalker/db-sqlite-node';
|
|
85
|
+
import { SqliteFilesApi } from '@statewalker/webrun-files-sqlite';
|
|
81
86
|
|
|
82
|
-
const files = new SqliteFilesApi(
|
|
87
|
+
const files = new SqliteFilesApi(await newNodeSqliteDb({ path: 'files.db' }));
|
|
83
88
|
await files.init();
|
|
84
89
|
|
|
85
90
|
await files.write('/video.mp4', chunks); // any Iterable or AsyncIterable of Uint8Array
|
|
@@ -91,9 +96,10 @@ for await (const chunk of files.read('/video.mp4', { start: 10_000_000, length:
|
|
|
91
96
|
### Inside a Durable Object
|
|
92
97
|
|
|
93
98
|
```typescript
|
|
94
|
-
import {
|
|
99
|
+
import { newDoSqliteDb } from '@statewalker/db-sqlite-do';
|
|
100
|
+
import { SqliteFilesApi } from '@statewalker/webrun-files-sqlite';
|
|
95
101
|
|
|
96
|
-
const files = new SqliteFilesApi(
|
|
102
|
+
const files = new SqliteFilesApi(newDoSqliteDb(this.ctx.storage));
|
|
97
103
|
await files.init();
|
|
98
104
|
```
|
|
99
105
|
|
|
@@ -106,12 +112,12 @@ const removed = await files.sweep({ olderThan: 60 * 60_000 }); // contents untou
|
|
|
106
112
|
### A portable archive with `SqlarFilesApi`
|
|
107
113
|
|
|
108
114
|
```typescript
|
|
109
|
-
import {
|
|
115
|
+
import { newNodeSqliteDb } from '@statewalker/db-sqlite-node';
|
|
110
116
|
import * as pako from 'pako';
|
|
111
117
|
import { readText, writeText } from '@statewalker/webrun-files';
|
|
112
|
-
import {
|
|
118
|
+
import { pakoCodec, SqlarFilesApi } from '@statewalker/webrun-files-sqlite';
|
|
113
119
|
|
|
114
|
-
const files = new SqlarFilesApi(
|
|
120
|
+
const files = new SqlarFilesApi(await newNodeSqliteDb({ path: 'site.sqlar' }), {
|
|
115
121
|
codec: pakoCodec(pako, { level: 9 }),
|
|
116
122
|
});
|
|
117
123
|
await files.init(); // creates the sqlar table if missing
|
|
@@ -123,38 +129,23 @@ await files.symlink('/latest.md', './docs/index.md');
|
|
|
123
129
|
await files.readlink('/latest.md'); // "./docs/index.md"
|
|
124
130
|
```
|
|
125
131
|
|
|
126
|
-
### Your own SQLite binding
|
|
127
|
-
|
|
128
|
-
Implement the driver interface:
|
|
129
|
-
|
|
130
|
-
```typescript
|
|
131
|
-
import type { SqlDriver, SqlStatement } from '@statewalker/webrun-files-sqlite';
|
|
132
|
-
|
|
133
|
-
interface SqlStatement {
|
|
134
|
-
sql: string;
|
|
135
|
-
params?: unknown[];
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
interface SqlDriver {
|
|
139
|
-
/** Rows as objects keyed by column name. */
|
|
140
|
-
all<T>(sql: string, ...params: unknown[]): T[] | Promise<T[]>;
|
|
141
|
-
/** One statement, for its effect. */
|
|
142
|
-
run(sql: string, ...params: unknown[]): void | Promise<void>;
|
|
143
|
-
/** The statements in order, as one transaction: all apply or none do. Returns rows changed per
|
|
144
|
-
* statement. No statement may depend on another's result. */
|
|
145
|
-
transaction(statements: SqlStatement[]): number[] | Promise<number[]>;
|
|
146
|
-
}
|
|
147
|
-
```
|
|
148
|
-
|
|
149
132
|
## Internals
|
|
150
133
|
|
|
151
|
-
###
|
|
134
|
+
### Why every step runs through `inTransaction`, and what breaks it
|
|
152
135
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
`
|
|
136
|
+
Each step that changes what a path shows runs as one SQLite transaction through `inTransaction`
|
|
137
|
+
from `@statewalker/db-sqlite`. It runs as `BEGIN IMMEDIATE … COMMIT` behind a mutex on Node and in
|
|
138
|
+
the browser, and through `ctx.storage.transaction()` in a Durable Object, where `sql.exec` rejects
|
|
139
|
+
`BEGIN`. The rows a statement changed come from `SELECT changes()` inside that transaction. Steps
|
|
140
|
+
never depend on reading another statement's result: conditions live in SQL (`WHERE EXISTS …`).
|
|
141
|
+
|
|
142
|
+
`inTransaction` finds the driver's transaction support through a hook on the `Db` it was given.
|
|
143
|
+
**A `Db` wrapped in your own proxy (logging, metrics) loses the hook.** The first `write`,
|
|
144
|
+
`remove`, `copy`, `move` or `sweep` then rejects with: `inTransaction: this Db has no SQLite
|
|
145
|
+
transaction hook. It is not a @statewalker/db-sqlite* Db, or it wraps one (a wrapper object loses
|
|
146
|
+
the hook - pass the driver's Db itself), or it is the handle of a running transaction (nesting is
|
|
147
|
+
not supported).` Pass the driver's `Db` itself, or forward `db[SQLITE_TRANSACTION]` (exported by
|
|
148
|
+
`@statewalker/db-sqlite`) in your wrapper.
|
|
158
149
|
|
|
159
150
|
### How `SqliteFilesApi` stores content
|
|
160
151
|
|
|
@@ -178,9 +169,9 @@ it too. A content and its blocks are deleted when the last path pointing at it g
|
|
|
178
169
|
### Why the block size is capped at 1.5 MiB
|
|
179
170
|
|
|
180
171
|
A deflated block can be slightly larger than its input. 1.5 MiB keeps even incompressible input
|
|
181
|
-
under the 2 MB row limit of Durable Objects
|
|
182
|
-
|
|
183
|
-
|
|
172
|
+
under the 2 MB row limit of Durable Objects. Larger blocks mean fewer rows; smaller blocks make
|
|
173
|
+
small range reads cheaper. Each content records its own block size, so changing `blockSize` never
|
|
174
|
+
affects existing files.
|
|
184
175
|
|
|
185
176
|
### Streaming and memory
|
|
186
177
|
|
|
@@ -236,14 +227,27 @@ does. A dangling link, or a chain longer than 8, behaves as a missing path. `rem
|
|
|
236
227
|
### `SqlarFilesApi` limits
|
|
237
228
|
|
|
238
229
|
- A file is one blob: `write` buffers it, `read` inflates it whole and yields one chunk.
|
|
239
|
-
- On Durable Objects
|
|
240
|
-
|
|
230
|
+
- On Durable Objects the stored (possibly compressed) size of one file must stay under 2 MB, since
|
|
231
|
+
SQLAR keeps each file in one row. A larger file fails the `write` with `string or blob too big`
|
|
232
|
+
— a value limit, not corruption. Store big files with `SqliteFilesApi`, whose blocks stay under
|
|
233
|
+
the limit.
|
|
234
|
+
- Multi-row mutations (`copy`, `move`, `remove` of a tree) are not wrapped in a transaction: a
|
|
235
|
+
failure part-way leaves the rows already changed in place, and a `move` can leave the entry under
|
|
236
|
+
both paths.
|
|
241
237
|
- `list` loads every row under the directory, then sorts the entries into path order before
|
|
242
238
|
applying `after`.
|
|
243
239
|
|
|
244
240
|
### Dependencies
|
|
245
241
|
|
|
246
|
-
`@statewalker/
|
|
242
|
+
- `@statewalker/db-api` — the `Db` contract both implementations run on.
|
|
243
|
+
- `@statewalker/db-sqlite` — `inTransaction`, for `SqliteFilesApi`'s atomic steps.
|
|
244
|
+
- `@statewalker/webrun-files` — the `FilesApi` contract and path helpers.
|
|
245
|
+
|
|
246
|
+
The browser driver (`@statewalker/db-sqlite-browser`) is supported through the same `Db` contract, but
|
|
247
|
+
this package's tests run on `node:sqlite` and the Durable Object fake only.
|
|
248
|
+
|
|
249
|
+
The drivers (`@statewalker/db-sqlite-node`, `@statewalker/db-sqlite-do`, …) are the caller's
|
|
250
|
+
choice and are not dependencies; neither is `pako`, which is passed in.
|
|
247
251
|
|
|
248
252
|
## License
|
|
249
253
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
1
2
|
import type { FileInfo, FileStats, FilesApi, ListOptions, ReadOptions } from "@statewalker/webrun-files";
|
|
2
|
-
import type { SqlDriver } from "../sql.types.js";
|
|
3
3
|
import { type StreamCodec } from "./stream-codec.js";
|
|
4
4
|
export interface SqliteFilesApiOptions {
|
|
5
5
|
/** Prefix of the three table names. Defaults to "fs_". */
|
|
@@ -20,7 +20,7 @@ export interface SqliteFilesApiOptions {
|
|
|
20
20
|
* A FilesApi over three SQLite tables: paths point at immutable contents, and
|
|
21
21
|
* a content is a sequence of bounded, optionally compressed blocks. Reads and
|
|
22
22
|
* writes stream one block at a time, so no file is ever held whole and no row
|
|
23
|
-
* outgrows the 2 MB limit of Durable Objects
|
|
23
|
+
* outgrows the 2 MB row limit of Durable Objects.
|
|
24
24
|
*
|
|
25
25
|
* Streaming happens outside transactions; every step that changes what a path
|
|
26
26
|
* shows is one transaction, so an interruption leaves at most an unreferenced
|
|
@@ -28,7 +28,7 @@ export interface SqliteFilesApiOptions {
|
|
|
28
28
|
*/
|
|
29
29
|
export declare class SqliteFilesApi implements FilesApi {
|
|
30
30
|
#private;
|
|
31
|
-
constructor(
|
|
31
|
+
constructor(db: Db, opts?: SqliteFilesApiOptions);
|
|
32
32
|
/** Creates the tables and indexes if missing. Await it before any other call. */
|
|
33
33
|
init(): Promise<void>;
|
|
34
34
|
read(path: string, options?: ReadOptions): AsyncIterable<Uint8Array>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite-files-api.d.ts","sourceRoot":"","sources":["../../src/blocks/sqlite-files-api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"sqlite-files-api.d.ts","sourceRoot":"","sources":["../../src/blocks/sqlite-files-api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,qBAAqB,CAAC;AAE9C,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;AAMnC,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,mBAAmB,CAAC;AAOtE,MAAM,WAAW,qBAAqB;IACpC,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAcD;;;;;;;;;GASG;AACH,qBAAa,cAAe,YAAW,QAAQ;;IAS7C,YAAY,EAAE,EAAE,EAAE,EAAE,IAAI,GAAE,qBAA0B,EAqBnD;IAED,iFAAiF;IAC3E,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CA0B1B;IAIM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,aAAa,CAAC,UAAU,CAAC,CA8C9E;IAID;;;;OAIG;IACG,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,IAAI,CAAC,CAuEf;IA8CK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQvC;IAIK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAKxD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAE3C;IAEM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,aAAa,CAAC,QAAQ,CAAC,CA4B5E;IAIK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3C;IAgCK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAE3D;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAE3D;IA6CD;;;;;;;;OAQG;IACG,KAAK,CAAC,OAAO,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAqB3D;CAqEF"}
|
package/dist/bytes.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/** BLOBs arrive as Uint8Array from every @statewalker/db-sqlite driver; SQLAR may hold TEXT. */
|
|
2
2
|
export declare function toBytes(value: unknown): Uint8Array;
|
|
3
3
|
//# sourceMappingURL=bytes.d.ts.map
|
package/dist/bytes.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bytes.d.ts","sourceRoot":"","sources":["../src/bytes.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,
|
|
1
|
+
{"version":3,"file":"bytes.d.ts","sourceRoot":"","sources":["../src/bytes.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAIlD"}
|
package/dist/cjs/index.cjs
CHANGED
|
@@ -1,4 +1,19 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
//#region ../../node_modules/.pnpm/@statewalker+db-sqlite@0.1.0/node_modules/@statewalker/db-sqlite/dist/index.mjs
|
|
3
|
+
/** Key under which a SQLite Db carries its transaction hook. Shared across bundles via Symbol.for. */
|
|
4
|
+
const SQLITE_TRANSACTION = Symbol.for("@statewalker/db-sqlite:transaction");
|
|
5
|
+
const NO_HOOK = "inTransaction: this Db has no SQLite transaction hook. It is not a @statewalker/db-sqlite* Db, or it wraps one (a wrapper object loses the hook - pass the driver's Db itself), or it is the handle of a running transaction (nesting is not supported).";
|
|
6
|
+
/**
|
|
7
|
+
* Run `cb` in one SQLite transaction: every statement issued through `tx` commits together, or
|
|
8
|
+
* none does if `cb` throws. Other calls on `db` wait until the transaction ends. `cb` must not do
|
|
9
|
+
* external I/O and must be idempotent: a Durable Object may run it more than once.
|
|
10
|
+
*/
|
|
11
|
+
function inTransaction(db, cb) {
|
|
12
|
+
const hook = db[SQLITE_TRANSACTION];
|
|
13
|
+
if (typeof hook !== "function") return Promise.reject(/* @__PURE__ */ new Error(NO_HOOK));
|
|
14
|
+
return hook(cb);
|
|
15
|
+
}
|
|
16
|
+
//#endregion
|
|
2
17
|
//#region ../webrun-files/dist/esm/index.js
|
|
3
18
|
/**
|
|
4
19
|
* The order every `FilesApi.list()` yields: paths compared by Unicode code
|
|
@@ -38,15 +53,31 @@ function normalizePath(filePath) {
|
|
|
38
53
|
}
|
|
39
54
|
//#endregion
|
|
40
55
|
//#region src/bytes.ts
|
|
41
|
-
/**
|
|
56
|
+
/** BLOBs arrive as Uint8Array from every @statewalker/db-sqlite driver; SQLAR may hold TEXT. */
|
|
42
57
|
function toBytes(value) {
|
|
43
58
|
if (value instanceof Uint8Array) return value;
|
|
44
|
-
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
45
|
-
if (Array.isArray(value)) return Uint8Array.from(value);
|
|
46
59
|
if (typeof value === "string") return new TextEncoder().encode(value);
|
|
47
60
|
return /* @__PURE__ */ new Uint8Array(0);
|
|
48
61
|
}
|
|
49
62
|
//#endregion
|
|
63
|
+
//#region src/run-statements.ts
|
|
64
|
+
/**
|
|
65
|
+
* Runs the statements in order on a transaction's Db and returns, per
|
|
66
|
+
* statement, the rows it changed — zero exactly when it changed nothing. The
|
|
67
|
+
* count is SQLite's `changes()` on the same connection, so index writes are
|
|
68
|
+
* not counted. Call it only inside `inTransaction`; it does nothing but SQL,
|
|
69
|
+
* so a retried transaction callback is safe.
|
|
70
|
+
*/
|
|
71
|
+
async function runStatements(tx, statements) {
|
|
72
|
+
const counts = [];
|
|
73
|
+
for (const { sql, params = [] } of statements) {
|
|
74
|
+
await tx.query(sql, params);
|
|
75
|
+
const [row] = await tx.query("SELECT changes() AS n");
|
|
76
|
+
counts.push(Number(row.n));
|
|
77
|
+
}
|
|
78
|
+
return counts;
|
|
79
|
+
}
|
|
80
|
+
//#endregion
|
|
50
81
|
//#region src/blocks/byte-reader.ts
|
|
51
82
|
/**
|
|
52
83
|
* Pull bytes from a chunk source in bounded pieces.
|
|
@@ -353,33 +384,33 @@ async function* iterate(stream) {
|
|
|
353
384
|
//#endregion
|
|
354
385
|
//#region src/blocks/sqlite-files-api.ts
|
|
355
386
|
const MiB = 1048576;
|
|
356
|
-
/** Rows on Durable Objects
|
|
387
|
+
/** Rows on Durable Objects are capped at 2 MB; leave room for deflate's worst case. */
|
|
357
388
|
const MAX_BLOCK_SIZE_LIMIT = 1.5 * MiB;
|
|
358
389
|
const LIST_PAGE = 256;
|
|
359
390
|
/**
|
|
360
391
|
* A FilesApi over three SQLite tables: paths point at immutable contents, and
|
|
361
392
|
* a content is a sequence of bounded, optionally compressed blocks. Reads and
|
|
362
393
|
* writes stream one block at a time, so no file is ever held whole and no row
|
|
363
|
-
* outgrows the 2 MB limit of Durable Objects
|
|
394
|
+
* outgrows the 2 MB row limit of Durable Objects.
|
|
364
395
|
*
|
|
365
396
|
* Streaming happens outside transactions; every step that changes what a path
|
|
366
397
|
* shows is one transaction, so an interruption leaves at most an unreferenced
|
|
367
398
|
* content, which {@link sweep} removes.
|
|
368
399
|
*/
|
|
369
400
|
var SqliteFilesApi = class {
|
|
370
|
-
#
|
|
401
|
+
#db;
|
|
371
402
|
#codec;
|
|
372
403
|
#blockSize;
|
|
373
404
|
#paths;
|
|
374
405
|
#files;
|
|
375
406
|
#blocks;
|
|
376
407
|
#prefix;
|
|
377
|
-
constructor(
|
|
408
|
+
constructor(db, opts = {}) {
|
|
378
409
|
const prefix = opts.tablePrefix ?? "fs_";
|
|
379
410
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(prefix)) throw new Error(`SqliteFilesApi: tablePrefix must be a plain SQL identifier, got ${prefix}`);
|
|
380
411
|
this.#blockSize = opts.blockSize ?? MiB;
|
|
381
412
|
if (!Number.isInteger(this.#blockSize) || this.#blockSize < 1 || this.#blockSize > MAX_BLOCK_SIZE_LIMIT) throw new Error(`SqliteFilesApi: blockSize must be an integer from 1 to ${MAX_BLOCK_SIZE_LIMIT}, got ${this.#blockSize}`);
|
|
382
|
-
this.#
|
|
413
|
+
this.#db = db;
|
|
383
414
|
this.#codec = opts.compression === void 0 ? availableWebCodec() : opts.compression;
|
|
384
415
|
this.#prefix = prefix;
|
|
385
416
|
this.#paths = `${prefix}paths`;
|
|
@@ -412,7 +443,7 @@ var SqliteFilesApi = class {
|
|
|
412
443
|
`CREATE INDEX IF NOT EXISTS ${this.#prefix}paths_fid ON ${this.#paths}(fid)`,
|
|
413
444
|
`CREATE INDEX IF NOT EXISTS ${this.#prefix}files_hash ON ${this.#files}(hash, size)`
|
|
414
445
|
];
|
|
415
|
-
for (const statement of statements) await this.#
|
|
446
|
+
for (const statement of statements) await this.#db.query(statement);
|
|
416
447
|
}
|
|
417
448
|
async *read(path, options = {}) {
|
|
418
449
|
options.signal?.throwIfAborted();
|
|
@@ -456,8 +487,12 @@ var SqliteFilesApi = class {
|
|
|
456
487
|
async write(path, content) {
|
|
457
488
|
const name = normalizePath(path);
|
|
458
489
|
const compression = this.#codec?.name ?? "none";
|
|
459
|
-
const fid =
|
|
460
|
-
|
|
490
|
+
const fid = await inTransaction(this.#db, async (tx) => (await tx.query(`INSERT INTO ${this.#files}(size, compression, block_size, hash, updated)
|
|
491
|
+
VALUES (NULL, ?, ?, NULL, ?) RETURNING fid`, [
|
|
492
|
+
compression,
|
|
493
|
+
this.#blockSize,
|
|
494
|
+
Date.now()
|
|
495
|
+
]))[0].fid);
|
|
461
496
|
const limit = this.#blockSize;
|
|
462
497
|
const reader = new ByteReader(content);
|
|
463
498
|
const hash = new Sha256();
|
|
@@ -478,7 +513,7 @@ var SqliteFilesApi = class {
|
|
|
478
513
|
}
|
|
479
514
|
};
|
|
480
515
|
const block = await collectBlock(this.#codec ? this.#codec.compress(pieces()) : pieces());
|
|
481
|
-
const [alive] = await this.#
|
|
516
|
+
const [alive] = await this.#transaction([{
|
|
482
517
|
sql: `UPDATE ${this.#files} SET updated = ? WHERE fid = ? AND size IS NULL`,
|
|
483
518
|
params: [Date.now(), fid]
|
|
484
519
|
}, {
|
|
@@ -495,14 +530,14 @@ var SqliteFilesApi = class {
|
|
|
495
530
|
size += raw;
|
|
496
531
|
}
|
|
497
532
|
const previous = await this.#entry(name);
|
|
498
|
-
const [finished] = await this.#
|
|
533
|
+
const [finished] = await this.#transaction(this.#finishWrite(name, fid, size, hash.digestHex(), compression, previous?.fid ?? null));
|
|
499
534
|
if (finished === 0) throw sweptDuringWrite(name);
|
|
500
535
|
} catch (error) {
|
|
501
536
|
try {
|
|
502
537
|
await reader.close();
|
|
503
538
|
} catch {}
|
|
504
539
|
try {
|
|
505
|
-
await this.#
|
|
540
|
+
await this.#transaction([{
|
|
506
541
|
sql: `DELETE FROM ${this.#files} WHERE fid = ? AND size IS NULL`,
|
|
507
542
|
params: [fid]
|
|
508
543
|
}, ...this.#deleteBlocksOfMissing(fid)]);
|
|
@@ -567,7 +602,7 @@ var SqliteFilesApi = class {
|
|
|
567
602
|
}
|
|
568
603
|
async mkdir(path) {
|
|
569
604
|
const now = Date.now();
|
|
570
|
-
for (const dir of ancestorsAndSelf$1(normalizePath(path))) await this.#
|
|
605
|
+
for (const dir of ancestorsAndSelf$1(normalizePath(path))) await this.#db.query(`INSERT INTO ${this.#paths}(path, fid, mtime) VALUES (?, NULL, ?) ON CONFLICT(path) DO NOTHING`, [dir, now]);
|
|
571
606
|
}
|
|
572
607
|
async stats(path) {
|
|
573
608
|
const name = normalizePath(path);
|
|
@@ -588,10 +623,10 @@ var SqliteFilesApi = class {
|
|
|
588
623
|
for (;;) {
|
|
589
624
|
const params = [...range.params, cursor];
|
|
590
625
|
if (!options.recursive) params.push(prefixLength + 1);
|
|
591
|
-
const rows = await this.#
|
|
626
|
+
const rows = await this.#db.query(`SELECT p.path, p.fid, p.mtime, f.size, f.compression, f.block_size
|
|
592
627
|
FROM ${this.#paths} p LEFT JOIN ${this.#files} f ON f.fid = p.fid
|
|
593
628
|
WHERE ${range.where} AND p.path > ? ${directOnly}
|
|
594
|
-
ORDER BY p.path LIMIT ${LIST_PAGE}`,
|
|
629
|
+
ORDER BY p.path LIMIT ${LIST_PAGE}`, params);
|
|
595
630
|
for (const row of rows) {
|
|
596
631
|
const stats = toStats$1(row);
|
|
597
632
|
if (stats) yield {
|
|
@@ -605,7 +640,7 @@ var SqliteFilesApi = class {
|
|
|
605
640
|
}
|
|
606
641
|
}
|
|
607
642
|
async remove(path) {
|
|
608
|
-
const counts = await this.#
|
|
643
|
+
const counts = await this.#transaction(this.#removeTree(normalizePath(path)));
|
|
609
644
|
return counts[counts.length - 1] > 0;
|
|
610
645
|
}
|
|
611
646
|
/**
|
|
@@ -695,7 +730,7 @@ var SqliteFilesApi = class {
|
|
|
695
730
|
]
|
|
696
731
|
}
|
|
697
732
|
];
|
|
698
|
-
const counts = await this.#
|
|
733
|
+
const counts = await this.#transaction(statements);
|
|
699
734
|
return counts[counts.length - 1] > 0;
|
|
700
735
|
}
|
|
701
736
|
/**
|
|
@@ -714,7 +749,7 @@ var SqliteFilesApi = class {
|
|
|
714
749
|
const eligible = `f.updated < ? AND NOT EXISTS (
|
|
715
750
|
SELECT 1 FROM ${this.#paths} p WHERE p.fid = f.fid
|
|
716
751
|
)`;
|
|
717
|
-
const [, contents] = await this.#
|
|
752
|
+
const [, contents] = await this.#transaction([{
|
|
718
753
|
sql: `DELETE FROM ${this.#blocks} WHERE fid IN (SELECT f.fid FROM ${this.#files} f WHERE ${eligible})`,
|
|
719
754
|
params: [cutoff]
|
|
720
755
|
}, {
|
|
@@ -723,10 +758,14 @@ var SqliteFilesApi = class {
|
|
|
723
758
|
}]);
|
|
724
759
|
return contents;
|
|
725
760
|
}
|
|
761
|
+
/** The statements as one transaction; per statement, the rows it changed. */
|
|
762
|
+
#transaction(statements) {
|
|
763
|
+
return inTransaction(this.#db, (tx) => runStatements(tx, statements));
|
|
764
|
+
}
|
|
726
765
|
async #entry(name) {
|
|
727
|
-
return (await this.#
|
|
766
|
+
return (await this.#db.query(`SELECT p.fid, p.mtime, f.size, f.compression, f.block_size
|
|
728
767
|
FROM ${this.#paths} p LEFT JOIN ${this.#files} f ON f.fid = p.fid
|
|
729
|
-
WHERE p.path = ?`, name))[0];
|
|
768
|
+
WHERE p.path = ?`, [name]))[0];
|
|
730
769
|
}
|
|
731
770
|
#decoder(name, compression) {
|
|
732
771
|
if (compression === "none") return null;
|
|
@@ -738,7 +777,7 @@ var SqliteFilesApi = class {
|
|
|
738
777
|
throw new Error(`sqlite-files: ${name} is compressed with ${compression}, and no codec reads it`);
|
|
739
778
|
}
|
|
740
779
|
async #block(fid, shift) {
|
|
741
|
-
return (await this.#
|
|
780
|
+
return (await this.#db.query(`SELECT shift, block FROM ${this.#blocks} WHERE fid = ? AND shift = ?`, [fid, shift]))[0];
|
|
742
781
|
}
|
|
743
782
|
/** A subquery selecting the content `fid` while it is still pending. */
|
|
744
783
|
#pending(fid) {
|
|
@@ -872,79 +911,6 @@ async function pipe(bytes, transform) {
|
|
|
872
911
|
return new Uint8Array(await new Response(stream).arrayBuffer());
|
|
873
912
|
}
|
|
874
913
|
//#endregion
|
|
875
|
-
//#region src/drivers.ts
|
|
876
|
-
/** node:sqlite — local development, tests and any Node deployment. */
|
|
877
|
-
var NodeSqlDriver = class {
|
|
878
|
-
#db;
|
|
879
|
-
constructor(db) {
|
|
880
|
-
this.#db = db;
|
|
881
|
-
}
|
|
882
|
-
all(sql, ...params) {
|
|
883
|
-
return this.#db.prepare(sql).all(...params);
|
|
884
|
-
}
|
|
885
|
-
run(sql, ...params) {
|
|
886
|
-
this.#db.prepare(sql).run(...params);
|
|
887
|
-
}
|
|
888
|
-
/** Synchronous from BEGIN to COMMIT, so no other call can interleave on the connection. */
|
|
889
|
-
transaction(statements) {
|
|
890
|
-
this.#db.exec("BEGIN IMMEDIATE");
|
|
891
|
-
try {
|
|
892
|
-
const counts = statements.map(({ sql, params = [] }) => Number(this.#db.prepare(sql).run(...params).changes));
|
|
893
|
-
this.#db.exec("COMMIT");
|
|
894
|
-
return counts;
|
|
895
|
-
} catch (error) {
|
|
896
|
-
this.#db.exec("ROLLBACK");
|
|
897
|
-
throw error;
|
|
898
|
-
}
|
|
899
|
-
}
|
|
900
|
-
};
|
|
901
|
-
/**
|
|
902
|
-
* Durable Object SQLite storage: pass `ctx.storage`. Synchronous; BLOBs come
|
|
903
|
-
* back as ArrayBuffer. Transactions use `transactionSync`, since `sql.exec`
|
|
904
|
-
* rejects `BEGIN`.
|
|
905
|
-
*/
|
|
906
|
-
var DoSqlDriver = class {
|
|
907
|
-
#storage;
|
|
908
|
-
constructor(storage) {
|
|
909
|
-
this.#storage = storage;
|
|
910
|
-
}
|
|
911
|
-
all(sql, ...params) {
|
|
912
|
-
return this.#storage.sql.exec(sql, ...params).toArray();
|
|
913
|
-
}
|
|
914
|
-
run(sql, ...params) {
|
|
915
|
-
this.#storage.sql.exec(sql, ...params).toArray();
|
|
916
|
-
}
|
|
917
|
-
/**
|
|
918
|
-
* Counts come from SQLite's `changes()` on the same connection, not from the
|
|
919
|
-
* cursor's `rowsWritten`, which also counts every index row a write touches.
|
|
920
|
-
*/
|
|
921
|
-
transaction(statements) {
|
|
922
|
-
return this.#storage.transactionSync(() => statements.map(({ sql, params = [] }) => {
|
|
923
|
-
this.#storage.sql.exec(sql, ...params).toArray();
|
|
924
|
-
const [row] = this.#storage.sql.exec("SELECT changes() AS n").toArray();
|
|
925
|
-
return row.n;
|
|
926
|
-
}));
|
|
927
|
-
}
|
|
928
|
-
};
|
|
929
|
-
/** Cloudflare D1. Asynchronous; BLOBs come back as number[]; transactions are `batch()`. */
|
|
930
|
-
var D1SqlDriver = class {
|
|
931
|
-
#db;
|
|
932
|
-
constructor(db) {
|
|
933
|
-
this.#db = db;
|
|
934
|
-
}
|
|
935
|
-
async all(sql, ...params) {
|
|
936
|
-
const { results } = await this.#db.prepare(sql).bind(...params).all();
|
|
937
|
-
return results;
|
|
938
|
-
}
|
|
939
|
-
async run(sql, ...params) {
|
|
940
|
-
await this.#db.prepare(sql).bind(...params).run();
|
|
941
|
-
}
|
|
942
|
-
async transaction(statements) {
|
|
943
|
-
if (statements.length === 0) return [];
|
|
944
|
-
return (await this.#db.batch(statements.map(({ sql, params = [] }) => this.#db.prepare(sql).bind(...params)))).map((result) => result.meta.changes);
|
|
945
|
-
}
|
|
946
|
-
};
|
|
947
|
-
//#endregion
|
|
948
914
|
//#region src/sqlar-files-api.ts
|
|
949
915
|
/**
|
|
950
916
|
* The table, verbatim from the SQLite Archive specification. Five columns,
|
|
@@ -971,22 +937,22 @@ const MAX_SYMLINK_HOPS = 8;
|
|
|
971
937
|
* missing path — and {@link symlink} / {@link readlink} expose the link itself.
|
|
972
938
|
*/
|
|
973
939
|
var SqlarFilesApi = class {
|
|
974
|
-
#
|
|
940
|
+
#db;
|
|
975
941
|
#codec;
|
|
976
|
-
constructor(
|
|
977
|
-
this.#
|
|
942
|
+
constructor(db, opts = {}) {
|
|
943
|
+
this.#db = db;
|
|
978
944
|
this.#codec = opts.codec ?? defaultCodec();
|
|
979
945
|
}
|
|
980
946
|
/** Creates the table if it is missing. Await it before any other call. */
|
|
981
947
|
async init() {
|
|
982
|
-
await this.#
|
|
948
|
+
await this.#db.query(SCHEMA);
|
|
983
949
|
}
|
|
984
950
|
/** A missing path or a directory yields nothing; a file yields one chunk. */
|
|
985
951
|
async *read(path, options = {}) {
|
|
986
952
|
options.signal?.throwIfAborted();
|
|
987
953
|
const resolved = await this.#resolve(toName(path));
|
|
988
954
|
if (resolved?.stats.kind !== "file") return;
|
|
989
|
-
const rows = await this.#
|
|
955
|
+
const rows = await this.#db.query("SELECT data FROM sqlar WHERE name = ?", [resolved.name]);
|
|
990
956
|
const content = await this.#content(resolved.name, resolved.stats.size, toBytes(rows[0]?.data));
|
|
991
957
|
options.signal?.throwIfAborted();
|
|
992
958
|
const start = options.start ?? 0;
|
|
@@ -1051,7 +1017,7 @@ var SqlarFilesApi = class {
|
|
|
1051
1017
|
if (resolved?.stats.kind !== "directory") return;
|
|
1052
1018
|
const base = resolved.name;
|
|
1053
1019
|
const range = descendants(base);
|
|
1054
|
-
const rows = await this.#
|
|
1020
|
+
const rows = await this.#db.query(`SELECT ${ROW_COLUMNS} FROM sqlar WHERE ${range.where} ORDER BY name`, range.params);
|
|
1055
1021
|
const prefixLength = base === "" ? 0 : base.length + 1;
|
|
1056
1022
|
const seen = /* @__PURE__ */ new Set();
|
|
1057
1023
|
for (const row of rows) {
|
|
@@ -1081,8 +1047,8 @@ var SqlarFilesApi = class {
|
|
|
1081
1047
|
}
|
|
1082
1048
|
async remove(path) {
|
|
1083
1049
|
const range = selfAndDescendants(toName(path));
|
|
1084
|
-
if ((await this.#
|
|
1085
|
-
await this.#
|
|
1050
|
+
if ((await this.#db.query(`SELECT 1 FROM sqlar WHERE ${range.where} LIMIT 1`, range.params)).length === 0) return false;
|
|
1051
|
+
await this.#db.query(`DELETE FROM sqlar WHERE ${range.where}`, range.params);
|
|
1086
1052
|
return true;
|
|
1087
1053
|
}
|
|
1088
1054
|
/**
|
|
@@ -1093,7 +1059,7 @@ var SqlarFilesApi = class {
|
|
|
1093
1059
|
const from = toName(source);
|
|
1094
1060
|
const to = toName(target);
|
|
1095
1061
|
const range = selfAndDescendants(from);
|
|
1096
|
-
const rows = await this.#
|
|
1062
|
+
const rows = await this.#db.query(`SELECT name, mode, sz, typeof(data) AS dtype, data FROM sqlar WHERE ${range.where}`, range.params);
|
|
1097
1063
|
if (rows.length === 0) return false;
|
|
1098
1064
|
await this.mkdir(parentName(to));
|
|
1099
1065
|
for (const row of rows) {
|
|
@@ -1147,16 +1113,22 @@ var SqlarFilesApi = class {
|
|
|
1147
1113
|
}
|
|
1148
1114
|
}
|
|
1149
1115
|
async #row(name) {
|
|
1150
|
-
return (await this.#
|
|
1116
|
+
return (await this.#db.query(`SELECT ${ROW_COLUMNS} FROM sqlar WHERE name = ?`, [name]))[0];
|
|
1151
1117
|
}
|
|
1152
1118
|
async #hasDescendants(name) {
|
|
1153
1119
|
const range = descendants(name);
|
|
1154
|
-
return (await this.#
|
|
1120
|
+
return (await this.#db.query(`SELECT 1 FROM sqlar WHERE ${range.where} LIMIT 1`, range.params)).length > 0;
|
|
1155
1121
|
}
|
|
1156
1122
|
async #put(name, mode, sz, data) {
|
|
1157
|
-
await this.#
|
|
1123
|
+
await this.#db.query(`INSERT INTO sqlar (name, mode, mtime, sz, data) VALUES (?, ?, ?, ?, ?)
|
|
1158
1124
|
ON CONFLICT(name) DO UPDATE SET
|
|
1159
|
-
mode = excluded.mode, mtime = excluded.mtime, sz = excluded.sz, data = excluded.data`,
|
|
1125
|
+
mode = excluded.mode, mtime = excluded.mtime, sz = excluded.sz, data = excluded.data`, [
|
|
1126
|
+
name,
|
|
1127
|
+
mode,
|
|
1128
|
+
Math.floor(Date.now() / 1e3),
|
|
1129
|
+
sz,
|
|
1130
|
+
data
|
|
1131
|
+
]);
|
|
1160
1132
|
}
|
|
1161
1133
|
};
|
|
1162
1134
|
const ROW_COLUMNS = "name, mode, mtime, sz, typeof(data) AS dtype, CASE WHEN sz = -1 THEN data END AS target";
|
|
@@ -1251,9 +1223,6 @@ async function concat(content) {
|
|
|
1251
1223
|
return out;
|
|
1252
1224
|
}
|
|
1253
1225
|
//#endregion
|
|
1254
|
-
exports.D1SqlDriver = D1SqlDriver;
|
|
1255
|
-
exports.DoSqlDriver = DoSqlDriver;
|
|
1256
|
-
exports.NodeSqlDriver = NodeSqlDriver;
|
|
1257
1226
|
exports.SqlarFilesApi = SqlarFilesApi;
|
|
1258
1227
|
exports.SqliteFilesApi = SqliteFilesApi;
|
|
1259
1228
|
exports.defaultCodec = defaultCodec;
|