@queria-io/fdl-js 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 flo8s
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,62 @@
1
- # Temporary Holding Version
1
+ # fdl-js
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Read DuckLake tables published as [FDL](https://fdl.flo8s.com) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
4
+
5
+ It reads; it does not write. It reads only tables whose files can be read as they are: a table with deleted rows or renamed columns is refused, since reading its Parquet files without the catalog would return deleted rows or miss columns. Read those with DuckDB.
6
+
7
+ ## Install
8
+
9
+ ```bash
10
+ npm install @queria-io/fdl-js
11
+ ```
12
+
13
+ It is an ES module with type definitions, for Node 20 or later and any bundler.
14
+
15
+ ## Use
16
+
17
+ ```ts
18
+ import { httpStore, open, openParquet, readWhere, tableFiles } from "@queria-io/fdl-js";
19
+
20
+ const store = httpStore("https://data.example.org/towns/");
21
+ const version = await open(store); // HEAD, or open(store, "2025-final"), or a version ID
22
+ const [file] = tableFiles(version, "main.town") ?? [];
23
+ const parquet = await openParquet(store, file);
24
+ const rows = await readWhere(parquet, "lg_code", "131016", ["name"]);
25
+ ```
26
+
27
+ `readWhere` reads only the row groups whose min/max can hold the value, so it reads little from files sorted by that column.
28
+
29
+ ## Stores
30
+
31
+ A store is where the repository's root is. Keys are the layout's paths (`fdl.json`, `refs/heads/main`, `versions/<ID>/files.json`, `data/...`).
32
+
33
+ | Store | Reads from |
34
+ | --- | --- |
35
+ | `httpStore(root, { fetch, headers })` | A URL, with `Range` requests |
36
+ | `r2Store(bucket, prefix)` | An R2 binding in Workers |
37
+ | `fileStore(dir)` from `@queria-io/fdl-js/node` | A local folder |
38
+
39
+ Each retries a failed request once. A store is three functions (`text`, `size`, `range`), so another one is a few lines.
40
+
41
+ ## Caching
42
+
43
+ - Refs move. `refreshing(() => open(store), 5 * 60_000)` keeps the version for five minutes and keeps the last one when a ref cannot be read for a moment
44
+ - Data files never change, so their footers can be kept forever: pass `{ cache: cacheApiFooterCache(caches.default) }` (Workers) or `memoryFooterCache()` to `openParquet`. With `footer_bytes` in `files.json`, an uncached footer is one read
45
+
46
+ ## What it follows
47
+
48
+ [Reading](https://fdl.flo8s.com/spec/reading) in the specification: `HEAD` and refs are followed, `version.json` must name its own ID and a known `fdl` major version, a version with `expired.json` is refused, and file paths from `files.json` are read under `data_dir`. A version without `files.json` cannot be read with this library.
49
+
50
+ ## Develop
51
+
52
+ ```bash
53
+ pnpm install
54
+ pnpm test
55
+ pnpm typecheck
56
+ pnpm check
57
+ pnpm build # dist/
58
+ ```
59
+
60
+ To release, set `version` in `package.json` in a pull request, merge it, and push the tag `v<version>`. The Release workflow publishes it to npm.
61
+
62
+ `test/fixture/repo` is a small repository made by the `fdl` command: `FDL=fdl test/make-fixture.sh` remakes it.
@@ -0,0 +1,3 @@
1
+ export * from "./store.js";
2
+ export * from "./repo.js";
3
+ export * from "./parquet.js";
package/dist/index.js ADDED
@@ -0,0 +1,4 @@
1
+ export * from "./store.js";
2
+ export * from "./repo.js";
3
+ export * from "./parquet.js";
4
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC"}
package/dist/node.d.ts ADDED
@@ -0,0 +1,3 @@
1
+ import { type Store } from "./store.js";
2
+ /** A repository in the folder dir */
3
+ export declare const fileStore: (dir: string) => Store;
package/dist/node.js ADDED
@@ -0,0 +1,22 @@
1
+ // A repository in a local folder, for Node and other runtimes with node:fs
2
+ import { open, readFile, stat } from "node:fs/promises";
3
+ import { join } from "node:path";
4
+ import { NotFoundError } from "./store.js";
5
+ const missing = (e) => e.code === "ENOENT";
6
+ /** A repository in the folder dir */
7
+ export const fileStore = (dir) => ({
8
+ text: (key) => readFile(join(dir, key), "utf8").catch((e) => (missing(e) ? null : Promise.reject(e))),
9
+ size: (key) => stat(join(dir, key)).then((s) => s.size, (e) => (missing(e) ? null : Promise.reject(e))),
10
+ range: async (key, offset, length) => {
11
+ const file = await open(join(dir, key)).catch((e) => Promise.reject(missing(e) ? new NotFoundError(key) : e));
12
+ try {
13
+ const buffer = new Uint8Array(length);
14
+ const { bytesRead } = await file.read(buffer, 0, length, offset);
15
+ return buffer.buffer.slice(0, bytesRead);
16
+ }
17
+ finally {
18
+ await file.close();
19
+ }
20
+ },
21
+ });
22
+ //# sourceMappingURL=node.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"node.js","sourceRoot":"","sources":["../src/node.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAE,CAAuB,CAAC,IAAI,KAAK,QAAQ,CAAC;AAE3E,qCAAqC;AACrC,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAS,EAAE,CAAC,CAAC;IAChD,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CACZ,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACxF,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CACZ,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CACvB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EACb,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAC/C;IACH,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAClD,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACxD,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YACjE,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;CACF,CAAC,CAAC"}
@@ -0,0 +1,49 @@
1
+ import { type AsyncBuffer, type Compressors, type FileMetaData } from "hyparquet";
2
+ import type { DataFile } from "./repo.js";
3
+ import type { Store } from "./store.js";
4
+ /** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
5
+ export declare const compressors: Compressors;
6
+ /** Called after every read, with the bytes read and the milliseconds spent waiting */
7
+ export type OnRead = (bytes: number, ms: number) => void;
8
+ /** A data file as hyparquet's AsyncBuffer */
9
+ export declare const asyncBuffer: (store: Store, key: string, byteLength: number, onRead?: OnRead) => AsyncBuffer;
10
+ /** Where footers are kept between requests. Keys are data file keys; files never change */
11
+ export type FooterCache = {
12
+ get: (key: string) => Promise<ArrayBuffer | undefined>;
13
+ put: (key: string, footer: ArrayBuffer) => Promise<void>;
14
+ };
15
+ /** A FooterCache on a Cache API cache (caches.default in Workers, caches.open() in browsers) */
16
+ export declare const cacheApiFooterCache: (cache: Cache, base?: string) => FooterCache;
17
+ /** A FooterCache in memory, for one process or isolate */
18
+ export declare const memoryFooterCache: () => FooterCache;
19
+ export type ParquetFile = {
20
+ key: string;
21
+ file: AsyncBuffer;
22
+ metadata: FileMetaData;
23
+ };
24
+ export type OpenOptions = {
25
+ cache?: FooterCache;
26
+ onRead?: OnRead;
27
+ };
28
+ /** Opens a data file: reads its footer, from the cache when it is there */
29
+ export declare const openParquet: (store: Store, { key, bytes, footerBytes }: DataFile, { cache, onRead }?: OpenOptions) => Promise<ParquetFile>;
30
+ export type RowRange = {
31
+ rowStart: number;
32
+ rowEnd: number;
33
+ };
34
+ type Value = string | number | bigint;
35
+ /**
36
+ * The rows of the row groups whose min/max for column can hold value. Row groups without
37
+ * statistics are kept. Adjacent row groups are merged into one range
38
+ */
39
+ export declare const rowRangesFor: (metadata: FileMetaData, column: string, value: Value) => RowRange[];
40
+ /** Reads the rows given (all of them by default), the columns given (all by default) */
41
+ export declare const readRows: <T = Record<string, unknown>>({ file, metadata }: ParquetFile, { columns, ...range }?: Partial<RowRange> & {
42
+ columns?: string[];
43
+ }) => Promise<T[]>;
44
+ /**
45
+ * The rows where column equals value. Reads only the row groups that can hold it, then drops
46
+ * the other rows. Works best on files sorted by column
47
+ */
48
+ export declare const readWhere: <T = Record<string, unknown>>(parquet: ParquetFile, column: string, value: Value, columns?: string[]) => Promise<T[]>;
49
+ export {};
@@ -0,0 +1,137 @@
1
+ // Read a version's Parquet files by range with hyparquet: the footer first (cached, since data
2
+ // files never change), then only the row groups whose min/max can hold the value asked for
3
+ import { decompress } from "fzstd";
4
+ import { parquetMetadata, parquetReadObjects, } from "hyparquet";
5
+ /** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
6
+ export const compressors = { ZSTD: (input) => decompress(input) };
7
+ /** A data file as hyparquet's AsyncBuffer */
8
+ export const asyncBuffer = (store, key, byteLength, onRead) => ({
9
+ byteLength,
10
+ slice: async (start, end = byteLength) => {
11
+ const t0 = Date.now();
12
+ const body = await store.range(key, start, end - start);
13
+ onRead?.(end - start, Date.now() - t0);
14
+ return body;
15
+ },
16
+ });
17
+ /** A FooterCache on a Cache API cache (caches.default in Workers, caches.open() in browsers) */
18
+ export const cacheApiFooterCache = (cache, base = "https://fdl.invalid/footer/") => ({
19
+ get: async (key) => (await cache.match(base + encodeURIComponent(key)))?.arrayBuffer(),
20
+ put: (key, footer) => cache.put(base + encodeURIComponent(key), new Response(footer, {
21
+ headers: {
22
+ "content-type": "application/octet-stream",
23
+ "cache-control": "public, max-age=31536000, immutable",
24
+ },
25
+ })),
26
+ });
27
+ /** A FooterCache in memory, for one process or isolate */
28
+ export const memoryFooterCache = () => {
29
+ const kept = new Map();
30
+ return {
31
+ get: async (key) => kept.get(key),
32
+ put: async (key, footer) => void kept.set(key, footer),
33
+ };
34
+ };
35
+ const MAGIC = 0x31524150; // "PAR1"
36
+ /** The footer (metadata and the last 8 bytes). One read when files.json gives footer_bytes */
37
+ const readFooter = async (file, footerBytes) => {
38
+ const size = file.byteLength;
39
+ const guess = footerBytes !== undefined && footerBytes + 8 <= size ? footerBytes + 8 : 8;
40
+ const tail = await file.slice(size - guess, size);
41
+ const view = new DataView(tail);
42
+ if (view.getUint32(guess - 4, true) !== MAGIC)
43
+ throw new Error("not a Parquet file (no PAR1 at the end)");
44
+ const length = view.getUint32(guess - 8, true) + 8;
45
+ if (length > size)
46
+ throw new Error("Parquet footer is longer than the file");
47
+ return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
48
+ };
49
+ /** Opens a data file: reads its footer, from the cache when it is there */
50
+ export const openParquet = async (store, { key, bytes, footerBytes }, { cache, onRead } = {}) => {
51
+ const file = asyncBuffer(store, key, bytes, onRead);
52
+ // The cache only saves reads: when it fails, the footer is read from the file
53
+ const hit = await cache?.get(key).catch(() => undefined);
54
+ if (hit)
55
+ return { key, file, metadata: parquetMetadata(hit) };
56
+ const footer = await readFooter(file, footerBytes);
57
+ await cache?.put(key, footer).catch(() => { });
58
+ return { key, file, metadata: parquetMetadata(footer) };
59
+ };
60
+ const columnIndex = (metadata, column) => {
61
+ // schema[0] is the root; leaves follow in column order (flat tables only)
62
+ const i = metadata.schema.findIndex((s, n) => n > 0 && s.name === column) - 1;
63
+ if (i < 0)
64
+ throw new Error(`no column ${column}`);
65
+ return i;
66
+ };
67
+ // A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
68
+ // and above) after U+E000-U+FFFF
69
+ const unit = (s, i) => {
70
+ const u = s.charCodeAt(i);
71
+ return u >= 0xe000 ? u - 0x800 : u >= 0xd800 ? u + 0x2000 : u;
72
+ };
73
+ // Strings compare in code point order, which is UTF-8 byte order, the order of Parquet's
74
+ // statistics. JavaScript's < compares UTF-16 code units, which differs past U+FFFF
75
+ const atMost = (a, b) => {
76
+ if (typeof a !== "string" || typeof b !== "string")
77
+ return a <= b;
78
+ for (let i = 0; i < Math.min(a.length, b.length); i++) {
79
+ const d = unit(a, i) - unit(b, i);
80
+ if (d !== 0)
81
+ return d < 0;
82
+ }
83
+ return a.length <= b.length;
84
+ };
85
+ const within = (value, min, max) => (min === undefined || min === null || atMost(min, value)) &&
86
+ (max === undefined || max === null || atMost(value, max));
87
+ /**
88
+ * The rows of the row groups whose min/max for column can hold value. Row groups without
89
+ * statistics are kept. Adjacent row groups are merged into one range
90
+ */
91
+ export const rowRangesFor = (metadata, column, value) => {
92
+ const c = columnIndex(metadata, column);
93
+ return metadata.row_groups.reduce(({ start, ranges }, group) => {
94
+ const end = start + Number(group.num_rows);
95
+ const stats = group.columns[c]?.meta_data?.statistics;
96
+ if (!within(value, stats?.min_value ?? stats?.min, stats?.max_value ?? stats?.max))
97
+ return { start: end, ranges };
98
+ const last = ranges.at(-1);
99
+ return {
100
+ start: end,
101
+ ranges: last?.rowEnd === start
102
+ ? [...ranges.slice(0, -1), { rowStart: last.rowStart, rowEnd: end }]
103
+ : [...ranges, { rowStart: start, rowEnd: end }],
104
+ };
105
+ }, { start: 0, ranges: [] }).ranges;
106
+ };
107
+ /** Reads the rows given (all of them by default), the columns given (all by default) */
108
+ export const readRows = ({ file, metadata }, { columns, ...range } = {}) => {
109
+ const names = new Set(metadata.schema.slice(1).map((s) => s.name));
110
+ return parquetReadObjects({
111
+ file,
112
+ metadata,
113
+ compressors,
114
+ ...range,
115
+ // Columns the file lacks are left out, so a reader can ask for a column added later
116
+ ...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
117
+ });
118
+ };
119
+ // INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
120
+ // by value, and unlike BigInt() it does not throw for 1.5
121
+ const same = (a, b) => a === b ||
122
+ ((typeof a === "bigint" || typeof b === "bigint") &&
123
+ (typeof a === "bigint" || typeof a === "number") &&
124
+ typeof b !== "string" &&
125
+ // oxlint-disable-next-line eqeqeq
126
+ a == b);
127
+ /**
128
+ * The rows where column equals value. Reads only the row groups that can hold it, then drops
129
+ * the other rows. Works best on files sorted by column
130
+ */
131
+ export const readWhere = async (parquet, column, value, columns) => {
132
+ const ranges = rowRangesFor(parquet.metadata, column, value);
133
+ const wanted = columns && [column, ...columns.filter((c) => c !== column)];
134
+ const rows = await Promise.all(ranges.map((r) => readRows(parquet, { ...r, ...(wanted ? { columns: wanted } : {}) })));
135
+ return rows.flat().filter((row) => same(row[column], value));
136
+ };
137
+ //# sourceMappingURL=parquet.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AAInB,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AAMF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAY,EACrC,EAAE,KAAK,EAAE,MAAM,KAAkB,EAAE,EACb,EAAE;IACxB,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,8EAA8E;IAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzD,IAAI,GAAG;QAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;AAC1D,CAAC,CAAC;AAKF,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,0EAA0E;IAC1E,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,EAAE,IAAI,EAAE,QAAQ,EAAe,EAC/B,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,OAAO,kBAAkB,CAAC;QACxB,IAAI;QACJ,QAAQ;QACR,WAAW;QACX,GAAG,KAAK;QACR,oFAAoF;QACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrE,CAAiB,CAAC;AACrB,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
package/dist/repo.d.ts ADDED
@@ -0,0 +1,80 @@
1
+ import { type Store } from "./store.js";
2
+ export declare class FdlError extends Error {
3
+ }
4
+ export type Config = {
5
+ fdl: string;
6
+ id?: string;
7
+ data_dir: string;
8
+ };
9
+ export type VersionRecord = {
10
+ fdl: string;
11
+ id: string;
12
+ seq: number;
13
+ parents: string[];
14
+ created_at: string;
15
+ author: {
16
+ name: string;
17
+ [k: string]: unknown;
18
+ };
19
+ message: string;
20
+ files?: {
21
+ path: string;
22
+ bytes: number;
23
+ sha256: string;
24
+ };
25
+ [k: string]: unknown;
26
+ };
27
+ export type FileEntry = {
28
+ path: string;
29
+ rows: number;
30
+ bytes: number;
31
+ footer_bytes?: number;
32
+ deletes?: {
33
+ path: string;
34
+ rows: number;
35
+ bytes: number;
36
+ };
37
+ };
38
+ export type FileList = {
39
+ tables: Record<string, {
40
+ files: FileEntry[];
41
+ renamed_columns?: boolean;
42
+ }>;
43
+ };
44
+ /** One version of a repository, with its files.json */
45
+ export type Version = {
46
+ config: Config;
47
+ record: VersionRecord;
48
+ files: FileList;
49
+ };
50
+ /** A data file to read: its key in the store, rows and sizes from files.json */
51
+ export type DataFile = {
52
+ key: string;
53
+ rows: number;
54
+ bytes: number;
55
+ footerBytes?: number;
56
+ };
57
+ export declare const readConfig: (store: Store) => Promise<Config>;
58
+ /**
59
+ * The version ID that rev names: a version ID itself, "HEAD", a full ref ("refs/tags/2025-final")
60
+ * or a short one ("main", "2025-final"). Symbolic refs ("ref: refs/heads/main") are followed.
61
+ * Refs change, so callers decide how long to keep the answer
62
+ */
63
+ export declare const resolve: (store: Store, rev?: string) => Promise<string>;
64
+ /** Reads a version: checks its record and expiry, and reads its files.json */
65
+ export declare const readVersion: (store: Store, id: string, config?: Config) => Promise<Version>;
66
+ /** Resolves rev and reads the version it names */
67
+ export declare const open: (store: Store, rev?: string) => Promise<Version>;
68
+ /** The names (schema.table) of the version's tables */
69
+ export declare const tableNames: (version: Version) => string[];
70
+ /**
71
+ * The data files of a table (schema.table), or undefined when the version has no such table.
72
+ * Throws for a table with delete files or renamed columns: reading its Parquet files as they are
73
+ * would return deleted rows or miss columns
74
+ */
75
+ export declare const tableFiles: (version: Version, table: string) => DataFile[] | undefined;
76
+ /**
77
+ * Wraps load so that its result is kept for ttlMs. When a reload fails, the last result is
78
+ * used until the next try, so a ref that cannot be read for a moment does not stop a reader
79
+ */
80
+ export declare const refreshing: <T>(load: () => Promise<T>, ttlMs: number, now?: () => number) => () => Promise<T>;
package/dist/repo.js ADDED
@@ -0,0 +1,149 @@
1
+ // Resolve a ref to a version and list the data files of its tables, following
2
+ // https://fdl.flo8s.com/spec/reading. Only files.json is read: the catalog is never opened
3
+ import { NotFoundError } from "./store.js";
4
+ /** The FDL major versions this reader knows */
5
+ const KNOWN_MAJOR = "0";
6
+ export class FdlError extends Error {
7
+ }
8
+ const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
9
+ const required = async (store, key) => {
10
+ const text = await store.text(key);
11
+ if (text === null)
12
+ throw new NotFoundError(key);
13
+ return text;
14
+ };
15
+ const json = async (store, key) => JSON.parse(await required(store, key));
16
+ const checkMajor = (fdl, what) => {
17
+ if (fdl.split(".")[0] !== KNOWN_MAJOR)
18
+ throw new FdlError(`${what} is FDL ${fdl}, unknown to this reader`);
19
+ };
20
+ // A relative path that stays under the root. Dot segments are also caught percent-encoded,
21
+ // since URLs resolve %2e%2e as ..
22
+ const safe = (path) => !path.startsWith("/") &&
23
+ !path.includes("://") &&
24
+ !path.replace(/%2e/gi, ".").split(/[/\\]/).includes("..");
25
+ export const readConfig = async (store) => {
26
+ const raw = await json(store, "fdl.json");
27
+ checkMajor(String(raw.fdl), "the repository");
28
+ const dataDir = raw.data_dir ?? "data/";
29
+ if (!safe(dataDir))
30
+ throw new FdlError(`data_dir is outside the repository: ${dataDir}`);
31
+ return {
32
+ ...raw,
33
+ fdl: String(raw.fdl),
34
+ data_dir: dataDir === "" || dataDir.endsWith("/") ? dataDir : `${dataDir}/`,
35
+ };
36
+ };
37
+ /** Candidates for a short name, in Git's order (refs/<name>, refs/tags/<name>, refs/heads/<name>) */
38
+ const candidates = (rev) => rev === "HEAD" || rev.startsWith("refs/")
39
+ ? [rev]
40
+ : [`refs/${rev}`, `refs/tags/${rev}`, `refs/heads/${rev}`];
41
+ /**
42
+ * The version ID that rev names: a version ID itself, "HEAD", a full ref ("refs/tags/2025-final")
43
+ * or a short one ("main", "2025-final"). Symbolic refs ("ref: refs/heads/main") are followed.
44
+ * Refs change, so callers decide how long to keep the answer
45
+ */
46
+ export const resolve = async (store, rev = "HEAD") => {
47
+ if (VERSION_ID.test(rev))
48
+ return rev;
49
+ if (!safe(rev))
50
+ throw new FdlError(`no ref ${rev}`);
51
+ const follow = async (key, depth) => {
52
+ const text = await store.text(key);
53
+ if (text === null)
54
+ return null;
55
+ const value = text.trim();
56
+ if (!value.startsWith("ref: ")) {
57
+ if (!VERSION_ID.test(value))
58
+ throw new FdlError(`${key} does not hold a version ID`);
59
+ return value;
60
+ }
61
+ if (depth > 4)
62
+ throw new FdlError(`${key}: too many symbolic refs`);
63
+ const target = value.slice(5).trim();
64
+ if (!target.startsWith("refs/") || !safe(target))
65
+ throw new FdlError(`${key} names a ref outside refs/: ${target}`);
66
+ return follow(target, depth + 1);
67
+ };
68
+ const found = await candidates(rev).reduce(async (prev, key) => (await prev) ?? follow(key, 0), Promise.resolve(null));
69
+ if (!found)
70
+ throw new FdlError(`no ref ${rev}`);
71
+ return found;
72
+ };
73
+ /** Reads a version: checks its record and expiry, and reads its files.json */
74
+ export const readVersion = async (store, id, config) => {
75
+ const dir = `versions/${id}/`;
76
+ const [cfg, text, expired] = await Promise.all([
77
+ config ?? readConfig(store),
78
+ store.text(dir + "version.json"),
79
+ store.text(dir + "expired.json"),
80
+ ]);
81
+ // A withdrawn version may have lost its version.json, so expiry is checked first
82
+ if (expired !== null)
83
+ throw new FdlError(`version ${id} is past retention or withdrawn`);
84
+ if (text === null)
85
+ throw new NotFoundError(dir + "version.json");
86
+ const record = JSON.parse(text);
87
+ if (record.id !== id)
88
+ throw new FdlError(`${dir}version.json holds ${record.id}`);
89
+ checkMajor(String(record.fdl), `version ${id}`);
90
+ const filesPath = record.files?.path ?? "files.json";
91
+ if (!safe(filesPath))
92
+ throw new FdlError(`version ${id} names files.json outside it: ${filesPath}`);
93
+ const files = await store.text(dir + filesPath);
94
+ if (files === null)
95
+ throw new FdlError(`version ${id} has no files.json; read it with DuckDB`);
96
+ return { config: cfg, record, files: JSON.parse(files) };
97
+ };
98
+ /** Resolves rev and reads the version it names */
99
+ export const open = async (store, rev = "HEAD") => {
100
+ const [config, id] = await Promise.all([readConfig(store), resolve(store, rev)]);
101
+ return readVersion(store, id, config);
102
+ };
103
+ /** The names (schema.table) of the version's tables */
104
+ export const tableNames = (version) => Object.keys(version.files.tables);
105
+ /**
106
+ * The data files of a table (schema.table), or undefined when the version has no such table.
107
+ * Throws for a table with delete files or renamed columns: reading its Parquet files as they are
108
+ * would return deleted rows or miss columns
109
+ */
110
+ export const tableFiles = (version, table) => {
111
+ const entry = version.files.tables[table];
112
+ if (!entry)
113
+ return undefined;
114
+ if (entry.renamed_columns)
115
+ throw new FdlError(`${table} has renamed columns`);
116
+ return entry.files.map((f) => {
117
+ if (f.deletes)
118
+ throw new FdlError(`${table} has deleted rows`);
119
+ if (!safe(f.path))
120
+ throw new FdlError(`${table} names a file outside the repository: ${f.path}`);
121
+ return {
122
+ key: version.config.data_dir + f.path,
123
+ rows: f.rows,
124
+ bytes: f.bytes,
125
+ ...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
126
+ };
127
+ });
128
+ };
129
+ /**
130
+ * Wraps load so that its result is kept for ttlMs. When a reload fails, the last result is
131
+ * used until the next try, so a ref that cannot be read for a moment does not stop a reader
132
+ */
133
+ export const refreshing = (load, ttlMs, now = () => Date.now()) => {
134
+ let kept = null;
135
+ return () => {
136
+ if (kept && now() - kept.at < ttlMs)
137
+ return kept.value;
138
+ const previous = kept?.value;
139
+ const value = previous ? load().catch(() => previous) : load();
140
+ const entry = { at: now(), value };
141
+ kept = entry;
142
+ value.catch(() => {
143
+ if (kept === entry)
144
+ kept = null;
145
+ });
146
+ return value;
147
+ };
148
+ };
149
+ //# sourceMappingURL=repo.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiCtC,MAAM,UAAU,GAAG,gEAAgE,CAAC;AAEpF,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO;QACL,GAAG,GAAG;QACN,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACpB,QAAQ,EAAE,OAAO,KAAK,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG;KAC5E,CAAC;AACJ,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QAClB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACf,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAI,IAAsB,EAAE,KAAa,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE;IAC7F,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
@@ -0,0 +1,39 @@
1
+ /** A read-only view of a repository's root */
2
+ export type Store = {
3
+ /** The object as text, or null when it is missing */
4
+ text: (key: string) => Promise<string | null>;
5
+ /** The object's size in bytes, or null when it is missing */
6
+ size: (key: string) => Promise<number | null>;
7
+ /** Bytes [offset, offset + length) of the object. Throws when it is missing */
8
+ range: (key: string, offset: number, length: number) => Promise<ArrayBuffer>;
9
+ };
10
+ export declare class NotFoundError extends Error {
11
+ readonly key: string;
12
+ constructor(key: string);
13
+ }
14
+ /** Calls f again once when it fails. Object stores fail now and then for a moment */
15
+ export declare const retryOnce: <T>(f: () => Promise<T>) => Promise<T>;
16
+ /** The part of an R2 binding (or anything shaped like it) that a reader needs */
17
+ export type R2Like = {
18
+ get: (key: string, options?: {
19
+ range?: {
20
+ offset: number;
21
+ length: number;
22
+ };
23
+ }) => Promise<{
24
+ text: () => Promise<string>;
25
+ arrayBuffer: () => Promise<ArrayBuffer>;
26
+ } | null>;
27
+ head: (key: string) => Promise<{
28
+ size: number;
29
+ } | null>;
30
+ };
31
+ /** A repository in an R2 bucket (a Workers binding), under prefix (ending in "/", or "") */
32
+ export declare const r2Store: (bucket: R2Like, prefix?: string) => Store;
33
+ export type HttpOptions = {
34
+ fetch?: typeof fetch;
35
+ /** Sent with every request, for example an Authorization header */
36
+ headers?: Record<string, string>;
37
+ };
38
+ /** A repository served over HTTP at root (ending in "/") */
39
+ export declare const httpStore: (root: string, { fetch: f, headers }?: HttpOptions) => Store;
package/dist/store.js ADDED
@@ -0,0 +1,69 @@
1
+ // Where a repository is read from. Keys are the layout's paths relative to the root
2
+ // (fdl.json, refs/heads/main, versions/<ID>/version.json, data/...)
3
+ export class NotFoundError extends Error {
4
+ key;
5
+ constructor(key) {
6
+ super(`not found: ${key}`);
7
+ this.key = key;
8
+ }
9
+ }
10
+ /** Calls f again once when it fails. Object stores fail now and then for a moment */
11
+ export const retryOnce = (f) => f().catch(() => f());
12
+ /** A repository in an R2 bucket (a Workers binding), under prefix (ending in "/", or "") */
13
+ export const r2Store = (bucket, prefix = "") => ({
14
+ text: async (key) => {
15
+ const o = await retryOnce(() => bucket.get(prefix + key));
16
+ return o ? o.text() : null;
17
+ },
18
+ size: async (key) => (await retryOnce(() => bucket.head(prefix + key)))?.size ?? null,
19
+ range: async (key, offset, length) => {
20
+ const o = await retryOnce(() => bucket.get(prefix + key, { range: { offset, length } }));
21
+ if (!o)
22
+ throw new NotFoundError(prefix + key);
23
+ return o.arrayBuffer();
24
+ },
25
+ });
26
+ // HEAD, refs and expired.json change; everything else in a repository never does
27
+ const mutable = (key) => key === "HEAD" || key === "refs.json" || key.startsWith("refs/") || key.endsWith("/expired.json");
28
+ /** A repository served over HTTP at root (ending in "/") */
29
+ export const httpStore = (root, { fetch: f = fetch, headers = {} } = {}) => {
30
+ const request = (key, init = {}) => retryOnce(async () => {
31
+ const res = await f(root + key, { ...init, headers: { ...headers, ...init.headers } });
32
+ if (res.status >= 500)
33
+ throw new Error(`${res.status} ${root + key}`);
34
+ return res;
35
+ });
36
+ const ok = (res, key) => {
37
+ if (res.status === 404)
38
+ return false;
39
+ if (!res.ok)
40
+ throw new Error(`${res.status} ${root + key}`);
41
+ return true;
42
+ };
43
+ return {
44
+ text: async (key) => {
45
+ const res = await request(key, mutable(key) ? { cache: "no-store" } : {});
46
+ return ok(res, key) ? res.text() : null;
47
+ },
48
+ size: async (key) => {
49
+ const res = await request(key, { method: "HEAD" });
50
+ if (!ok(res, key))
51
+ return null;
52
+ const length = res.headers.get("content-length");
53
+ if (length === null)
54
+ throw new Error(`no content-length for ${root + key}`);
55
+ return Number(length);
56
+ },
57
+ range: async (key, offset, length) => {
58
+ const res = await request(key, {
59
+ headers: { range: `bytes=${offset}-${offset + length - 1}` },
60
+ });
61
+ if (!ok(res, key))
62
+ throw new NotFoundError(root + key);
63
+ const body = await res.arrayBuffer();
64
+ // A server that ignores Range sends the whole object
65
+ return res.status === 206 ? body : body.slice(offset, offset + length);
66
+ },
67
+ };
68
+ };
69
+ //# sourceMappingURL=store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,oFAAoF;AACpF,oEAAoE;AAYpE,MAAM,OAAO,aAAc,SAAQ,KAAK;IACjB;IAArB,YAAqB,GAAW;QAC9B,KAAK,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;QADR,QAAG,GAAH,GAAG,CAAQ;IAEhC,CAAC;CACF;AAED,qFAAqF;AACrF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAI,CAAmB,EAAc,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;AAWtF,4FAA4F;AAC5F,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,MAAM,GAAG,EAAE,EAAS,EAAE,CAAC,CAAC;IAC9D,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAClB,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7B,CAAC;IACD,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,IAAI;IACrF,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,aAAa,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QAC9C,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACzB,CAAC;CACF,CAAC,CAAC;AAQH,iFAAiF;AACjF,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAC9B,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;AAEpG,4DAA4D;AAC5D,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,IAAY,EACZ,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,EAAE,OAAO,GAAG,EAAE,KAAkB,EAAE,EAC7C,EAAE;IACT,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,OAAoB,EAAE,EAAE,EAAE,CACtD,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,IAAI,GAAG,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACvF,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC;QACtE,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IACL,MAAM,EAAE,GAAG,CAAC,GAAa,EAAE,GAAW,EAAE,EAAE;QACxC,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,KAAK,CAAC;QACrC,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,OAAO;QACL,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAClB,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1E,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1C,CAAC;QACD,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAClB,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YACnD,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACjD,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC;YAC5E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QACD,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;gBAC7B,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,MAAM,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,EAAE;aAC7D,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;gBAAE,MAAM,IAAI,aAAa,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;YACrC,qDAAqD;YACrD,OAAO,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;QACzE,CAAC;KACF,CAAC;AACJ,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,61 @@
1
1
  {
2
2
  "name": "@queria-io/fdl-js",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.1.0",
4
+ "description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
5
+ "keywords": [
6
+ "cloudflare-workers",
7
+ "ducklake",
8
+ "fdl",
9
+ "hyparquet",
10
+ "parquet"
11
+ ],
12
+ "homepage": "https://github.com/queria-io/fdl-js#readme",
13
+ "license": "MIT",
14
+ "repository": {
15
+ "type": "git",
16
+ "url": "git+https://github.com/queria-io/fdl-js.git"
17
+ },
18
+ "files": [
19
+ "dist",
20
+ "src"
21
+ ],
22
+ "type": "module",
23
+ "sideEffects": false,
24
+ "exports": {
25
+ ".": {
26
+ "types": "./dist/index.d.ts",
27
+ "default": "./dist/index.js"
28
+ },
29
+ "./node": {
30
+ "types": "./dist/node.d.ts",
31
+ "default": "./dist/node.js"
32
+ }
33
+ },
34
+ "publishConfig": {
35
+ "access": "public"
36
+ },
37
+ "scripts": {
38
+ "build": "rm -rf dist && tsc -p tsconfig.build.json",
39
+ "prepack": "pnpm build",
40
+ "test": "vitest run",
41
+ "typecheck": "tsc --noEmit",
42
+ "check": "oxlint && oxfmt --check .",
43
+ "fix": "oxlint --fix && oxfmt ."
44
+ },
45
+ "dependencies": {
46
+ "fzstd": "^0.1.1",
47
+ "hyparquet": "^1.32.0"
48
+ },
49
+ "devDependencies": {
50
+ "@cloudflare/workers-types": "^5.20261101.1",
51
+ "@types/node": "^22.0.0",
52
+ "oxfmt": "^0.73.0",
53
+ "oxlint": "^1.76.0",
54
+ "typescript": "^5.7.4",
55
+ "vitest": "^4.1.0"
56
+ },
57
+ "engines": {
58
+ "node": ">=20"
59
+ },
60
+ "packageManager": "pnpm@10.17.1"
61
+ }
package/src/index.ts ADDED
@@ -0,0 +1,3 @@
1
+ export * from "./store.js";
2
+ export * from "./repo.js";
3
+ export * from "./parquet.js";
package/src/node.ts ADDED
@@ -0,0 +1,29 @@
1
+ // A repository in a local folder, for Node and other runtimes with node:fs
2
+ import { open, readFile, stat } from "node:fs/promises";
3
+ import { join } from "node:path";
4
+ import { NotFoundError, type Store } from "./store.js";
5
+
6
+ const missing = (e: unknown) => (e as { code?: string }).code === "ENOENT";
7
+
8
+ /** A repository in the folder dir */
9
+ export const fileStore = (dir: string): Store => ({
10
+ text: (key) =>
11
+ readFile(join(dir, key), "utf8").catch((e) => (missing(e) ? null : Promise.reject(e))),
12
+ size: (key) =>
13
+ stat(join(dir, key)).then(
14
+ (s) => s.size,
15
+ (e) => (missing(e) ? null : Promise.reject(e)),
16
+ ),
17
+ range: async (key, offset, length) => {
18
+ const file = await open(join(dir, key)).catch((e) =>
19
+ Promise.reject(missing(e) ? new NotFoundError(key) : e),
20
+ );
21
+ try {
22
+ const buffer = new Uint8Array(length);
23
+ const { bytesRead } = await file.read(buffer, 0, length, offset);
24
+ return buffer.buffer.slice(0, bytesRead);
25
+ } finally {
26
+ await file.close();
27
+ }
28
+ },
29
+ });
package/src/parquet.ts ADDED
@@ -0,0 +1,204 @@
1
+ // Read a version's Parquet files by range with hyparquet: the footer first (cached, since data
2
+ // files never change), then only the row groups whose min/max can hold the value asked for
3
+ import { decompress } from "fzstd";
4
+ import {
5
+ type AsyncBuffer,
6
+ type Compressors,
7
+ type FileMetaData,
8
+ parquetMetadata,
9
+ parquetReadObjects,
10
+ } from "hyparquet";
11
+ import type { DataFile } from "./repo.js";
12
+ import type { Store } from "./store.js";
13
+
14
+ /** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
15
+ export const compressors: Compressors = { ZSTD: (input) => decompress(input) };
16
+
17
+ /** Called after every read, with the bytes read and the milliseconds spent waiting */
18
+ export type OnRead = (bytes: number, ms: number) => void;
19
+
20
+ /** A data file as hyparquet's AsyncBuffer */
21
+ export const asyncBuffer = (
22
+ store: Store,
23
+ key: string,
24
+ byteLength: number,
25
+ onRead?: OnRead,
26
+ ): AsyncBuffer => ({
27
+ byteLength,
28
+ slice: async (start, end = byteLength) => {
29
+ const t0 = Date.now();
30
+ const body = await store.range(key, start, end - start);
31
+ onRead?.(end - start, Date.now() - t0);
32
+ return body;
33
+ },
34
+ });
35
+
36
+ /** Where footers are kept between requests. Keys are data file keys; files never change */
37
+ export type FooterCache = {
38
+ get: (key: string) => Promise<ArrayBuffer | undefined>;
39
+ put: (key: string, footer: ArrayBuffer) => Promise<void>;
40
+ };
41
+
42
+ /** A FooterCache on a Cache API cache (caches.default in Workers, caches.open() in browsers) */
43
+ export const cacheApiFooterCache = (
44
+ cache: Cache,
45
+ base = "https://fdl.invalid/footer/",
46
+ ): FooterCache => ({
47
+ get: async (key) => (await cache.match(base + encodeURIComponent(key)))?.arrayBuffer(),
48
+ put: (key, footer) =>
49
+ cache.put(
50
+ base + encodeURIComponent(key),
51
+ new Response(footer, {
52
+ headers: {
53
+ "content-type": "application/octet-stream",
54
+ "cache-control": "public, max-age=31536000, immutable",
55
+ },
56
+ }),
57
+ ),
58
+ });
59
+
60
+ /** A FooterCache in memory, for one process or isolate */
61
+ export const memoryFooterCache = (): FooterCache => {
62
+ const kept = new Map<string, ArrayBuffer>();
63
+ return {
64
+ get: async (key) => kept.get(key),
65
+ put: async (key, footer) => void kept.set(key, footer),
66
+ };
67
+ };
68
+
69
+ const MAGIC = 0x31524150; // "PAR1"
70
+
71
+ /** The footer (metadata and the last 8 bytes). One read when files.json gives footer_bytes */
72
+ const readFooter = async (file: AsyncBuffer, footerBytes?: number): Promise<ArrayBuffer> => {
73
+ const size = file.byteLength;
74
+ const guess = footerBytes !== undefined && footerBytes + 8 <= size ? footerBytes + 8 : 8;
75
+ const tail = await file.slice(size - guess, size);
76
+ const view = new DataView(tail);
77
+ if (view.getUint32(guess - 4, true) !== MAGIC)
78
+ throw new Error("not a Parquet file (no PAR1 at the end)");
79
+ const length = view.getUint32(guess - 8, true) + 8;
80
+ if (length > size) throw new Error("Parquet footer is longer than the file");
81
+ return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
82
+ };
83
+
84
+ export type ParquetFile = { key: string; file: AsyncBuffer; metadata: FileMetaData };
85
+
86
+ export type OpenOptions = { cache?: FooterCache; onRead?: OnRead };
87
+
88
+ /** Opens a data file: reads its footer, from the cache when it is there */
89
+ export const openParquet = async (
90
+ store: Store,
91
+ { key, bytes, footerBytes }: DataFile,
92
+ { cache, onRead }: OpenOptions = {},
93
+ ): Promise<ParquetFile> => {
94
+ const file = asyncBuffer(store, key, bytes, onRead);
95
+ // The cache only saves reads: when it fails, the footer is read from the file
96
+ const hit = await cache?.get(key).catch(() => undefined);
97
+ if (hit) return { key, file, metadata: parquetMetadata(hit) };
98
+ const footer = await readFooter(file, footerBytes);
99
+ await cache?.put(key, footer).catch(() => {});
100
+ return { key, file, metadata: parquetMetadata(footer) };
101
+ };
102
+
103
+ export type RowRange = { rowStart: number; rowEnd: number };
104
+ type Value = string | number | bigint;
105
+
106
+ const columnIndex = (metadata: FileMetaData, column: string): number => {
107
+ // schema[0] is the root; leaves follow in column order (flat tables only)
108
+ const i = metadata.schema.findIndex((s, n) => n > 0 && s.name === column) - 1;
109
+ if (i < 0) throw new Error(`no column ${column}`);
110
+ return i;
111
+ };
112
+
113
+ // A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
114
+ // and above) after U+E000-U+FFFF
115
+ const unit = (s: string, i: number) => {
116
+ const u = s.charCodeAt(i);
117
+ return u >= 0xe000 ? u - 0x800 : u >= 0xd800 ? u + 0x2000 : u;
118
+ };
119
+
120
+ // Strings compare in code point order, which is UTF-8 byte order, the order of Parquet's
121
+ // statistics. JavaScript's < compares UTF-16 code units, which differs past U+FFFF
122
+ const atMost = (a: Value, b: Value): boolean => {
123
+ if (typeof a !== "string" || typeof b !== "string") return a <= b;
124
+ for (let i = 0; i < Math.min(a.length, b.length); i++) {
125
+ const d = unit(a, i) - unit(b, i);
126
+ if (d !== 0) return d < 0;
127
+ }
128
+ return a.length <= b.length;
129
+ };
130
+
131
+ const within = (value: Value, min: unknown, max: unknown) =>
132
+ (min === undefined || min === null || atMost(min as Value, value)) &&
133
+ (max === undefined || max === null || atMost(value, max as Value));
134
+
135
+ /**
136
+ * The rows of the row groups whose min/max for column can hold value. Row groups without
137
+ * statistics are kept. Adjacent row groups are merged into one range
138
+ */
139
+ export const rowRangesFor = (metadata: FileMetaData, column: string, value: Value): RowRange[] => {
140
+ const c = columnIndex(metadata, column);
141
+ return metadata.row_groups.reduce<{ start: number; ranges: RowRange[] }>(
142
+ ({ start, ranges }, group) => {
143
+ const end = start + Number(group.num_rows);
144
+ const stats = group.columns[c]?.meta_data?.statistics;
145
+ if (!within(value, stats?.min_value ?? stats?.min, stats?.max_value ?? stats?.max))
146
+ return { start: end, ranges };
147
+ const last = ranges.at(-1);
148
+ return {
149
+ start: end,
150
+ ranges:
151
+ last?.rowEnd === start
152
+ ? [...ranges.slice(0, -1), { rowStart: last.rowStart, rowEnd: end }]
153
+ : [...ranges, { rowStart: start, rowEnd: end }],
154
+ };
155
+ },
156
+ { start: 0, ranges: [] },
157
+ ).ranges;
158
+ };
159
+
160
+ /** Reads the rows given (all of them by default), the columns given (all by default) */
161
+ export const readRows = <T = Record<string, unknown>>(
162
+ { file, metadata }: ParquetFile,
163
+ { columns, ...range }: Partial<RowRange> & { columns?: string[] } = {},
164
+ ): Promise<T[]> => {
165
+ const names = new Set(metadata.schema.slice(1).map((s) => s.name));
166
+ return parquetReadObjects({
167
+ file,
168
+ metadata,
169
+ compressors,
170
+ ...range,
171
+ // Columns the file lacks are left out, so a reader can ask for a column added later
172
+ ...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
173
+ }) as Promise<T[]>;
174
+ };
175
+
176
+ // INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
177
+ // by value, and unlike BigInt() it does not throw for 1.5
178
+ const same = (a: unknown, b: Value) =>
179
+ a === b ||
180
+ ((typeof a === "bigint" || typeof b === "bigint") &&
181
+ (typeof a === "bigint" || typeof a === "number") &&
182
+ typeof b !== "string" &&
183
+ // oxlint-disable-next-line eqeqeq
184
+ a == b);
185
+
186
+ /**
187
+ * The rows where column equals value. Reads only the row groups that can hold it, then drops
188
+ * the other rows. Works best on files sorted by column
189
+ */
190
+ export const readWhere = async <T = Record<string, unknown>>(
191
+ parquet: ParquetFile,
192
+ column: string,
193
+ value: Value,
194
+ columns?: string[],
195
+ ): Promise<T[]> => {
196
+ const ranges = rowRangesFor(parquet.metadata, column, value);
197
+ const wanted = columns && [column, ...columns.filter((c) => c !== column)];
198
+ const rows = await Promise.all(
199
+ ranges.map((r) =>
200
+ readRows<Record<string, unknown>>(parquet, { ...r, ...(wanted ? { columns: wanted } : {}) }),
201
+ ),
202
+ );
203
+ return rows.flat().filter((row) => same(row[column], value)) as T[];
204
+ };
package/src/repo.ts ADDED
@@ -0,0 +1,182 @@
1
+ // Resolve a ref to a version and list the data files of its tables, following
2
+ // https://fdl.flo8s.com/spec/reading. Only files.json is read: the catalog is never opened
3
+ import { NotFoundError, type Store } from "./store.js";
4
+
5
+ /** The FDL major versions this reader knows */
6
+ const KNOWN_MAJOR = "0";
7
+
8
+ export class FdlError extends Error {}
9
+
10
+ export type Config = { fdl: string; id?: string; data_dir: string };
11
+
12
+ export type VersionRecord = {
13
+ fdl: string;
14
+ id: string;
15
+ seq: number;
16
+ parents: string[];
17
+ created_at: string;
18
+ author: { name: string; [k: string]: unknown };
19
+ message: string;
20
+ files?: { path: string; bytes: number; sha256: string };
21
+ [k: string]: unknown;
22
+ };
23
+
24
+ export type FileEntry = {
25
+ path: string;
26
+ rows: number;
27
+ bytes: number;
28
+ footer_bytes?: number;
29
+ deletes?: { path: string; rows: number; bytes: number };
30
+ };
31
+ export type FileList = {
32
+ tables: Record<string, { files: FileEntry[]; renamed_columns?: boolean }>;
33
+ };
34
+
35
+ /** One version of a repository, with its files.json */
36
+ export type Version = { config: Config; record: VersionRecord; files: FileList };
37
+
38
+ /** A data file to read: its key in the store, rows and sizes from files.json */
39
+ export type DataFile = { key: string; rows: number; bytes: number; footerBytes?: number };
40
+
41
+ const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
42
+
43
+ const required = async (store: Store, key: string): Promise<string> => {
44
+ const text = await store.text(key);
45
+ if (text === null) throw new NotFoundError(key);
46
+ return text;
47
+ };
48
+
49
+ const json = async <T>(store: Store, key: string): Promise<T> =>
50
+ JSON.parse(await required(store, key)) as T;
51
+
52
+ const checkMajor = (fdl: string, what: string) => {
53
+ if (fdl.split(".")[0] !== KNOWN_MAJOR)
54
+ throw new FdlError(`${what} is FDL ${fdl}, unknown to this reader`);
55
+ };
56
+
57
+ // A relative path that stays under the root. Dot segments are also caught percent-encoded,
58
+ // since URLs resolve %2e%2e as ..
59
+ const safe = (path: string) =>
60
+ !path.startsWith("/") &&
61
+ !path.includes("://") &&
62
+ !path.replace(/%2e/gi, ".").split(/[/\\]/).includes("..");
63
+
64
+ export const readConfig = async (store: Store): Promise<Config> => {
65
+ const raw = await json<Partial<Config>>(store, "fdl.json");
66
+ checkMajor(String(raw.fdl), "the repository");
67
+ const dataDir = raw.data_dir ?? "data/";
68
+ if (!safe(dataDir)) throw new FdlError(`data_dir is outside the repository: ${dataDir}`);
69
+ return {
70
+ ...raw,
71
+ fdl: String(raw.fdl),
72
+ data_dir: dataDir === "" || dataDir.endsWith("/") ? dataDir : `${dataDir}/`,
73
+ };
74
+ };
75
+
76
+ /** Candidates for a short name, in Git's order (refs/<name>, refs/tags/<name>, refs/heads/<name>) */
77
+ const candidates = (rev: string): string[] =>
78
+ rev === "HEAD" || rev.startsWith("refs/")
79
+ ? [rev]
80
+ : [`refs/${rev}`, `refs/tags/${rev}`, `refs/heads/${rev}`];
81
+
82
+ /**
83
+ * The version ID that rev names: a version ID itself, "HEAD", a full ref ("refs/tags/2025-final")
84
+ * or a short one ("main", "2025-final"). Symbolic refs ("ref: refs/heads/main") are followed.
85
+ * Refs change, so callers decide how long to keep the answer
86
+ */
87
+ export const resolve = async (store: Store, rev = "HEAD"): Promise<string> => {
88
+ if (VERSION_ID.test(rev)) return rev;
89
+ if (!safe(rev)) throw new FdlError(`no ref ${rev}`);
90
+ const follow = async (key: string, depth: number): Promise<string | null> => {
91
+ const text = await store.text(key);
92
+ if (text === null) return null;
93
+ const value = text.trim();
94
+ if (!value.startsWith("ref: ")) {
95
+ if (!VERSION_ID.test(value)) throw new FdlError(`${key} does not hold a version ID`);
96
+ return value;
97
+ }
98
+ if (depth > 4) throw new FdlError(`${key}: too many symbolic refs`);
99
+ const target = value.slice(5).trim();
100
+ if (!target.startsWith("refs/") || !safe(target))
101
+ throw new FdlError(`${key} names a ref outside refs/: ${target}`);
102
+ return follow(target, depth + 1);
103
+ };
104
+ const found = await candidates(rev).reduce<Promise<string | null>>(
105
+ async (prev, key) => (await prev) ?? follow(key, 0),
106
+ Promise.resolve(null),
107
+ );
108
+ if (!found) throw new FdlError(`no ref ${rev}`);
109
+ return found;
110
+ };
111
+
112
+ /** Reads a version: checks its record and expiry, and reads its files.json */
113
+ export const readVersion = async (store: Store, id: string, config?: Config): Promise<Version> => {
114
+ const dir = `versions/${id}/`;
115
+ const [cfg, text, expired] = await Promise.all([
116
+ config ?? readConfig(store),
117
+ store.text(dir + "version.json"),
118
+ store.text(dir + "expired.json"),
119
+ ]);
120
+ // A withdrawn version may have lost its version.json, so expiry is checked first
121
+ if (expired !== null) throw new FdlError(`version ${id} is past retention or withdrawn`);
122
+ if (text === null) throw new NotFoundError(dir + "version.json");
123
+ const record = JSON.parse(text) as VersionRecord;
124
+ if (record.id !== id) throw new FdlError(`${dir}version.json holds ${record.id}`);
125
+ checkMajor(String(record.fdl), `version ${id}`);
126
+ const filesPath = record.files?.path ?? "files.json";
127
+ if (!safe(filesPath))
128
+ throw new FdlError(`version ${id} names files.json outside it: ${filesPath}`);
129
+ const files = await store.text(dir + filesPath);
130
+ if (files === null) throw new FdlError(`version ${id} has no files.json; read it with DuckDB`);
131
+ return { config: cfg, record, files: JSON.parse(files) as FileList };
132
+ };
133
+
134
+ /** Resolves rev and reads the version it names */
135
+ export const open = async (store: Store, rev = "HEAD"): Promise<Version> => {
136
+ const [config, id] = await Promise.all([readConfig(store), resolve(store, rev)]);
137
+ return readVersion(store, id, config);
138
+ };
139
+
140
+ /** The names (schema.table) of the version's tables */
141
+ export const tableNames = (version: Version): string[] => Object.keys(version.files.tables);
142
+
143
+ /**
144
+ * The data files of a table (schema.table), or undefined when the version has no such table.
145
+ * Throws for a table with delete files or renamed columns: reading its Parquet files as they are
146
+ * would return deleted rows or miss columns
147
+ */
148
+ export const tableFiles = (version: Version, table: string): DataFile[] | undefined => {
149
+ const entry = version.files.tables[table];
150
+ if (!entry) return undefined;
151
+ if (entry.renamed_columns) throw new FdlError(`${table} has renamed columns`);
152
+ return entry.files.map((f) => {
153
+ if (f.deletes) throw new FdlError(`${table} has deleted rows`);
154
+ if (!safe(f.path))
155
+ throw new FdlError(`${table} names a file outside the repository: ${f.path}`);
156
+ return {
157
+ key: version.config.data_dir + f.path,
158
+ rows: f.rows,
159
+ bytes: f.bytes,
160
+ ...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
161
+ };
162
+ });
163
+ };
164
+
165
+ /**
166
+ * Wraps load so that its result is kept for ttlMs. When a reload fails, the last result is
167
+ * used until the next try, so a ref that cannot be read for a moment does not stop a reader
168
+ */
169
+ export const refreshing = <T>(load: () => Promise<T>, ttlMs: number, now = () => Date.now()) => {
170
+ let kept: { at: number; value: Promise<T> } | null = null;
171
+ return (): Promise<T> => {
172
+ if (kept && now() - kept.at < ttlMs) return kept.value;
173
+ const previous = kept?.value;
174
+ const value = previous ? load().catch(() => previous) : load();
175
+ const entry = { at: now(), value };
176
+ kept = entry;
177
+ value.catch(() => {
178
+ if (kept === entry) kept = null;
179
+ });
180
+ return value;
181
+ };
182
+ };
package/src/store.ts ADDED
@@ -0,0 +1,94 @@
1
+ // Where a repository is read from. Keys are the layout's paths relative to the root
2
+ // (fdl.json, refs/heads/main, versions/<ID>/version.json, data/...)
3
+
4
+ /** A read-only view of a repository's root */
5
+ export type Store = {
6
+ /** The object as text, or null when it is missing */
7
+ text: (key: string) => Promise<string | null>;
8
+ /** The object's size in bytes, or null when it is missing */
9
+ size: (key: string) => Promise<number | null>;
10
+ /** Bytes [offset, offset + length) of the object. Throws when it is missing */
11
+ range: (key: string, offset: number, length: number) => Promise<ArrayBuffer>;
12
+ };
13
+
14
+ export class NotFoundError extends Error {
15
+ constructor(readonly key: string) {
16
+ super(`not found: ${key}`);
17
+ }
18
+ }
19
+
20
+ /** Calls f again once when it fails. Object stores fail now and then for a moment */
21
+ export const retryOnce = <T>(f: () => Promise<T>): Promise<T> => f().catch(() => f());
22
+
23
+ /** The part of an R2 binding (or anything shaped like it) that a reader needs */
24
+ export type R2Like = {
25
+ get: (
26
+ key: string,
27
+ options?: { range?: { offset: number; length: number } },
28
+ ) => Promise<{ text: () => Promise<string>; arrayBuffer: () => Promise<ArrayBuffer> } | null>;
29
+ head: (key: string) => Promise<{ size: number } | null>;
30
+ };
31
+
32
+ /** A repository in an R2 bucket (a Workers binding), under prefix (ending in "/", or "") */
33
+ export const r2Store = (bucket: R2Like, prefix = ""): Store => ({
34
+ text: async (key) => {
35
+ const o = await retryOnce(() => bucket.get(prefix + key));
36
+ return o ? o.text() : null;
37
+ },
38
+ size: async (key) => (await retryOnce(() => bucket.head(prefix + key)))?.size ?? null,
39
+ range: async (key, offset, length) => {
40
+ const o = await retryOnce(() => bucket.get(prefix + key, { range: { offset, length } }));
41
+ if (!o) throw new NotFoundError(prefix + key);
42
+ return o.arrayBuffer();
43
+ },
44
+ });
45
+
46
+ export type HttpOptions = {
47
+ fetch?: typeof fetch;
48
+ /** Sent with every request, for example an Authorization header */
49
+ headers?: Record<string, string>;
50
+ };
51
+
52
+ // HEAD, refs and expired.json change; everything else in a repository never does
53
+ const mutable = (key: string) =>
54
+ key === "HEAD" || key === "refs.json" || key.startsWith("refs/") || key.endsWith("/expired.json");
55
+
56
+ /** A repository served over HTTP at root (ending in "/") */
57
+ export const httpStore = (
58
+ root: string,
59
+ { fetch: f = fetch, headers = {} }: HttpOptions = {},
60
+ ): Store => {
61
+ const request = (key: string, init: RequestInit = {}) =>
62
+ retryOnce(async () => {
63
+ const res = await f(root + key, { ...init, headers: { ...headers, ...init.headers } });
64
+ if (res.status >= 500) throw new Error(`${res.status} ${root + key}`);
65
+ return res;
66
+ });
67
+ const ok = (res: Response, key: string) => {
68
+ if (res.status === 404) return false;
69
+ if (!res.ok) throw new Error(`${res.status} ${root + key}`);
70
+ return true;
71
+ };
72
+ return {
73
+ text: async (key) => {
74
+ const res = await request(key, mutable(key) ? { cache: "no-store" } : {});
75
+ return ok(res, key) ? res.text() : null;
76
+ },
77
+ size: async (key) => {
78
+ const res = await request(key, { method: "HEAD" });
79
+ if (!ok(res, key)) return null;
80
+ const length = res.headers.get("content-length");
81
+ if (length === null) throw new Error(`no content-length for ${root + key}`);
82
+ return Number(length);
83
+ },
84
+ range: async (key, offset, length) => {
85
+ const res = await request(key, {
86
+ headers: { range: `bytes=${offset}-${offset + length - 1}` },
87
+ });
88
+ if (!ok(res, key)) throw new NotFoundError(root + key);
89
+ const body = await res.arrayBuffer();
90
+ // A server that ignores Range sends the whole object
91
+ return res.status === 206 ? body : body.slice(offset, offset + length);
92
+ },
93
+ };
94
+ };