@queria-io/fdl-js 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/store.js ADDED
@@ -0,0 +1,69 @@
1
+ // Where a repository is read from. Keys are the layout's paths relative to the root
2
+ // (fdl.json, refs/heads/main, versions/<ID>/version.json, data/...)
3
+ export class NotFoundError extends Error {
4
+ key;
5
+ constructor(key) {
6
+ super(`not found: ${key}`);
7
+ this.key = key;
8
+ }
9
+ }
10
+ /** Calls f again once when it fails. Object stores fail now and then for a moment */
11
+ export const retryOnce = (f) => f().catch(() => f());
12
+ /** A repository in an R2 bucket (a Workers binding), under prefix (ending in "/", or "") */
13
+ export const r2Store = (bucket, prefix = "") => ({
14
+ text: async (key) => {
15
+ const o = await retryOnce(() => bucket.get(prefix + key));
16
+ return o ? o.text() : null;
17
+ },
18
+ size: async (key) => (await retryOnce(() => bucket.head(prefix + key)))?.size ?? null,
19
+ range: async (key, offset, length) => {
20
+ const o = await retryOnce(() => bucket.get(prefix + key, { range: { offset, length } }));
21
+ if (!o)
22
+ throw new NotFoundError(prefix + key);
23
+ return o.arrayBuffer();
24
+ },
25
+ });
26
+ // HEAD, refs and expired.json change; everything else in a repository never does
27
+ const mutable = (key) => key === "HEAD" || key === "refs.json" || key.startsWith("refs/") || key.endsWith("/expired.json");
28
+ /** A repository served over HTTP at root (ending in "/") */
29
+ export const httpStore = (root, { fetch: f = fetch, headers = {} } = {}) => {
30
+ const request = (key, init = {}) => retryOnce(async () => {
31
+ const res = await f(root + key, { ...init, headers: { ...headers, ...init.headers } });
32
+ if (res.status >= 500)
33
+ throw new Error(`${res.status} ${root + key}`);
34
+ return res;
35
+ });
36
+ const ok = (res, key) => {
37
+ if (res.status === 404)
38
+ return false;
39
+ if (!res.ok)
40
+ throw new Error(`${res.status} ${root + key}`);
41
+ return true;
42
+ };
43
+ return {
44
+ text: async (key) => {
45
+ const res = await request(key, mutable(key) ? { cache: "no-store" } : {});
46
+ return ok(res, key) ? res.text() : null;
47
+ },
48
+ size: async (key) => {
49
+ const res = await request(key, { method: "HEAD" });
50
+ if (!ok(res, key))
51
+ return null;
52
+ const length = res.headers.get("content-length");
53
+ if (length === null)
54
+ throw new Error(`no content-length for ${root + key}`);
55
+ return Number(length);
56
+ },
57
+ range: async (key, offset, length) => {
58
+ const res = await request(key, {
59
+ headers: { range: `bytes=${offset}-${offset + length - 1}` },
60
+ });
61
+ if (!ok(res, key))
62
+ throw new NotFoundError(root + key);
63
+ const body = await res.arrayBuffer();
64
+ // A server that ignores Range sends the whole object
65
+ return res.status === 206 ? body : body.slice(offset, offset + length);
66
+ },
67
+ };
68
+ };
69
+ //# sourceMappingURL=store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,oFAAoF;AACpF,oEAAoE;AAYpE,MAAM,OAAO,aAAc,SAAQ,KAAK;IACjB;IAArB,YAAqB,GAAW;QAC9B,KAAK,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;QADR,QAAG,GAAH,GAAG,CAAQ;IAEhC,CAAC;CACF;AAED,qFAAqF;AACrF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAI,CAAmB,EAAc,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;AAWtF,4FAA4F;AAC5F,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,MAAM,GAAG,EAAE,EAAS,EAAE,CAAC,CAAC;IAC9D,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAClB,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7B,CAAC;IACD,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,IAAI;IACrF,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,aAAa,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QAC9C,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACzB,CAAC;CACF,CAAC,CAAC;AAQH,iFAAiF;AACjF,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAC9B,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;AAEpG,4DAA4D;AAC5D,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,IAAY,EACZ,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,EAAE,OAAO,GAAG,EAAE,KAAkB,EAAE,EAC7C,EAAE;IACT,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,OAAoB,EAAE,EAAE,EAAE,CACtD,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,IAAI,GAAG,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACvF,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC;QACtE,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IACL,MAAM,EAAE,GAAG,CAAC,GAAa,EAAE,GAAW,EAAE,EAAE;QACxC,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,KAAK,CAAC;QACrC,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,OAAO;QACL,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAClB,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1E,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1C,CAAC;QACD,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAClB,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YACnD,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACjD,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC;YAC5E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QACD,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;gBAC7B,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,MAAM,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,EAAE;aAC7D,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;gBAAE,MAAM,IAAI,aAAa,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;YACrC,qDAAqD;YACrD,OAAO,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;QACzE,CAAC;KACF,CAAC;AACJ,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,61 @@
1
1
  {
2
2
  "name": "@queria-io/fdl-js",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.2.0",
4
+ "description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
5
+ "keywords": [
6
+ "cloudflare-workers",
7
+ "ducklake",
8
+ "fdl",
9
+ "hyparquet",
10
+ "parquet"
11
+ ],
12
+ "homepage": "https://github.com/queria-io/fdl-js#readme",
13
+ "license": "MIT",
14
+ "repository": {
15
+ "type": "git",
16
+ "url": "git+https://github.com/queria-io/fdl-js.git"
17
+ },
18
+ "files": [
19
+ "dist",
20
+ "src"
21
+ ],
22
+ "type": "module",
23
+ "sideEffects": false,
24
+ "exports": {
25
+ ".": {
26
+ "types": "./dist/index.d.ts",
27
+ "default": "./dist/index.js"
28
+ },
29
+ "./node": {
30
+ "types": "./dist/node.d.ts",
31
+ "default": "./dist/node.js"
32
+ }
33
+ },
34
+ "publishConfig": {
35
+ "access": "public"
36
+ },
37
+ "scripts": {
38
+ "build": "rm -rf dist && tsc -p tsconfig.build.json",
39
+ "prepack": "pnpm build",
40
+ "test": "vitest run",
41
+ "typecheck": "tsc --noEmit",
42
+ "check": "oxlint && oxfmt --check .",
43
+ "fix": "oxlint --fix && oxfmt ."
44
+ },
45
+ "dependencies": {
46
+ "fzstd": "^0.1.1",
47
+ "hyparquet": "^1.32.0"
48
+ },
49
+ "devDependencies": {
50
+ "@cloudflare/workers-types": "^5.20261101.1",
51
+ "@types/node": "^22.0.0",
52
+ "oxfmt": "^0.73.0",
53
+ "oxlint": "^1.76.0",
54
+ "typescript": "^5.7.4",
55
+ "vitest": "^4.1.0"
56
+ },
57
+ "engines": {
58
+ "node": ">=20"
59
+ },
60
+ "packageManager": "pnpm@10.17.1"
61
+ }
package/src/index.ts ADDED
@@ -0,0 +1,3 @@
1
+ export * from "./store.js";
2
+ export * from "./repo.js";
3
+ export * from "./parquet.js";
package/src/node.ts ADDED
@@ -0,0 +1,29 @@
1
+ // A repository in a local folder, for Node and other runtimes with node:fs
2
+ import { open, readFile, stat } from "node:fs/promises";
3
+ import { join } from "node:path";
4
+ import { NotFoundError, type Store } from "./store.js";
5
+
6
+ const missing = (e: unknown) => (e as { code?: string }).code === "ENOENT";
7
+
8
+ /** A repository in the folder dir */
9
+ export const fileStore = (dir: string): Store => ({
10
+ text: (key) =>
11
+ readFile(join(dir, key), "utf8").catch((e) => (missing(e) ? null : Promise.reject(e))),
12
+ size: (key) =>
13
+ stat(join(dir, key)).then(
14
+ (s) => s.size,
15
+ (e) => (missing(e) ? null : Promise.reject(e)),
16
+ ),
17
+ range: async (key, offset, length) => {
18
+ const file = await open(join(dir, key)).catch((e) =>
19
+ Promise.reject(missing(e) ? new NotFoundError(key) : e),
20
+ );
21
+ try {
22
+ const buffer = new Uint8Array(length);
23
+ const { bytesRead } = await file.read(buffer, 0, length, offset);
24
+ return buffer.buffer.slice(0, bytesRead);
25
+ } finally {
26
+ await file.close();
27
+ }
28
+ },
29
+ });
package/src/parquet.ts ADDED
@@ -0,0 +1,204 @@
1
+ // Read a version's Parquet files by range with hyparquet: the footer first (cached, since data
2
+ // files never change), then only the row groups whose min/max can hold the value asked for
3
+ import { decompress } from "fzstd";
4
+ import {
5
+ type AsyncBuffer,
6
+ type Compressors,
7
+ type FileMetaData,
8
+ parquetMetadata,
9
+ parquetReadObjects,
10
+ } from "hyparquet";
11
+ import type { DataFile } from "./repo.js";
12
+ import type { Store } from "./store.js";
13
+
14
+ /** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
15
+ export const compressors: Compressors = { ZSTD: (input) => decompress(input) };
16
+
17
+ /** Called after every read, with the bytes read and the milliseconds spent waiting */
18
+ export type OnRead = (bytes: number, ms: number) => void;
19
+
20
+ /** A data file as hyparquet's AsyncBuffer */
21
+ export const asyncBuffer = (
22
+ store: Store,
23
+ key: string,
24
+ byteLength: number,
25
+ onRead?: OnRead,
26
+ ): AsyncBuffer => ({
27
+ byteLength,
28
+ slice: async (start, end = byteLength) => {
29
+ const t0 = Date.now();
30
+ const body = await store.range(key, start, end - start);
31
+ onRead?.(end - start, Date.now() - t0);
32
+ return body;
33
+ },
34
+ });
35
+
36
+ /** Where footers are kept between requests. Keys are data file keys; files never change */
37
+ export type FooterCache = {
38
+ get: (key: string) => Promise<ArrayBuffer | undefined>;
39
+ put: (key: string, footer: ArrayBuffer) => Promise<void>;
40
+ };
41
+
42
+ /** A FooterCache on a Cache API cache (caches.default in Workers, caches.open() in browsers) */
43
+ export const cacheApiFooterCache = (
44
+ cache: Cache,
45
+ base = "https://fdl.invalid/footer/",
46
+ ): FooterCache => ({
47
+ get: async (key) => (await cache.match(base + encodeURIComponent(key)))?.arrayBuffer(),
48
+ put: (key, footer) =>
49
+ cache.put(
50
+ base + encodeURIComponent(key),
51
+ new Response(footer, {
52
+ headers: {
53
+ "content-type": "application/octet-stream",
54
+ "cache-control": "public, max-age=31536000, immutable",
55
+ },
56
+ }),
57
+ ),
58
+ });
59
+
60
+ /** A FooterCache in memory, for one process or isolate */
61
+ export const memoryFooterCache = (): FooterCache => {
62
+ const kept = new Map<string, ArrayBuffer>();
63
+ return {
64
+ get: async (key) => kept.get(key),
65
+ put: async (key, footer) => void kept.set(key, footer),
66
+ };
67
+ };
68
+
69
+ const MAGIC = 0x31524150; // "PAR1"
70
+
71
+ /** The footer (metadata and the last 8 bytes). One read when files.json gives footer_bytes */
72
+ const readFooter = async (file: AsyncBuffer, footerBytes?: number): Promise<ArrayBuffer> => {
73
+ const size = file.byteLength;
74
+ const guess = footerBytes !== undefined && footerBytes + 8 <= size ? footerBytes + 8 : 8;
75
+ const tail = await file.slice(size - guess, size);
76
+ const view = new DataView(tail);
77
+ if (view.getUint32(guess - 4, true) !== MAGIC)
78
+ throw new Error("not a Parquet file (no PAR1 at the end)");
79
+ const length = view.getUint32(guess - 8, true) + 8;
80
+ if (length > size) throw new Error("Parquet footer is longer than the file");
81
+ return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
82
+ };
83
+
84
+ export type ParquetFile = { key: string; file: AsyncBuffer; metadata: FileMetaData };
85
+
86
+ export type OpenOptions = { cache?: FooterCache; onRead?: OnRead };
87
+
88
+ /** Opens a data file: reads its footer, from the cache when it is there */
89
+ export const openParquet = async (
90
+ store: Store,
91
+ { key, bytes, footerBytes }: DataFile,
92
+ { cache, onRead }: OpenOptions = {},
93
+ ): Promise<ParquetFile> => {
94
+ const file = asyncBuffer(store, key, bytes, onRead);
95
+ // The cache only saves reads: when it fails, the footer is read from the file
96
+ const hit = await cache?.get(key).catch(() => undefined);
97
+ if (hit) return { key, file, metadata: parquetMetadata(hit) };
98
+ const footer = await readFooter(file, footerBytes);
99
+ await cache?.put(key, footer).catch(() => {});
100
+ return { key, file, metadata: parquetMetadata(footer) };
101
+ };
102
+
103
+ export type RowRange = { rowStart: number; rowEnd: number };
104
+ type Value = string | number | bigint;
105
+
106
+ const columnIndex = (metadata: FileMetaData, column: string): number => {
107
+ // schema[0] is the root; leaves follow in column order (flat tables only)
108
+ const i = metadata.schema.findIndex((s, n) => n > 0 && s.name === column) - 1;
109
+ if (i < 0) throw new Error(`no column ${column}`);
110
+ return i;
111
+ };
112
+
113
+ // A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
114
+ // and above) after U+E000-U+FFFF
115
+ const unit = (s: string, i: number) => {
116
+ const u = s.charCodeAt(i);
117
+ return u >= 0xe000 ? u - 0x800 : u >= 0xd800 ? u + 0x2000 : u;
118
+ };
119
+
120
+ // Strings compare in code point order, which is UTF-8 byte order, the order of Parquet's
121
+ // statistics. JavaScript's < compares UTF-16 code units, which differs past U+FFFF
122
+ const atMost = (a: Value, b: Value): boolean => {
123
+ if (typeof a !== "string" || typeof b !== "string") return a <= b;
124
+ for (let i = 0; i < Math.min(a.length, b.length); i++) {
125
+ const d = unit(a, i) - unit(b, i);
126
+ if (d !== 0) return d < 0;
127
+ }
128
+ return a.length <= b.length;
129
+ };
130
+
131
+ const within = (value: Value, min: unknown, max: unknown) =>
132
+ (min === undefined || min === null || atMost(min as Value, value)) &&
133
+ (max === undefined || max === null || atMost(value, max as Value));
134
+
135
+ /**
136
+ * The rows of the row groups whose min/max for column can hold value. Row groups without
137
+ * statistics are kept. Adjacent row groups are merged into one range
138
+ */
139
+ export const rowRangesFor = (metadata: FileMetaData, column: string, value: Value): RowRange[] => {
140
+ const c = columnIndex(metadata, column);
141
+ return metadata.row_groups.reduce<{ start: number; ranges: RowRange[] }>(
142
+ ({ start, ranges }, group) => {
143
+ const end = start + Number(group.num_rows);
144
+ const stats = group.columns[c]?.meta_data?.statistics;
145
+ if (!within(value, stats?.min_value ?? stats?.min, stats?.max_value ?? stats?.max))
146
+ return { start: end, ranges };
147
+ const last = ranges.at(-1);
148
+ return {
149
+ start: end,
150
+ ranges:
151
+ last?.rowEnd === start
152
+ ? [...ranges.slice(0, -1), { rowStart: last.rowStart, rowEnd: end }]
153
+ : [...ranges, { rowStart: start, rowEnd: end }],
154
+ };
155
+ },
156
+ { start: 0, ranges: [] },
157
+ ).ranges;
158
+ };
159
+
160
+ /** Reads the rows given (all of them by default), the columns given (all by default) */
161
+ export const readRows = <T = Record<string, unknown>>(
162
+ { file, metadata }: ParquetFile,
163
+ { columns, ...range }: Partial<RowRange> & { columns?: string[] } = {},
164
+ ): Promise<T[]> => {
165
+ const names = new Set(metadata.schema.slice(1).map((s) => s.name));
166
+ return parquetReadObjects({
167
+ file,
168
+ metadata,
169
+ compressors,
170
+ ...range,
171
+ // Columns the file lacks are left out, so a reader can ask for a column added later
172
+ ...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
173
+ }) as Promise<T[]>;
174
+ };
175
+
176
+ // INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
177
+ // by value, and unlike BigInt() it does not throw for 1.5
178
+ const same = (a: unknown, b: Value) =>
179
+ a === b ||
180
+ ((typeof a === "bigint" || typeof b === "bigint") &&
181
+ (typeof a === "bigint" || typeof a === "number") &&
182
+ typeof b !== "string" &&
183
+ // oxlint-disable-next-line eqeqeq
184
+ a == b);
185
+
186
+ /**
187
+ * The rows where column equals value. Reads only the row groups that can hold it, then drops
188
+ * the other rows. Works best on files sorted by column
189
+ */
190
+ export const readWhere = async <T = Record<string, unknown>>(
191
+ parquet: ParquetFile,
192
+ column: string,
193
+ value: Value,
194
+ columns?: string[],
195
+ ): Promise<T[]> => {
196
+ const ranges = rowRangesFor(parquet.metadata, column, value);
197
+ const wanted = columns && [column, ...columns.filter((c) => c !== column)];
198
+ const rows = await Promise.all(
199
+ ranges.map((r) =>
200
+ readRows<Record<string, unknown>>(parquet, { ...r, ...(wanted ? { columns: wanted } : {}) }),
201
+ ),
202
+ );
203
+ return rows.flat().filter((row) => same(row[column], value)) as T[];
204
+ };
package/src/repo.ts ADDED
@@ -0,0 +1,198 @@
1
+ // Resolve a ref to a version and list the data files of its tables, following
2
+ // https://fdl.flo8s.com/spec/reading. Only files.json is read: the catalog is never opened
3
+ import { NotFoundError, type Store } from "./store.js";
4
+
5
+ /** The FDL major versions this reader knows */
6
+ const KNOWN_MAJOR = "0";
7
+
8
+ export class FdlError extends Error {}
9
+
10
+ export type Config = { fdl: string; id?: string; data_dir: string };
11
+
12
+ export type VersionRecord = {
13
+ fdl: string;
14
+ id: string;
15
+ seq: number;
16
+ parents: string[];
17
+ created_at: string;
18
+ author: { name: string; [k: string]: unknown };
19
+ message: string;
20
+ files?: { path: string; bytes: number; sha256: string };
21
+ [k: string]: unknown;
22
+ };
23
+
24
+ export type FileEntry = {
25
+ path: string;
26
+ rows: number;
27
+ bytes: number;
28
+ footer_bytes?: number;
29
+ deletes?: { path: string; rows: number; bytes: number };
30
+ };
31
+ export type FileList = {
32
+ tables: Record<string, { files: FileEntry[]; renamed_columns?: boolean }>;
33
+ };
34
+
35
+ /** One version of a repository, with its files.json */
36
+ export type Version = { config: Config; record: VersionRecord; files: FileList };
37
+
38
+ /** A data file to read: its key in the store, rows and sizes from files.json */
39
+ export type DataFile = { key: string; rows: number; bytes: number; footerBytes?: number };
40
+
41
+ const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
42
+
43
+ const required = async (store: Store, key: string): Promise<string> => {
44
+ const text = await store.text(key);
45
+ if (text === null) throw new NotFoundError(key);
46
+ return text;
47
+ };
48
+
49
+ const json = async <T>(store: Store, key: string): Promise<T> =>
50
+ JSON.parse(await required(store, key)) as T;
51
+
52
+ const checkMajor = (fdl: string, what: string) => {
53
+ if (fdl.split(".")[0] !== KNOWN_MAJOR)
54
+ throw new FdlError(`${what} is FDL ${fdl}, unknown to this reader`);
55
+ };
56
+
57
+ // A relative path that stays under the root. Dot segments are also caught percent-encoded,
58
+ // since URLs resolve %2e%2e as ..
59
+ const safe = (path: string) =>
60
+ !path.startsWith("/") &&
61
+ !path.includes("://") &&
62
+ !path.replace(/%2e/gi, ".").split(/[/\\]/).includes("..");
63
+
64
+ export const readConfig = async (store: Store): Promise<Config> => {
65
+ const raw = await json<Partial<Config>>(store, "fdl.json");
66
+ checkMajor(String(raw.fdl), "the repository");
67
+ const dataDir = raw.data_dir ?? "data/";
68
+ if (!safe(dataDir)) throw new FdlError(`data_dir is outside the repository: ${dataDir}`);
69
+ return {
70
+ ...raw,
71
+ fdl: String(raw.fdl),
72
+ data_dir: dataDir === "" || dataDir.endsWith("/") ? dataDir : `${dataDir}/`,
73
+ };
74
+ };
75
+
76
+ /** Candidates for a short name, in Git's order (refs/<name>, refs/tags/<name>, refs/heads/<name>) */
77
+ const candidates = (rev: string): string[] =>
78
+ rev === "HEAD" || rev.startsWith("refs/")
79
+ ? [rev]
80
+ : [`refs/${rev}`, `refs/tags/${rev}`, `refs/heads/${rev}`];
81
+
82
+ /**
83
+ * The version ID that rev names: a version ID itself, "HEAD", a full ref ("refs/tags/2025-final")
84
+ * or a short one ("main", "2025-final"). Symbolic refs ("ref: refs/heads/main") are followed.
85
+ * Refs change, so callers decide how long to keep the answer
86
+ */
87
+ export const resolve = async (store: Store, rev = "HEAD"): Promise<string> => {
88
+ if (VERSION_ID.test(rev)) return rev;
89
+ if (!safe(rev)) throw new FdlError(`no ref ${rev}`);
90
+ const follow = async (key: string, depth: number): Promise<string | null> => {
91
+ const text = await store.text(key);
92
+ if (text === null) return null;
93
+ const value = text.trim();
94
+ if (!value.startsWith("ref: ")) {
95
+ if (!VERSION_ID.test(value)) throw new FdlError(`${key} does not hold a version ID`);
96
+ return value;
97
+ }
98
+ if (depth > 4) throw new FdlError(`${key}: too many symbolic refs`);
99
+ const target = value.slice(5).trim();
100
+ if (!target.startsWith("refs/") || !safe(target))
101
+ throw new FdlError(`${key} names a ref outside refs/: ${target}`);
102
+ return follow(target, depth + 1);
103
+ };
104
+ const found = await candidates(rev).reduce<Promise<string | null>>(
105
+ async (prev, key) => (await prev) ?? follow(key, 0),
106
+ Promise.resolve(null),
107
+ );
108
+ if (!found) throw new FdlError(`no ref ${rev}`);
109
+ return found;
110
+ };
111
+
112
+ /** Reads a version: checks its record and expiry, and reads its files.json */
113
+ export const readVersion = async (store: Store, id: string, config?: Config): Promise<Version> => {
114
+ const dir = `versions/${id}/`;
115
+ const [cfg, text, expired] = await Promise.all([
116
+ config ?? readConfig(store),
117
+ store.text(dir + "version.json"),
118
+ store.text(dir + "expired.json"),
119
+ ]);
120
+ // A withdrawn version may have lost its version.json, so expiry is checked first
121
+ if (expired !== null) throw new FdlError(`version ${id} is past retention or withdrawn`);
122
+ if (text === null) throw new NotFoundError(dir + "version.json");
123
+ const record = JSON.parse(text) as VersionRecord;
124
+ if (record.id !== id) throw new FdlError(`${dir}version.json holds ${record.id}`);
125
+ checkMajor(String(record.fdl), `version ${id}`);
126
+ const filesPath = record.files?.path ?? "files.json";
127
+ if (!safe(filesPath))
128
+ throw new FdlError(`version ${id} names files.json outside it: ${filesPath}`);
129
+ const files = await store.text(dir + filesPath);
130
+ if (files === null) throw new FdlError(`version ${id} has no files.json; read it with DuckDB`);
131
+ return { config: cfg, record, files: JSON.parse(files) as FileList };
132
+ };
133
+
134
+ /** Resolves rev and reads the version it names */
135
+ export const open = async (store: Store, rev = "HEAD"): Promise<Version> => {
136
+ const [config, id] = await Promise.all([readConfig(store), resolve(store, rev)]);
137
+ return readVersion(store, id, config);
138
+ };
139
+
140
+ /** The names (schema.table) of the version's tables */
141
+ export const tableNames = (version: Version): string[] => Object.keys(version.files.tables);
142
+
143
+ /**
144
+ * The data files of a table (schema.table), or undefined when the version has no such table.
145
+ * Throws for a table with delete files or renamed columns: reading its Parquet files as they are
146
+ * would return deleted rows or miss columns
147
+ */
148
+ export const tableFiles = (version: Version, table: string): DataFile[] | undefined => {
149
+ const entry = version.files.tables[table];
150
+ if (!entry) return undefined;
151
+ if (entry.renamed_columns) throw new FdlError(`${table} has renamed columns`);
152
+ return entry.files.map((f) => {
153
+ if (f.deletes) throw new FdlError(`${table} has deleted rows`);
154
+ if (!safe(f.path))
155
+ throw new FdlError(`${table} names a file outside the repository: ${f.path}`);
156
+ return {
157
+ key: version.config.data_dir + f.path,
158
+ rows: f.rows,
159
+ bytes: f.bytes,
160
+ ...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
161
+ };
162
+ });
163
+ };
164
+
165
+ export type RefreshingOptions = {
166
+ /** Called when a reload fails and the last result is kept, so the failure can be logged */
167
+ onError?: (error: unknown) => void;
168
+ now?: () => number;
169
+ };
170
+
171
+ /**
172
+ * Wraps load so that its result is kept for ttlMs. When a reload fails, the last result is
173
+ * used until the next try, so a ref that cannot be read for a moment does not stop a reader.
174
+ * The failure goes to onError; with no last result, the returned promise rejects instead
175
+ */
176
+ export const refreshing = <T>(
177
+ load: () => Promise<T>,
178
+ ttlMs: number,
179
+ { onError, now = () => Date.now() }: RefreshingOptions = {},
180
+ ) => {
181
+ let kept: { at: number; value: Promise<T> } | null = null;
182
+ return (): Promise<T> => {
183
+ if (kept && now() - kept.at < ttlMs) return kept.value;
184
+ const previous = kept?.value;
185
+ const value = previous
186
+ ? load().catch((error: unknown) => {
187
+ onError?.(error);
188
+ return previous;
189
+ })
190
+ : load();
191
+ const entry = { at: now(), value };
192
+ kept = entry;
193
+ value.catch(() => {
194
+ if (kept === entry) kept = null;
195
+ });
196
+ return value;
197
+ };
198
+ };
package/src/store.ts ADDED
@@ -0,0 +1,94 @@
1
+ // Where a repository is read from. Keys are the layout's paths relative to the root
2
+ // (fdl.json, refs/heads/main, versions/<ID>/version.json, data/...)
3
+
4
+ /** A read-only view of a repository's root */
5
+ export type Store = {
6
+ /** The object as text, or null when it is missing */
7
+ text: (key: string) => Promise<string | null>;
8
+ /** The object's size in bytes, or null when it is missing */
9
+ size: (key: string) => Promise<number | null>;
10
+ /** Bytes [offset, offset + length) of the object. Throws when it is missing */
11
+ range: (key: string, offset: number, length: number) => Promise<ArrayBuffer>;
12
+ };
13
+
14
+ export class NotFoundError extends Error {
15
+ constructor(readonly key: string) {
16
+ super(`not found: ${key}`);
17
+ }
18
+ }
19
+
20
+ /** Calls f again once when it fails. Object stores fail now and then for a moment */
21
+ export const retryOnce = <T>(f: () => Promise<T>): Promise<T> => f().catch(() => f());
22
+
23
+ /** The part of an R2 binding (or anything shaped like it) that a reader needs */
24
+ export type R2Like = {
25
+ get: (
26
+ key: string,
27
+ options?: { range?: { offset: number; length: number } },
28
+ ) => Promise<{ text: () => Promise<string>; arrayBuffer: () => Promise<ArrayBuffer> } | null>;
29
+ head: (key: string) => Promise<{ size: number } | null>;
30
+ };
31
+
32
+ /** A repository in an R2 bucket (a Workers binding), under prefix (ending in "/", or "") */
33
+ export const r2Store = (bucket: R2Like, prefix = ""): Store => ({
34
+ text: async (key) => {
35
+ const o = await retryOnce(() => bucket.get(prefix + key));
36
+ return o ? o.text() : null;
37
+ },
38
+ size: async (key) => (await retryOnce(() => bucket.head(prefix + key)))?.size ?? null,
39
+ range: async (key, offset, length) => {
40
+ const o = await retryOnce(() => bucket.get(prefix + key, { range: { offset, length } }));
41
+ if (!o) throw new NotFoundError(prefix + key);
42
+ return o.arrayBuffer();
43
+ },
44
+ });
45
+
46
+ export type HttpOptions = {
47
+ fetch?: typeof fetch;
48
+ /** Sent with every request, for example an Authorization header */
49
+ headers?: Record<string, string>;
50
+ };
51
+
52
+ // HEAD, refs and expired.json change; everything else in a repository never does
53
+ const mutable = (key: string) =>
54
+ key === "HEAD" || key === "refs.json" || key.startsWith("refs/") || key.endsWith("/expired.json");
55
+
56
+ /** A repository served over HTTP at root (ending in "/") */
57
+ export const httpStore = (
58
+ root: string,
59
+ { fetch: f = fetch, headers = {} }: HttpOptions = {},
60
+ ): Store => {
61
+ const request = (key: string, init: RequestInit = {}) =>
62
+ retryOnce(async () => {
63
+ const res = await f(root + key, { ...init, headers: { ...headers, ...init.headers } });
64
+ if (res.status >= 500) throw new Error(`${res.status} ${root + key}`);
65
+ return res;
66
+ });
67
+ const ok = (res: Response, key: string) => {
68
+ if (res.status === 404) return false;
69
+ if (!res.ok) throw new Error(`${res.status} ${root + key}`);
70
+ return true;
71
+ };
72
+ return {
73
+ text: async (key) => {
74
+ const res = await request(key, mutable(key) ? { cache: "no-store" } : {});
75
+ return ok(res, key) ? res.text() : null;
76
+ },
77
+ size: async (key) => {
78
+ const res = await request(key, { method: "HEAD" });
79
+ if (!ok(res, key)) return null;
80
+ const length = res.headers.get("content-length");
81
+ if (length === null) throw new Error(`no content-length for ${root + key}`);
82
+ return Number(length);
83
+ },
84
+ range: async (key, offset, length) => {
85
+ const res = await request(key, {
86
+ headers: { range: `bytes=${offset}-${offset + length - 1}` },
87
+ });
88
+ if (!ok(res, key)) throw new NotFoundError(root + key);
89
+ const body = await res.arrayBuffer();
90
+ // A server that ignores Range sends the whole object
91
+ return res.status === 206 ? body : body.slice(offset, offset + length);
92
+ },
93
+ };
94
+ };