@queria-io/fdl-js 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,8 +1,8 @@
1
1
  # fdl-js
2
2
 
3
- Read DuckLake tables published as [FDL](https://fdl.flo8s.com) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
3
+ Read DuckLake tables published as [FDL](https://fdl.queria.io) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
4
4
 
5
- It reads; it does not write. It reads only tables whose files can be read as they are: a table with deleted rows or renamed columns is refused, since reading its Parquet files without the catalog would return deleted rows or miss columns. Read those with DuckDB.
5
+ Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. It reads only tables whose files can be read as they are: a table with deleted rows or renamed columns is refused, since reading its Parquet files without the catalog would return deleted rows or miss columns. Read those with DuckDB.
6
6
 
7
7
  ## Install
8
8
 
@@ -43,9 +43,21 @@ Each retries a failed request once. A store is three functions (`text`, `size`,
43
43
  - Refs move. `refreshing(() => open(store), 5 * 60_000)` keeps the version for five minutes and keeps the last one when a ref cannot be read for a moment. Pass `{ onError }` to see those failures
44
44
  - Data files never change, so their footers can be kept forever: pass `{ cache: cacheApiFooterCache(caches.default) }` (Workers) or `memoryFooterCache()` to `openParquet`. With `footer_bytes` in `files.json`, an uncached footer is one read
45
45
 
46
+ ## Retention
47
+
48
+ For the host of a repository: `expire(store, { keep })` takes the versions behind the newest `keep` out of retention, as `fdl expire` does, and `collect(store, { graceMs })` deletes the data files no live version reaches once past the grace period, as `fdl gc` does. Both read `files.json` and never open a catalog. Assets are deleted only while no live version's rows hold assets.
49
+
50
+ ```ts
51
+ const store = r2MaintainedStore(env.BUCKET, "towns/");
52
+ await expire(store, { keep: 7 });
53
+ await collect(store, { graceMs: 86_400_000 });
54
+ ```
55
+
56
+ `expire` and `collect` take `{ dryRun: true }` to list the versions or keys without changing anything. Any store with `list` (keys and when they were written), `put` and `delete` works.
57
+
46
58
  ## What it follows
47
59
 
48
- [Reading](https://fdl.flo8s.com/spec/reading) in the specification: `HEAD` and refs are followed, `version.json` must name its own ID and a known `fdl` major version, a version with `expired.json` is refused, and file paths from `files.json` are read under `data_dir`. A version without `files.json` cannot be read with this library.
60
+ [Reading](https://fdl.queria.io/spec/reading) in the specification: `HEAD` and refs are followed, `version.json` must name its own ID and a known `fdl` major version, a version with `expired.json` is refused, and file paths from `files.json` are read under `data_dir`. A version without `files.json` cannot be read with this library.
49
61
 
50
62
  ## Develop
51
63
 
@@ -57,6 +69,6 @@ pnpm check
57
69
  pnpm build # dist/
58
70
  ```
59
71
 
60
- To release, set `version` in `package.json` in a pull request and merge it, then run the Release workflow (Actions → Release → Run workflow). It publishes to npm, tags the commit `v<version>` and makes a GitHub release.
72
+ Releases are made from the commit messages ([Conventional Commits](https://www.conventionalcommits.org/)). Each push to `main` runs the Release workflow: when the commits since the last tag include a `fix:` it publishes a patch version, a `feat:` a minor one, and a breaking change (`!`) a minor one while the version is below 1.0. It publishes to npm, tags the commit `v<version>` and makes a GitHub release. `docs:`, `chore:`, `test:` and the like release nothing.
61
73
 
62
74
  `test/fixture/repo` is a small repository made by the `fdl` command: `FDL=fdl test/make-fixture.sh` remakes it.
package/dist/index.d.ts CHANGED
@@ -1,3 +1,4 @@
1
1
  export * from "./store.js";
2
2
  export * from "./repo.js";
3
3
  export * from "./parquet.js";
4
+ export * from "./maintain.js";
package/dist/index.js CHANGED
@@ -1,4 +1,5 @@
1
1
  export * from "./store.js";
2
2
  export * from "./repo.js";
3
3
  export * from "./parquet.js";
4
+ export * from "./maintain.js";
4
5
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC"}
@@ -0,0 +1,66 @@
1
+ import { type R2Like, type Store } from "./store.js";
2
+ /** A listed object: its key and when it was written */
3
+ export type Placed = {
4
+ key: string;
5
+ uploaded: Date;
6
+ };
7
+ /** A store a host can change: a reader's store, plus listing, writing and deleting */
8
+ export type MaintainedStore = Store & {
9
+ list: (prefix: string) => Promise<Placed[]>;
10
+ put: (key: string, text: string) => Promise<void>;
11
+ delete: (keys: string[]) => Promise<void>;
12
+ };
13
+ /** The part of an R2 binding that maintenance needs, besides what a reader needs */
14
+ export type R2Writable = R2Like & {
15
+ list: (options: {
16
+ prefix: string;
17
+ cursor?: string;
18
+ }) => Promise<{
19
+ objects: {
20
+ key: string;
21
+ uploaded: Date;
22
+ }[];
23
+ truncated: boolean;
24
+ cursor?: string;
25
+ }>;
26
+ put: (key: string, value: string) => Promise<unknown>;
27
+ delete: (keys: string[]) => Promise<void>;
28
+ };
29
+ /**
30
+ * A repository in an R2 bucket that maintenance may change. Reads go through `reader`, which
31
+ * defaults to `r2Store(bucket, prefix)`; pass another one (with a cache, say) for the same place
32
+ */
33
+ export declare const r2MaintainedStore: (bucket: R2Writable, prefix: string, reader?: Store) => MaintainedStore;
34
+ export type ExpireOptions = {
35
+ /** How many of the newest versions on the ref's first-parent line to keep (at least 1) */
36
+ keep: number;
37
+ ref?: string;
38
+ /** Also keep the versions created at or after this time */
39
+ since?: Date;
40
+ now?: Date;
41
+ /** List only. Default false */
42
+ dryRun?: boolean;
43
+ };
44
+ /**
45
+ * Takes the versions behind the newest `keep` on the ref's first-parent line out of retention,
46
+ * as `fdl expire` does. Versions a ref points at, and with `since` those created at or after
47
+ * it, are kept. Stops at the first version already
48
+ * expired. Returns the IDs expired, newest first
49
+ */
50
+ export declare const expire: (store: MaintainedStore, { keep, ref, since, now, dryRun }: ExpireOptions) => Promise<string[]>;
51
+ export type CollectOptions = {
52
+ /** Objects written more recently than this are kept */
53
+ graceMs: number;
54
+ now?: Date;
55
+ /** List only. Default false */
56
+ dryRun?: boolean;
57
+ };
58
+ /**
59
+ * Deletes the data files and assets no live version reaches, written longer ago than the grace
60
+ * period, as `fdl gc` does. These are files only expired versions reached, and files working
61
+ * catalogs wrote and never pushed. Assets are kept while a live version's rows hold assets,
62
+ * since reading those rows needs the catalog. The objects are listed before the versions are
63
+ * read, and the versions are listed again just before deleting, so a version pushed meanwhile
64
+ * keeps what it reaches. Returns the keys deleted
65
+ */
66
+ export declare const collect: (store: MaintainedStore, { graceMs, now, dryRun }: CollectOptions) => Promise<string[]>;
@@ -0,0 +1,177 @@
1
+ // Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
2
+ // take older versions out of retention (as `fdl expire` does) and delete the data files no
3
+ // live version reaches once past the grace period (as `fdl gc` does). Catalogs are never
4
+ // opened: what a version reads comes from its files.json
5
+ import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
6
+ import { r2Store, retryOnce } from "./store.js";
7
+ /** Up to this many keys go in one R2 delete */
8
+ const DELETE_BATCH = 1000;
9
+ const chunks = (items, size) => Array.from({ length: Math.ceil(items.length / size) }, (_, i) => items.slice(i * size, (i + 1) * size));
10
+ /**
11
+ * A repository in an R2 bucket that maintenance may change. Reads go through `reader`, which
12
+ * defaults to `r2Store(bucket, prefix)`; pass another one (with a cache, say) for the same place
13
+ */
14
+ export const r2MaintainedStore = (bucket, prefix, reader = r2Store(bucket, prefix)) => {
15
+ const list = async (key) => {
16
+ const placed = [];
17
+ let cursor;
18
+ for (;;) {
19
+ const found = await retryOnce(() => bucket.list({ prefix: prefix + key, cursor }));
20
+ for (const o of found.objects)
21
+ placed.push({ key: o.key.slice(prefix.length), uploaded: o.uploaded });
22
+ if (!found.truncated)
23
+ return placed;
24
+ cursor = found.cursor;
25
+ }
26
+ };
27
+ return {
28
+ ...reader,
29
+ list,
30
+ put: async (key, text) => {
31
+ await retryOnce(() => bucket.put(prefix + key, text));
32
+ },
33
+ delete: async (keys) => {
34
+ for (const batch of chunks(keys, DELETE_BATCH))
35
+ await retryOnce(() => bucket.delete(batch.map((key) => prefix + key)));
36
+ },
37
+ };
38
+ };
39
+ const record = async (store, id) => {
40
+ const text = await store.text(`versions/${id}/version.json`);
41
+ return text === null ? null : JSON.parse(text);
42
+ };
43
+ const isExpired = async (store, id) => (await store.text(`versions/${id}/expired.json`)) !== null;
44
+ /**
45
+ * The versions some ref points at. A ref that does not hold a version ID is skipped; a failed
46
+ * read is not, since a version a ref points at must not be taken out
47
+ */
48
+ const held = async (store) => {
49
+ const keys = (await store.list("refs/")).map((p) => p.key);
50
+ const targets = await Promise.all(keys.map((key) => resolve(store, key).catch((e) => {
51
+ if (e instanceof FdlError)
52
+ return null;
53
+ throw e;
54
+ })));
55
+ return new Set(targets.filter((id) => id !== null));
56
+ };
57
+ /** Takes one version out of retention: the mark first, then its catalogs and indexes */
58
+ const expireOne = async (store, id, now) => {
59
+ const dir = `versions/${id}/`;
60
+ if (!(await isExpired(store, id)))
61
+ await store.put(`${dir}expired.json`, `${JSON.stringify({ expired_at: now.toISOString(), reason: "retention" })}\n`);
62
+ const indexed = (await store.list(dir))
63
+ .map((p) => p.key)
64
+ .filter((key) => key === `${dir}ducklake.duckdb` ||
65
+ key === `${dir}files.json` ||
66
+ key === `${dir}tree.json` ||
67
+ key.startsWith(`${dir}catalogs/`) ||
68
+ key.startsWith(`${dir}profiles/`));
69
+ await store.delete(indexed);
70
+ };
71
+ /**
72
+ * Takes the versions behind the newest `keep` on the ref's first-parent line out of retention,
73
+ * as `fdl expire` does. Versions a ref points at, and with `since` those created at or after
74
+ * it, are kept. Stops at the first version already
75
+ * expired. Returns the IDs expired, newest first
76
+ */
77
+ export const expire = async (store, { keep, ref = "refs/heads/main", since, now = new Date(), dryRun = false }) => {
78
+ if (keep < 1)
79
+ throw new RangeError(`keep must be at least 1, got ${keep}`);
80
+ const newest = await resolve(store, ref).catch(() => null);
81
+ if (newest === null)
82
+ return [];
83
+ const kept = await held(store);
84
+ const walk = async (id, position) => {
85
+ if (id === undefined || (position > 0 && (await isExpired(store, id))))
86
+ return [];
87
+ const current = await record(store, id);
88
+ if (current === null)
89
+ return [];
90
+ const recent = since !== undefined && new Date(current.created_at) >= since;
91
+ const expired = position >= keep && !recent && !kept.has(id);
92
+ if (expired && !dryRun)
93
+ await expireOne(store, id, now);
94
+ // A parent in any other form is not followed: the ID becomes part of a path
95
+ const parent = current.parents[0];
96
+ const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
97
+ return expired ? [id, ...rest] : rest;
98
+ };
99
+ return walk(newest, 0);
100
+ };
101
+ const SHA256 = /^assets\/[0-9a-f]{64}$/;
102
+ /** A data or delete file as DuckLake names it, under data_dir */
103
+ const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
104
+ /** What a live version reaches: its data files, its tree's assets, and whether rows hold assets */
105
+ const reachedBy = async (store, dataDir, id) => {
106
+ const dir = `versions/${id}/`;
107
+ const current = await record(store, id);
108
+ const name = (path, fallback) => {
109
+ const value = path ?? fallback;
110
+ if (!isFileName(value))
111
+ throw new FdlError(`version ${id} names a file outside it: ${value}`);
112
+ return dir + value;
113
+ };
114
+ const [files, described, tree] = await Promise.all([
115
+ store.text(name(current?.files?.path, "files.json")),
116
+ store.text(name(current?.dataset?.path, "dataset.json")),
117
+ store.text(name(current?.tree?.path, "tree.json")),
118
+ ]);
119
+ if (files === null)
120
+ throw new Error(`version ${id} is live but has no files.json to say what it reads`);
121
+ const list = JSON.parse(files);
122
+ const data = Object.values(list.tables).flatMap((t) => t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])]));
123
+ const fields = ((described ? JSON.parse(described) : {}).tables ?? [])
124
+ .filter((t) => t.kind !== "view")
125
+ .flatMap((t) => t.fields ?? []);
126
+ return {
127
+ keys: [
128
+ ...data.map((path) => dataDir + path),
129
+ ...(tree ? JSON.parse(tree).entries.map((e) => `assets/${e.sha256}`) : []),
130
+ ],
131
+ assetRows: fields.some((f) => f.asset === true),
132
+ };
133
+ };
134
+ /** The versions without expired.json, from one listing of versions/ */
135
+ const liveVersions = async (store) => {
136
+ const keys = new Set((await store.list("versions/")).map((p) => p.key));
137
+ return [...keys]
138
+ .map((key) => key.match(/^versions\/([^/]+)\/version\.json$/)?.[1])
139
+ .filter((id) => isVersionId(id) && !keys.has(`versions/${id}/expired.json`));
140
+ };
141
+ /**
142
+ * Deletes the data files and assets no live version reaches, written longer ago than the grace
143
+ * period, as `fdl gc` does. These are files only expired versions reached, and files working
144
+ * catalogs wrote and never pushed. Assets are kept while a live version's rows hold assets,
145
+ * since reading those rows needs the catalog. The objects are listed before the versions are
146
+ * read, and the versions are listed again just before deleting, so a version pushed meanwhile
147
+ * keeps what it reaches. Returns the keys deleted
148
+ */
149
+ export const collect = async (store, { graceMs, now = new Date(), dryRun = false }) => {
150
+ const { data_dir: dataDir } = await readConfig(store);
151
+ const before = now.getTime() - graceMs;
152
+ const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
153
+ const kept = new Set();
154
+ let assetRows = false;
155
+ const keep = async (ids) => {
156
+ for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
157
+ for (const key of r.keys)
158
+ kept.add(key);
159
+ assetRows ||= r.assetRows;
160
+ }
161
+ };
162
+ const read = await liveVersions(store);
163
+ await keep(read);
164
+ const old = (placed) => placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
165
+ const planned = () => [
166
+ ...old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length))),
167
+ ...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
168
+ ].sort();
169
+ if (dryRun)
170
+ return planned();
171
+ const seen = new Set(read);
172
+ await keep((await liveVersions(store)).filter((id) => !seen.has(id)));
173
+ const deleted = planned();
174
+ await store.delete(deleted);
175
+ return deleted;
176
+ };
177
+ //# sourceMappingURL=maintain.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F,yFAAyF;AACzF,yDAAyD;AACzD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnF,OAAO,EAA2B,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAuBzE,+CAA+C;AAC/C,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,MAAM,MAAM,GAAG,CAAI,KAAU,EAAE,IAAY,EAAS,EAAE,CACpD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9D,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CACtC,CAAC;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,MAAkB,EAClB,MAAc,EACd,SAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EACtB,EAAE;IACnB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAqB,EAAE;QACpD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAA0B,CAAC;QAC/B,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO;gBAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAAE,OAAO,MAAM,CAAC;YACpC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACxB,CAAC;IACH,CAAC,CAAC;IACF,OAAO;QACL,GAAG,MAAM;QACT,IAAI;QACJ,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;YACvB,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;gBAC5C,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAwB,EAAE;IACtE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC7D,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,EAAE,CACnD,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;AAE7D;;;GAGG;AACH,MAAM,IAAI,GAAG,KAAK,EAAE,KAAsB,EAAwB,EAAE;IAClE,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACf,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;QACvC,IAAI,CAAC,YAAY,QAAQ;YAAE,OAAO,IAAI,CAAC;QACvC,MAAM,CAAC,CAAC;IACV,CAAC,CAAC,CACH,CACF,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAsB,EAAE,EAAU,EAAE,GAAS,EAAE,EAAE;IACxE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC,GAAG,CACb,GAAG,GAAG,cAAc,EACpB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,IAAI,CAC9E,CAAC;IACJ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,MAAM,CACL,CAAC,GAAG,EAAE,EAAE,CACN,GAAG,KAAK,GAAG,GAAG,iBAAiB;QAC/B,GAAG,KAAK,GAAG,GAAG,YAAY;QAC1B,GAAG,KAAK,GAAG,GAAG,WAAW;QACzB,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC;QACjC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC,CACpC,CAAC;IACJ,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC,CAAC;AAaF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EACzB,KAAsB,EACtB,EAAE,IAAI,EAAE,GAAG,GAAG,iBAAiB,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAiB,EACtE,EAAE;IACrB,IAAI,IAAI,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,EAAE,EAAsB,EAAE,QAAgB,EAAqB,EAAE;QACjF,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;QAC5E,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,MAAM;YAAE,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACxD,4EAA4E;QAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACzB,CAAC,CAAC;AAQF,MAAM,MAAM,GAAG,wBAAwB,CAAC;AACxC,iEAAiE;AACjE,MAAM,SAAS,GAAG,gCAAgC,CAAC;AAEnD,mGAAmG;AACnG,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,OAAe,EAAE,EAAU,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,IAAwB,EAAE,QAAgB,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,IAAI,QAAQ,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,6BAA6B,KAAK,EAAE,CAAC,CAAC;QAC9F,OAAO,GAAG,GAAG,KAAK,CAAC;IACrB,CAAC,CAAC;IACF,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;KACnD,CAAC,CAAC;IACH,IAAI,KAAK,KAAK,IAAI;QAChB,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,qDAAqD,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACpD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzE,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;SAClF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;SAChC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAClC,OAAO;QACL,IAAI,EAAE;YACJ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrF;QACD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC;KAChD,CAAC;AACJ,CAAC,CAAC;AAEF,uEAAuE;AACvE,MAAM,YAAY,GAAG,KAAK,EAAE,KAAsB,EAAqB,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SAClE,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,CAAC;AAC/F,CAAC,CAAC;AAUF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAsB,EACtB,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAkB,EAC1C,EAAE;IACrB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;QACnC,KAAK,MAAM,CAAC,IAAI,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC;QAC5B,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB;QACE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACvE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;KACpE,CAAC,IAAI,EAAE,CAAC;IACX,IAAI,MAAM;QAAE,OAAO,OAAO,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}
package/dist/repo.d.ts CHANGED
@@ -54,6 +54,10 @@ export type DataFile = {
54
54
  bytes: number;
55
55
  footerBytes?: number;
56
56
  };
57
+ /** Whether a value is a version ID: a lower-case UUIDv7 */
58
+ export declare const isVersionId: (value: unknown) => value is string;
59
+ /** Whether a path in version.json is a plain file name in the version's folder */
60
+ export declare const isFileName: (value: unknown) => value is string;
57
61
  export declare const readConfig: (store: Store) => Promise<Config>;
58
62
  /**
59
63
  * The version ID that rev names: a version ID itself, "HEAD", a full ref ("refs/tags/2025-final")
package/dist/repo.js CHANGED
@@ -1,11 +1,20 @@
1
1
  // Resolve a ref to a version and list the data files of its tables, following
2
- // https://fdl.flo8s.com/spec/reading. Only files.json is read: the catalog is never opened
2
+ // https://fdl.queria.io/spec/reading. Only files.json is read: the catalog is never opened
3
3
  import { NotFoundError } from "./store.js";
4
4
  /** The FDL major versions this reader knows */
5
5
  const KNOWN_MAJOR = "0";
6
6
  export class FdlError extends Error {
7
7
  }
8
- const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
8
+ const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
9
+ /** Whether a value is a version ID: a lower-case UUIDv7 */
10
+ export const isVersionId = (value) => typeof value === "string" && VERSION_ID.test(value);
11
+ /** Whether a path in version.json is a plain file name in the version's folder */
12
+ export const isFileName = (value) => typeof value === "string" &&
13
+ value !== "" &&
14
+ value !== "." &&
15
+ value !== ".." &&
16
+ !value.includes("/") &&
17
+ !value.includes("\\");
9
18
  const required = async (store, key) => {
10
19
  const text = await store.text(key);
11
20
  if (text === null)
@@ -88,7 +97,7 @@ export const readVersion = async (store, id, config) => {
88
97
  throw new FdlError(`${dir}version.json holds ${record.id}`);
89
98
  checkMajor(String(record.fdl), `version ${id}`);
90
99
  const filesPath = record.files?.path ?? "files.json";
91
- if (!safe(filesPath))
100
+ if (!isFileName(filesPath))
92
101
  throw new FdlError(`version ${id} names files.json outside it: ${filesPath}`);
93
102
  const files = await store.text(dir + filesPath);
94
103
  if (files === null)
package/dist/repo.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiCtC,MAAM,UAAU,GAAG,gEAAgE,CAAC;AAEpF,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO;QACL,GAAG,GAAG;QACN,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACpB,QAAQ,EAAE,OAAO,KAAK,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG;KAC5E,CAAC;AACJ,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QAClB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACf,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
1
+ {"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiCtC,MAAM,UAAU,GAAG,uEAAuE,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC7D,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,KAAK,EAAE;IACZ,KAAK,KAAK,GAAG;IACb,KAAK,KAAK,IAAI;IACd,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACpB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExB,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO;QACL,GAAG,GAAG;QACN,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACpB,QAAQ,EAAE,OAAO,KAAK,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG;KAC5E,CAAC;AACJ,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACxB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACf,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@queria-io/fdl-js",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
5
5
  "keywords": [
6
6
  "cloudflare-workers",
@@ -54,6 +54,34 @@
54
54
  "typescript": "^5.7.4",
55
55
  "vitest": "^4.1.0"
56
56
  },
57
+ "release": {
58
+ "branches": [
59
+ "main"
60
+ ],
61
+ "plugins": [
62
+ [
63
+ "@semantic-release/commit-analyzer",
64
+ {
65
+ "releaseRules": [
66
+ {
67
+ "breaking": true,
68
+ "release": "minor"
69
+ }
70
+ ]
71
+ }
72
+ ],
73
+ "@semantic-release/release-notes-generator",
74
+ "@semantic-release/npm",
75
+ [
76
+ "@semantic-release/github",
77
+ {
78
+ "successComment": false,
79
+ "failComment": false,
80
+ "releasedLabels": false
81
+ }
82
+ ]
83
+ ]
84
+ },
57
85
  "engines": {
58
86
  "node": ">=20"
59
87
  },
package/src/index.ts CHANGED
@@ -1,3 +1,4 @@
1
1
  export * from "./store.js";
2
2
  export * from "./repo.js";
3
3
  export * from "./parquet.js";
4
+ export * from "./maintain.js";
@@ -0,0 +1,261 @@
1
+ // Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
2
+ // take older versions out of retention (as `fdl expire` does) and delete the data files no
3
+ // live version reaches once past the grace period (as `fdl gc` does). Catalogs are never
4
+ // opened: what a version reads comes from its files.json
5
+ import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
6
+ import { type R2Like, type Store, r2Store, retryOnce } from "./store.js";
7
+
8
+ /** A listed object: its key and when it was written */
9
+ export type Placed = { key: string; uploaded: Date };
10
+
11
+ /** A store a host can change: a reader's store, plus listing, writing and deleting */
12
+ export type MaintainedStore = Store & {
13
+ list: (prefix: string) => Promise<Placed[]>;
14
+ put: (key: string, text: string) => Promise<void>;
15
+ delete: (keys: string[]) => Promise<void>;
16
+ };
17
+
18
+ /** The part of an R2 binding that maintenance needs, besides what a reader needs */
19
+ export type R2Writable = R2Like & {
20
+ list: (options: { prefix: string; cursor?: string }) => Promise<{
21
+ objects: { key: string; uploaded: Date }[];
22
+ truncated: boolean;
23
+ cursor?: string;
24
+ }>;
25
+ put: (key: string, value: string) => Promise<unknown>;
26
+ delete: (keys: string[]) => Promise<void>;
27
+ };
28
+
29
+ /** Up to this many keys go in one R2 delete */
30
+ const DELETE_BATCH = 1000;
31
+
32
+ const chunks = <T>(items: T[], size: number): T[][] =>
33
+ Array.from({ length: Math.ceil(items.length / size) }, (_, i) =>
34
+ items.slice(i * size, (i + 1) * size),
35
+ );
36
+
37
+ /**
38
+ * A repository in an R2 bucket that maintenance may change. Reads go through `reader`, which
39
+ * defaults to `r2Store(bucket, prefix)`; pass another one (with a cache, say) for the same place
40
+ */
41
+ export const r2MaintainedStore = (
42
+ bucket: R2Writable,
43
+ prefix: string,
44
+ reader: Store = r2Store(bucket, prefix),
45
+ ): MaintainedStore => {
46
+ const list = async (key: string): Promise<Placed[]> => {
47
+ const placed: Placed[] = [];
48
+ let cursor: string | undefined;
49
+ for (;;) {
50
+ const found = await retryOnce(() => bucket.list({ prefix: prefix + key, cursor }));
51
+ for (const o of found.objects)
52
+ placed.push({ key: o.key.slice(prefix.length), uploaded: o.uploaded });
53
+ if (!found.truncated) return placed;
54
+ cursor = found.cursor;
55
+ }
56
+ };
57
+ return {
58
+ ...reader,
59
+ list,
60
+ put: async (key, text) => {
61
+ await retryOnce(() => bucket.put(prefix + key, text));
62
+ },
63
+ delete: async (keys) => {
64
+ for (const batch of chunks(keys, DELETE_BATCH))
65
+ await retryOnce(() => bucket.delete(batch.map((key) => prefix + key)));
66
+ },
67
+ };
68
+ };
69
+
70
+ type Head = {
71
+ id: string;
72
+ parents: string[];
73
+ created_at: string;
74
+ files?: { path: string };
75
+ tree?: { path: string };
76
+ dataset?: { path: string };
77
+ };
78
+
79
+ const record = async (store: Store, id: string): Promise<Head | null> => {
80
+ const text = await store.text(`versions/${id}/version.json`);
81
+ return text === null ? null : (JSON.parse(text) as Head);
82
+ };
83
+
84
+ const isExpired = async (store: Store, id: string) =>
85
+ (await store.text(`versions/${id}/expired.json`)) !== null;
86
+
87
+ /**
88
+ * The versions some ref points at. A ref that does not hold a version ID is skipped; a failed
89
+ * read is not, since a version a ref points at must not be taken out
90
+ */
91
+ const held = async (store: MaintainedStore): Promise<Set<string>> => {
92
+ const keys = (await store.list("refs/")).map((p) => p.key);
93
+ const targets = await Promise.all(
94
+ keys.map((key) =>
95
+ resolve(store, key).catch((e: unknown) => {
96
+ if (e instanceof FdlError) return null;
97
+ throw e;
98
+ }),
99
+ ),
100
+ );
101
+ return new Set(targets.filter((id): id is string => id !== null));
102
+ };
103
+
104
+ /** Takes one version out of retention: the mark first, then its catalogs and indexes */
105
+ const expireOne = async (store: MaintainedStore, id: string, now: Date) => {
106
+ const dir = `versions/${id}/`;
107
+ if (!(await isExpired(store, id)))
108
+ await store.put(
109
+ `${dir}expired.json`,
110
+ `${JSON.stringify({ expired_at: now.toISOString(), reason: "retention" })}\n`,
111
+ );
112
+ const indexed = (await store.list(dir))
113
+ .map((p) => p.key)
114
+ .filter(
115
+ (key) =>
116
+ key === `${dir}ducklake.duckdb` ||
117
+ key === `${dir}files.json` ||
118
+ key === `${dir}tree.json` ||
119
+ key.startsWith(`${dir}catalogs/`) ||
120
+ key.startsWith(`${dir}profiles/`),
121
+ );
122
+ await store.delete(indexed);
123
+ };
124
+
125
+ export type ExpireOptions = {
126
+ /** How many of the newest versions on the ref's first-parent line to keep (at least 1) */
127
+ keep: number;
128
+ ref?: string;
129
+ /** Also keep the versions created at or after this time */
130
+ since?: Date;
131
+ now?: Date;
132
+ /** List only. Default false */
133
+ dryRun?: boolean;
134
+ };
135
+
136
+ /**
137
+ * Takes the versions behind the newest `keep` on the ref's first-parent line out of retention,
138
+ * as `fdl expire` does. Versions a ref points at, and with `since` those created at or after
139
+ * it, are kept. Stops at the first version already
140
+ * expired. Returns the IDs expired, newest first
141
+ */
142
+ export const expire = async (
143
+ store: MaintainedStore,
144
+ { keep, ref = "refs/heads/main", since, now = new Date(), dryRun = false }: ExpireOptions,
145
+ ): Promise<string[]> => {
146
+ if (keep < 1) throw new RangeError(`keep must be at least 1, got ${keep}`);
147
+ const newest = await resolve(store, ref).catch(() => null);
148
+ if (newest === null) return [];
149
+ const kept = await held(store);
150
+ const walk = async (id: string | undefined, position: number): Promise<string[]> => {
151
+ if (id === undefined || (position > 0 && (await isExpired(store, id)))) return [];
152
+ const current = await record(store, id);
153
+ if (current === null) return [];
154
+ const recent = since !== undefined && new Date(current.created_at) >= since;
155
+ const expired = position >= keep && !recent && !kept.has(id);
156
+ if (expired && !dryRun) await expireOne(store, id, now);
157
+ // A parent in any other form is not followed: the ID becomes part of a path
158
+ const parent = current.parents[0];
159
+ const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
160
+ return expired ? [id, ...rest] : rest;
161
+ };
162
+ return walk(newest, 0);
163
+ };
164
+
165
+ type FileList = {
166
+ tables: { [table: string]: { files: { path: string; deletes?: { path: string } }[] } };
167
+ };
168
+ type Described = { tables?: { kind?: string; fields?: { asset?: boolean }[] }[] };
169
+ type Tree = { entries: { sha256: string }[] };
170
+
171
+ const SHA256 = /^assets\/[0-9a-f]{64}$/;
172
+ /** A data or delete file as DuckLake names it, under data_dir */
173
+ const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
174
+
175
+ /** What a live version reaches: its data files, its tree's assets, and whether rows hold assets */
176
+ const reachedBy = async (store: Store, dataDir: string, id: string) => {
177
+ const dir = `versions/${id}/`;
178
+ const current = await record(store, id);
179
+ const name = (path: string | undefined, fallback: string) => {
180
+ const value = path ?? fallback;
181
+ if (!isFileName(value)) throw new FdlError(`version ${id} names a file outside it: ${value}`);
182
+ return dir + value;
183
+ };
184
+ const [files, described, tree] = await Promise.all([
185
+ store.text(name(current?.files?.path, "files.json")),
186
+ store.text(name(current?.dataset?.path, "dataset.json")),
187
+ store.text(name(current?.tree?.path, "tree.json")),
188
+ ]);
189
+ if (files === null)
190
+ throw new Error(`version ${id} is live but has no files.json to say what it reads`);
191
+ const list = JSON.parse(files) as FileList;
192
+ const data = Object.values(list.tables).flatMap((t) =>
193
+ t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])]),
194
+ );
195
+ const fields = ((described ? (JSON.parse(described) as Described) : {}).tables ?? [])
196
+ .filter((t) => t.kind !== "view")
197
+ .flatMap((t) => t.fields ?? []);
198
+ return {
199
+ keys: [
200
+ ...data.map((path) => dataDir + path),
201
+ ...(tree ? (JSON.parse(tree) as Tree).entries.map((e) => `assets/${e.sha256}`) : []),
202
+ ],
203
+ assetRows: fields.some((f) => f.asset === true),
204
+ };
205
+ };
206
+
207
+ /** The versions without expired.json, from one listing of versions/ */
208
+ const liveVersions = async (store: MaintainedStore): Promise<string[]> => {
209
+ const keys = new Set((await store.list("versions/")).map((p) => p.key));
210
+ return [...keys]
211
+ .map((key) => key.match(/^versions\/([^/]+)\/version\.json$/)?.[1])
212
+ .filter((id): id is string => isVersionId(id) && !keys.has(`versions/${id}/expired.json`));
213
+ };
214
+
215
+ export type CollectOptions = {
216
+ /** Objects written more recently than this are kept */
217
+ graceMs: number;
218
+ now?: Date;
219
+ /** List only. Default false */
220
+ dryRun?: boolean;
221
+ };
222
+
223
+ /**
224
+ * Deletes the data files and assets no live version reaches, written longer ago than the grace
225
+ * period, as `fdl gc` does. These are files only expired versions reached, and files working
226
+ * catalogs wrote and never pushed. Assets are kept while a live version's rows hold assets,
227
+ * since reading those rows needs the catalog. The objects are listed before the versions are
228
+ * read, and the versions are listed again just before deleting, so a version pushed meanwhile
229
+ * keeps what it reaches. Returns the keys deleted
230
+ */
231
+ export const collect = async (
232
+ store: MaintainedStore,
233
+ { graceMs, now = new Date(), dryRun = false }: CollectOptions,
234
+ ): Promise<string[]> => {
235
+ const { data_dir: dataDir } = await readConfig(store);
236
+ const before = now.getTime() - graceMs;
237
+ const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
238
+ const kept = new Set<string>();
239
+ let assetRows = false;
240
+ const keep = async (ids: string[]) => {
241
+ for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
242
+ for (const key of r.keys) kept.add(key);
243
+ assetRows ||= r.assetRows;
244
+ }
245
+ };
246
+ const read = await liveVersions(store);
247
+ await keep(read);
248
+ const old = (placed: Placed[]) =>
249
+ placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
250
+ const planned = () =>
251
+ [
252
+ ...old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length))),
253
+ ...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
254
+ ].sort();
255
+ if (dryRun) return planned();
256
+ const seen = new Set(read);
257
+ await keep((await liveVersions(store)).filter((id) => !seen.has(id)));
258
+ const deleted = planned();
259
+ await store.delete(deleted);
260
+ return deleted;
261
+ };
package/src/repo.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  // Resolve a ref to a version and list the data files of its tables, following
2
- // https://fdl.flo8s.com/spec/reading. Only files.json is read: the catalog is never opened
2
+ // https://fdl.queria.io/spec/reading. Only files.json is read: the catalog is never opened
3
3
  import { NotFoundError, type Store } from "./store.js";
4
4
 
5
5
  /** The FDL major versions this reader knows */
@@ -38,7 +38,20 @@ export type Version = { config: Config; record: VersionRecord; files: FileList }
38
38
  /** A data file to read: its key in the store, rows and sizes from files.json */
39
39
  export type DataFile = { key: string; rows: number; bytes: number; footerBytes?: number };
40
40
 
41
- const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
41
+ const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
42
+
43
+ /** Whether a value is a version ID: a lower-case UUIDv7 */
44
+ export const isVersionId = (value: unknown): value is string =>
45
+ typeof value === "string" && VERSION_ID.test(value);
46
+
47
+ /** Whether a path in version.json is a plain file name in the version's folder */
48
+ export const isFileName = (value: unknown): value is string =>
49
+ typeof value === "string" &&
50
+ value !== "" &&
51
+ value !== "." &&
52
+ value !== ".." &&
53
+ !value.includes("/") &&
54
+ !value.includes("\\");
42
55
 
43
56
  const required = async (store: Store, key: string): Promise<string> => {
44
57
  const text = await store.text(key);
@@ -124,7 +137,7 @@ export const readVersion = async (store: Store, id: string, config?: Config): Pr
124
137
  if (record.id !== id) throw new FdlError(`${dir}version.json holds ${record.id}`);
125
138
  checkMajor(String(record.fdl), `version ${id}`);
126
139
  const filesPath = record.files?.path ?? "files.json";
127
- if (!safe(filesPath))
140
+ if (!isFileName(filesPath))
128
141
  throw new FdlError(`version ${id} names files.json outside it: ${filesPath}`);
129
142
  const files = await store.text(dir + filesPath);
130
143
  if (files === null) throw new FdlError(`version ${id} has no files.json; read it with DuckDB`);