@queria-io/fdl-js 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/maintain.d.ts +68 -0
- package/dist/maintain.js +190 -0
- package/dist/maintain.js.map +1 -0
- package/dist/repo.d.ts +4 -0
- package/dist/repo.js +12 -3
- package/dist/repo.js.map +1 -1
- package/package.json +40 -1
- package/src/index.ts +1 -0
- package/src/maintain.ts +276 -0
- package/src/repo.ts +16 -3
package/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# fdl-js
|
|
2
2
|
|
|
3
|
-
Read DuckLake tables published as [FDL](https://fdl.
|
|
3
|
+
Read DuckLake tables published as [FDL](https://fdl.queria.io) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. It reads only tables whose files can be read as they are: a table with deleted rows or renamed columns is refused, since reading its Parquet files without the catalog would return deleted rows or miss columns. Read those with DuckDB.
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
@@ -43,9 +43,21 @@ Each retries a failed request once. A store is three functions (`text`, `size`,
|
|
|
43
43
|
- Refs move. `refreshing(() => open(store), 5 * 60_000)` keeps the version for five minutes and keeps the last one when a ref cannot be read for a moment. Pass `{ onError }` to see those failures
|
|
44
44
|
- Data files never change, so their footers can be kept forever: pass `{ cache: cacheApiFooterCache(caches.default) }` (Workers) or `memoryFooterCache()` to `openParquet`. With `footer_bytes` in `files.json`, an uncached footer is one read
|
|
45
45
|
|
|
46
|
+
## Retention
|
|
47
|
+
|
|
48
|
+
For the host of a repository: `expire(store, { keep })` takes the versions behind the newest `keep` out of retention, as `fdl expire` does, and `collect(store, { graceMs })` deletes the data files no live version reaches once past the grace period, as `fdl gc` does. Both read `files.json` and never open a catalog. Assets are deleted only while no live version's rows hold assets.
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
const store = r2MaintainedStore(env.BUCKET, "towns/");
|
|
52
|
+
await expire(store, { keep: 7 });
|
|
53
|
+
await collect(store, { graceMs: 86_400_000 });
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`expire` and `collect` take `{ dryRun: true }` to list the versions or keys without changing anything. Any store with `list` (keys and when they were written), `put` and `delete` works.
|
|
57
|
+
|
|
46
58
|
## What it follows
|
|
47
59
|
|
|
48
|
-
[Reading](https://fdl.
|
|
60
|
+
[Reading](https://fdl.queria.io/spec/reading) in the specification: `HEAD` and refs are followed, `version.json` must name its own ID and a known `fdl` major version, a version with `expired.json` is refused, and file paths from `files.json` are read under `data_dir`. A version without `files.json` cannot be read with this library.
|
|
49
61
|
|
|
50
62
|
## Develop
|
|
51
63
|
|
|
@@ -57,6 +69,6 @@ pnpm check
|
|
|
57
69
|
pnpm build # dist/
|
|
58
70
|
```
|
|
59
71
|
|
|
60
|
-
|
|
72
|
+
Releases are made from the commit messages ([Conventional Commits](https://www.conventionalcommits.org/)). Each push to `main` runs the Release workflow: when the commits since the last tag include a `fix:` it publishes a patch version, a `feat:` a minor one, and a breaking change (`!` after the type, or a `BREAKING CHANGE:` footer) a minor one. It publishes to npm, tags the commit `v<version>` and makes a GitHub release. `docs:`, `chore:`, `test:` and the like release nothing.
|
|
61
73
|
|
|
62
74
|
`test/fixture/repo` is a small repository made by the `fdl` command: `FDL=fdl test/make-fixture.sh` remakes it.
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { type R2Like, type Store } from "./store.js";
|
|
2
|
+
/** A listed object: its key and when it was written */
|
|
3
|
+
export type Placed = {
|
|
4
|
+
key: string;
|
|
5
|
+
uploaded: Date;
|
|
6
|
+
};
|
|
7
|
+
/** A store a host can change: a reader's store, plus listing, writing and deleting */
|
|
8
|
+
export type MaintainedStore = Store & {
|
|
9
|
+
list: (prefix: string) => Promise<Placed[]>;
|
|
10
|
+
put: (key: string, text: string) => Promise<void>;
|
|
11
|
+
delete: (keys: string[]) => Promise<void>;
|
|
12
|
+
};
|
|
13
|
+
/** The part of an R2 binding that maintenance needs, besides what a reader needs */
|
|
14
|
+
export type R2Writable = R2Like & {
|
|
15
|
+
list: (options: {
|
|
16
|
+
prefix: string;
|
|
17
|
+
cursor?: string;
|
|
18
|
+
}) => Promise<{
|
|
19
|
+
objects: {
|
|
20
|
+
key: string;
|
|
21
|
+
uploaded: Date;
|
|
22
|
+
}[];
|
|
23
|
+
truncated: boolean;
|
|
24
|
+
cursor?: string;
|
|
25
|
+
}>;
|
|
26
|
+
put: (key: string, value: string) => Promise<unknown>;
|
|
27
|
+
delete: (keys: string[]) => Promise<void>;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* A repository in an R2 bucket that maintenance may change. Reads go through `reader`, which
|
|
31
|
+
* defaults to `r2Store(bucket, prefix)`; pass another one (with a cache, say) for the same place
|
|
32
|
+
*/
|
|
33
|
+
export declare const r2MaintainedStore: (bucket: R2Writable, prefix: string, reader?: Store) => MaintainedStore;
|
|
34
|
+
export type ExpireOptions = {
|
|
35
|
+
/** How many of the newest versions on the ref's first-parent line to keep (at least 1) */
|
|
36
|
+
keep: number;
|
|
37
|
+
ref?: string;
|
|
38
|
+
/** Also keep the versions created at or after this time */
|
|
39
|
+
since?: Date;
|
|
40
|
+
now?: Date;
|
|
41
|
+
/** List only. Default false */
|
|
42
|
+
dryRun?: boolean;
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* Takes the versions behind the newest `keep` on the ref's first-parent line out of retention,
|
|
46
|
+
* as `fdl expire` does. Versions a ref points at, and with `since` those created at or after
|
|
47
|
+
* it, are kept. Stops at the first version already
|
|
48
|
+
* expired. Returns the IDs expired, newest first
|
|
49
|
+
*/
|
|
50
|
+
export declare const expire: (store: MaintainedStore, { keep, ref, since, now, dryRun }: ExpireOptions) => Promise<string[]>;
|
|
51
|
+
export type CollectOptions = {
|
|
52
|
+
/** Objects written more recently than this are kept */
|
|
53
|
+
graceMs: number;
|
|
54
|
+
now?: Date;
|
|
55
|
+
/** List only. Default false */
|
|
56
|
+
dryRun?: boolean;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
60
|
+
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
61
|
+
* catalogs wrote and never pushed. A version reaches every data file its catalog names, listed
|
|
62
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
63
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
64
|
+
* since reading those rows needs the catalog. The objects are listed before the versions are
|
|
65
|
+
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
66
|
+
* keeps what it reaches. Returns the keys deleted
|
|
67
|
+
*/
|
|
68
|
+
export declare const collect: (store: MaintainedStore, { graceMs, now, dryRun }: CollectOptions) => Promise<string[]>;
|
package/dist/maintain.js
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
|
+
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
|
+
// live version reaches once past the grace period (as `fdl gc` does). Catalogs are never
|
|
4
|
+
// opened: what a version reaches comes from `named` in its files.json
|
|
5
|
+
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
|
+
import { r2Store, retryOnce } from "./store.js";
|
|
7
|
+
/** Up to this many keys go in one R2 delete */
|
|
8
|
+
const DELETE_BATCH = 1000;
|
|
9
|
+
const chunks = (items, size) => Array.from({ length: Math.ceil(items.length / size) }, (_, i) => items.slice(i * size, (i + 1) * size));
|
|
10
|
+
/**
|
|
11
|
+
* A repository in an R2 bucket that maintenance may change. Reads go through `reader`, which
|
|
12
|
+
* defaults to `r2Store(bucket, prefix)`; pass another one (with a cache, say) for the same place
|
|
13
|
+
*/
|
|
14
|
+
export const r2MaintainedStore = (bucket, prefix, reader = r2Store(bucket, prefix)) => {
|
|
15
|
+
const list = async (key) => {
|
|
16
|
+
const placed = [];
|
|
17
|
+
let cursor;
|
|
18
|
+
for (;;) {
|
|
19
|
+
const found = await retryOnce(() => bucket.list({ prefix: prefix + key, cursor }));
|
|
20
|
+
for (const o of found.objects)
|
|
21
|
+
placed.push({ key: o.key.slice(prefix.length), uploaded: o.uploaded });
|
|
22
|
+
if (!found.truncated)
|
|
23
|
+
return placed;
|
|
24
|
+
cursor = found.cursor;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
return {
|
|
28
|
+
...reader,
|
|
29
|
+
list,
|
|
30
|
+
put: async (key, text) => {
|
|
31
|
+
await retryOnce(() => bucket.put(prefix + key, text));
|
|
32
|
+
},
|
|
33
|
+
delete: async (keys) => {
|
|
34
|
+
for (const batch of chunks(keys, DELETE_BATCH))
|
|
35
|
+
await retryOnce(() => bucket.delete(batch.map((key) => prefix + key)));
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
const record = async (store, id) => {
|
|
40
|
+
const text = await store.text(`versions/${id}/version.json`);
|
|
41
|
+
return text === null ? null : JSON.parse(text);
|
|
42
|
+
};
|
|
43
|
+
const isExpired = async (store, id) => (await store.text(`versions/${id}/expired.json`)) !== null;
|
|
44
|
+
/**
|
|
45
|
+
* The versions some ref points at. A ref that does not hold a version ID is skipped; a failed
|
|
46
|
+
* read is not, since a version a ref points at must not be taken out
|
|
47
|
+
*/
|
|
48
|
+
const held = async (store) => {
|
|
49
|
+
const keys = (await store.list("refs/")).map((p) => p.key);
|
|
50
|
+
const targets = await Promise.all(keys.map((key) => resolve(store, key).catch((e) => {
|
|
51
|
+
if (e instanceof FdlError)
|
|
52
|
+
return null;
|
|
53
|
+
throw e;
|
|
54
|
+
})));
|
|
55
|
+
return new Set(targets.filter((id) => id !== null));
|
|
56
|
+
};
|
|
57
|
+
/** Takes one version out of retention: the mark first, then its catalogs and indexes */
|
|
58
|
+
const expireOne = async (store, id, now) => {
|
|
59
|
+
const dir = `versions/${id}/`;
|
|
60
|
+
if (!(await isExpired(store, id)))
|
|
61
|
+
await store.put(`${dir}expired.json`, `${JSON.stringify({ expired_at: now.toISOString(), reason: "retention" })}\n`);
|
|
62
|
+
const indexed = (await store.list(dir))
|
|
63
|
+
.map((p) => p.key)
|
|
64
|
+
.filter((key) => key === `${dir}ducklake.duckdb` ||
|
|
65
|
+
key === `${dir}files.json` ||
|
|
66
|
+
key === `${dir}tree.json` ||
|
|
67
|
+
key.startsWith(`${dir}catalogs/`) ||
|
|
68
|
+
key.startsWith(`${dir}profiles/`));
|
|
69
|
+
await store.delete(indexed);
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* Takes the versions behind the newest `keep` on the ref's first-parent line out of retention,
|
|
73
|
+
* as `fdl expire` does. Versions a ref points at, and with `since` those created at or after
|
|
74
|
+
* it, are kept. Stops at the first version already
|
|
75
|
+
* expired. Returns the IDs expired, newest first
|
|
76
|
+
*/
|
|
77
|
+
export const expire = async (store, { keep, ref = "refs/heads/main", since, now = new Date(), dryRun = false }) => {
|
|
78
|
+
if (keep < 1)
|
|
79
|
+
throw new RangeError(`keep must be at least 1, got ${keep}`);
|
|
80
|
+
const newest = await resolve(store, ref).catch(() => null);
|
|
81
|
+
if (newest === null)
|
|
82
|
+
return [];
|
|
83
|
+
const kept = await held(store);
|
|
84
|
+
const walk = async (id, position) => {
|
|
85
|
+
if (id === undefined || (position > 0 && (await isExpired(store, id))))
|
|
86
|
+
return [];
|
|
87
|
+
const current = await record(store, id);
|
|
88
|
+
if (current === null)
|
|
89
|
+
return [];
|
|
90
|
+
const recent = since !== undefined && new Date(current.created_at) >= since;
|
|
91
|
+
const expired = position >= keep && !recent && !kept.has(id);
|
|
92
|
+
if (expired && !dryRun)
|
|
93
|
+
await expireOne(store, id, now);
|
|
94
|
+
// A parent in any other form is not followed: the ID becomes part of a path
|
|
95
|
+
const parent = current.parents[0];
|
|
96
|
+
const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
|
|
97
|
+
return expired ? [id, ...rest] : rest;
|
|
98
|
+
};
|
|
99
|
+
return walk(newest, 0);
|
|
100
|
+
};
|
|
101
|
+
const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
102
|
+
/** A data or delete file as DuckLake names it, under data_dir */
|
|
103
|
+
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
104
|
+
/**
|
|
105
|
+
* What a live version reaches: the data files its catalog names, its tree's assets, whether rows
|
|
106
|
+
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
107
|
+
* files in `tables` are only those at the version's snapshot, and the catalog may name more
|
|
108
|
+
*/
|
|
109
|
+
const reachedBy = async (store, dataDir, id) => {
|
|
110
|
+
const dir = `versions/${id}/`;
|
|
111
|
+
const current = await record(store, id);
|
|
112
|
+
const name = (path, fallback) => {
|
|
113
|
+
const value = path ?? fallback;
|
|
114
|
+
if (!isFileName(value))
|
|
115
|
+
throw new FdlError(`version ${id} names a file outside it: ${value}`);
|
|
116
|
+
return dir + value;
|
|
117
|
+
};
|
|
118
|
+
const [files, described, tree] = await Promise.all([
|
|
119
|
+
store.text(name(current?.files?.path, "files.json")),
|
|
120
|
+
store.text(name(current?.dataset?.path, "dataset.json")),
|
|
121
|
+
store.text(name(current?.tree?.path, "tree.json")),
|
|
122
|
+
]);
|
|
123
|
+
if (files === null)
|
|
124
|
+
throw new Error(`version ${id} is live but has no files.json to say what it reads`);
|
|
125
|
+
const list = JSON.parse(files);
|
|
126
|
+
const data = [
|
|
127
|
+
...(list.named ?? []),
|
|
128
|
+
...Object.values(list.tables).flatMap((t) => t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])])),
|
|
129
|
+
];
|
|
130
|
+
const fields = ((described ? JSON.parse(described) : {}).tables ?? [])
|
|
131
|
+
.filter((t) => t.kind !== "view")
|
|
132
|
+
.flatMap((t) => t.fields ?? []);
|
|
133
|
+
return {
|
|
134
|
+
keys: [
|
|
135
|
+
...data.map((path) => dataDir + path),
|
|
136
|
+
...(tree ? JSON.parse(tree).entries.map((e) => `assets/${e.sha256}`) : []),
|
|
137
|
+
],
|
|
138
|
+
assetRows: fields.some((f) => f.asset === true),
|
|
139
|
+
named: list.named !== undefined,
|
|
140
|
+
};
|
|
141
|
+
};
|
|
142
|
+
/** The versions without expired.json, from one listing of versions/ */
|
|
143
|
+
const liveVersions = async (store) => {
|
|
144
|
+
const keys = new Set((await store.list("versions/")).map((p) => p.key));
|
|
145
|
+
return [...keys]
|
|
146
|
+
.map((key) => key.match(/^versions\/([^/]+)\/version\.json$/)?.[1])
|
|
147
|
+
.filter((id) => isVersionId(id) && !keys.has(`versions/${id}/expired.json`));
|
|
148
|
+
};
|
|
149
|
+
/**
|
|
150
|
+
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
151
|
+
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
152
|
+
* catalogs wrote and never pushed. A version reaches every data file its catalog names, listed
|
|
153
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
154
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
155
|
+
* since reading those rows needs the catalog. The objects are listed before the versions are
|
|
156
|
+
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
157
|
+
* keeps what it reaches. Returns the keys deleted
|
|
158
|
+
*/
|
|
159
|
+
export const collect = async (store, { graceMs, now = new Date(), dryRun = false }) => {
|
|
160
|
+
const { data_dir: dataDir } = await readConfig(store);
|
|
161
|
+
const before = now.getTime() - graceMs;
|
|
162
|
+
const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
|
|
163
|
+
const kept = new Set();
|
|
164
|
+
let assetRows = false;
|
|
165
|
+
let unnamed = false;
|
|
166
|
+
const keep = async (ids) => {
|
|
167
|
+
for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
|
|
168
|
+
for (const key of r.keys)
|
|
169
|
+
kept.add(key);
|
|
170
|
+
assetRows ||= r.assetRows;
|
|
171
|
+
unnamed ||= !r.named;
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
const read = await liveVersions(store);
|
|
175
|
+
await keep(read);
|
|
176
|
+
const old = (placed) => placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
|
|
177
|
+
const planned = () => [
|
|
178
|
+
// A live version whose files.json lacks `named` may reach data files no files.json lists
|
|
179
|
+
...(unnamed ? [] : old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length)))),
|
|
180
|
+
...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
|
|
181
|
+
].sort();
|
|
182
|
+
if (dryRun)
|
|
183
|
+
return planned();
|
|
184
|
+
const seen = new Set(read);
|
|
185
|
+
await keep((await liveVersions(store)).filter((id) => !seen.has(id)));
|
|
186
|
+
const deleted = planned();
|
|
187
|
+
await store.delete(deleted);
|
|
188
|
+
return deleted;
|
|
189
|
+
};
|
|
190
|
+
//# sourceMappingURL=maintain.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F,yFAAyF;AACzF,sEAAsE;AACtE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnF,OAAO,EAA2B,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAuBzE,+CAA+C;AAC/C,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,MAAM,MAAM,GAAG,CAAI,KAAU,EAAE,IAAY,EAAS,EAAE,CACpD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9D,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CACtC,CAAC;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,MAAkB,EAClB,MAAc,EACd,SAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EACtB,EAAE;IACnB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAqB,EAAE;QACpD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAA0B,CAAC;QAC/B,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO;gBAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAAE,OAAO,MAAM,CAAC;YACpC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACxB,CAAC;IACH,CAAC,CAAC;IACF,OAAO;QACL,GAAG,MAAM;QACT,IAAI;QACJ,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;YACvB,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;gBAC5C,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAwB,EAAE;IACtE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC7D,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,EAAE,CACnD,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;AAE7D;;;GAGG;AACH,MAAM,IAAI,GAAG,KAAK,EAAE,KAAsB,EAAwB,EAAE;IAClE,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACf,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;QACvC,IAAI,CAAC,YAAY,QAAQ;YAAE,OAAO,IAAI,CAAC;QACvC,MAAM,CAAC,CAAC;IACV,CAAC,CAAC,CACH,CACF,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAsB,EAAE,EAAU,EAAE,GAAS,EAAE,EAAE;IACxE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC,GAAG,CACb,GAAG,GAAG,cAAc,EACpB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,IAAI,CAC9E,CAAC;IACJ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,MAAM,CACL,CAAC,GAAG,EAAE,EAAE,CACN,GAAG,KAAK,GAAG,GAAG,iBAAiB;QAC/B,GAAG,KAAK,GAAG,GAAG,YAAY;QAC1B,GAAG,KAAK,GAAG,GAAG,WAAW;QACzB,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC;QACjC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC,CACpC,CAAC;IACJ,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC,CAAC;AAaF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EACzB,KAAsB,EACtB,EAAE,IAAI,EAAE,GAAG,GAAG,iBAAiB,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAiB,EACtE,EAAE;IACrB,IAAI,IAAI,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,EAAE,EAAsB,EAAE,QAAgB,EAAqB,EAAE;QACjF,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;QAC5E,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,MAAM;YAAE,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACxD,4EAA4E;QAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACzB,CAAC,CAAC;AAUF,MAAM,MAAM,GAAG,wBAAwB,CAAC;AACxC,iEAAiE;AACjE,MAAM,SAAS,GAAG,gCAAgC,CAAC;AAEnD;;;;GAIG;AACH,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,OAAe,EAAE,EAAU,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,IAAwB,EAAE,QAAgB,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,IAAI,QAAQ,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,6BAA6B,KAAK,EAAE,CAAC,CAAC;QAC9F,OAAO,GAAG,GAAG,KAAK,CAAC;IACrB,CAAC,CAAC;IACF,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;KACnD,CAAC,CAAC;IACH,IAAI,KAAK,KAAK,IAAI;QAChB,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,qDAAqD,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,CAAC;IAC3C,MAAM,IAAI,GAAG;QACX,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzE;KACF,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;SAClF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;SAChC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAClC,OAAO;QACL,IAAI,EAAE;YACJ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrF;QACD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC;QAC/C,KAAK,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS;KAChC,CAAC;AACJ,CAAC,CAAC;AAEF,uEAAuE;AACvE,MAAM,YAAY,GAAG,KAAK,EAAE,KAAsB,EAAqB,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SAClE,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,CAAC;AAC/F,CAAC,CAAC;AAUF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAsB,EACtB,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAkB,EAC1C,EAAE;IACrB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;QACnC,KAAK,MAAM,CAAC,IAAI,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC;YAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB;QACE,yFAAyF;QACzF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxF,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;KACpE,CAAC,IAAI,EAAE,CAAC;IACX,IAAI,MAAM;QAAE,OAAO,OAAO,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}
|
package/dist/repo.d.ts
CHANGED
|
@@ -54,6 +54,10 @@ export type DataFile = {
|
|
|
54
54
|
bytes: number;
|
|
55
55
|
footerBytes?: number;
|
|
56
56
|
};
|
|
57
|
+
/** Whether a value is a version ID: a lower-case UUIDv7 */
|
|
58
|
+
export declare const isVersionId: (value: unknown) => value is string;
|
|
59
|
+
/** Whether a path in version.json is a plain file name in the version's folder */
|
|
60
|
+
export declare const isFileName: (value: unknown) => value is string;
|
|
57
61
|
export declare const readConfig: (store: Store) => Promise<Config>;
|
|
58
62
|
/**
|
|
59
63
|
* The version ID that rev names: a version ID itself, "HEAD", a full ref ("refs/tags/2025-final")
|
package/dist/repo.js
CHANGED
|
@@ -1,11 +1,20 @@
|
|
|
1
1
|
// Resolve a ref to a version and list the data files of its tables, following
|
|
2
|
-
// https://fdl.
|
|
2
|
+
// https://fdl.queria.io/spec/reading. Only files.json is read: the catalog is never opened
|
|
3
3
|
import { NotFoundError } from "./store.js";
|
|
4
4
|
/** The FDL major versions this reader knows */
|
|
5
5
|
const KNOWN_MAJOR = "0";
|
|
6
6
|
export class FdlError extends Error {
|
|
7
7
|
}
|
|
8
|
-
const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{
|
|
8
|
+
const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
9
|
+
/** Whether a value is a version ID: a lower-case UUIDv7 */
|
|
10
|
+
export const isVersionId = (value) => typeof value === "string" && VERSION_ID.test(value);
|
|
11
|
+
/** Whether a path in version.json is a plain file name in the version's folder */
|
|
12
|
+
export const isFileName = (value) => typeof value === "string" &&
|
|
13
|
+
value !== "" &&
|
|
14
|
+
value !== "." &&
|
|
15
|
+
value !== ".." &&
|
|
16
|
+
!value.includes("/") &&
|
|
17
|
+
!value.includes("\\");
|
|
9
18
|
const required = async (store, key) => {
|
|
10
19
|
const text = await store.text(key);
|
|
11
20
|
if (text === null)
|
|
@@ -88,7 +97,7 @@ export const readVersion = async (store, id, config) => {
|
|
|
88
97
|
throw new FdlError(`${dir}version.json holds ${record.id}`);
|
|
89
98
|
checkMajor(String(record.fdl), `version ${id}`);
|
|
90
99
|
const filesPath = record.files?.path ?? "files.json";
|
|
91
|
-
if (!
|
|
100
|
+
if (!isFileName(filesPath))
|
|
92
101
|
throw new FdlError(`version ${id} names files.json outside it: ${filesPath}`);
|
|
93
102
|
const files = await store.text(dir + filesPath);
|
|
94
103
|
if (files === null)
|
package/dist/repo.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiCtC,MAAM,UAAU,GAAG,
|
|
1
|
+
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiCtC,MAAM,UAAU,GAAG,uEAAuE,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC7D,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,KAAK,EAAE;IACZ,KAAK,KAAK,GAAG;IACb,KAAK,KAAK,IAAI;IACd,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACpB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExB,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO;QACL,GAAG,GAAG;QACN,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACpB,QAAQ,EAAE,OAAO,KAAK,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG;KAC5E,CAAC;AACJ,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACxB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACf,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@queria-io/fdl-js",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare-workers",
|
|
@@ -54,6 +54,45 @@
|
|
|
54
54
|
"typescript": "^5.7.4",
|
|
55
55
|
"vitest": "^4.1.0"
|
|
56
56
|
},
|
|
57
|
+
"release": {
|
|
58
|
+
"branches": [
|
|
59
|
+
"main"
|
|
60
|
+
],
|
|
61
|
+
"plugins": [
|
|
62
|
+
[
|
|
63
|
+
"@semantic-release/commit-analyzer",
|
|
64
|
+
{
|
|
65
|
+
"preset": "conventionalcommits",
|
|
66
|
+
"releaseRules": [
|
|
67
|
+
{
|
|
68
|
+
"breaking": true,
|
|
69
|
+
"release": "minor"
|
|
70
|
+
}
|
|
71
|
+
]
|
|
72
|
+
}
|
|
73
|
+
],
|
|
74
|
+
[
|
|
75
|
+
"@semantic-release/release-notes-generator",
|
|
76
|
+
{
|
|
77
|
+
"preset": "conventionalcommits"
|
|
78
|
+
}
|
|
79
|
+
],
|
|
80
|
+
[
|
|
81
|
+
"@semantic-release/npm",
|
|
82
|
+
{
|
|
83
|
+
"npmPublish": false
|
|
84
|
+
}
|
|
85
|
+
],
|
|
86
|
+
[
|
|
87
|
+
"@semantic-release/github",
|
|
88
|
+
{
|
|
89
|
+
"successComment": false,
|
|
90
|
+
"failComment": false,
|
|
91
|
+
"releasedLabels": false
|
|
92
|
+
}
|
|
93
|
+
]
|
|
94
|
+
]
|
|
95
|
+
},
|
|
57
96
|
"engines": {
|
|
58
97
|
"node": ">=20"
|
|
59
98
|
},
|
package/src/index.ts
CHANGED
package/src/maintain.ts
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
|
+
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
|
+
// live version reaches once past the grace period (as `fdl gc` does). Catalogs are never
|
|
4
|
+
// opened: what a version reaches comes from `named` in its files.json
|
|
5
|
+
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
|
+
import { type R2Like, type Store, r2Store, retryOnce } from "./store.js";
|
|
7
|
+
|
|
8
|
+
/** A listed object: its key and when it was written */
|
|
9
|
+
export type Placed = { key: string; uploaded: Date };
|
|
10
|
+
|
|
11
|
+
/** A store a host can change: a reader's store, plus listing, writing and deleting */
|
|
12
|
+
export type MaintainedStore = Store & {
|
|
13
|
+
list: (prefix: string) => Promise<Placed[]>;
|
|
14
|
+
put: (key: string, text: string) => Promise<void>;
|
|
15
|
+
delete: (keys: string[]) => Promise<void>;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/** The part of an R2 binding that maintenance needs, besides what a reader needs */
|
|
19
|
+
export type R2Writable = R2Like & {
|
|
20
|
+
list: (options: { prefix: string; cursor?: string }) => Promise<{
|
|
21
|
+
objects: { key: string; uploaded: Date }[];
|
|
22
|
+
truncated: boolean;
|
|
23
|
+
cursor?: string;
|
|
24
|
+
}>;
|
|
25
|
+
put: (key: string, value: string) => Promise<unknown>;
|
|
26
|
+
delete: (keys: string[]) => Promise<void>;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/** Up to this many keys go in one R2 delete */
|
|
30
|
+
const DELETE_BATCH = 1000;
|
|
31
|
+
|
|
32
|
+
const chunks = <T>(items: T[], size: number): T[][] =>
|
|
33
|
+
Array.from({ length: Math.ceil(items.length / size) }, (_, i) =>
|
|
34
|
+
items.slice(i * size, (i + 1) * size),
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* A repository in an R2 bucket that maintenance may change. Reads go through `reader`, which
|
|
39
|
+
* defaults to `r2Store(bucket, prefix)`; pass another one (with a cache, say) for the same place
|
|
40
|
+
*/
|
|
41
|
+
export const r2MaintainedStore = (
|
|
42
|
+
bucket: R2Writable,
|
|
43
|
+
prefix: string,
|
|
44
|
+
reader: Store = r2Store(bucket, prefix),
|
|
45
|
+
): MaintainedStore => {
|
|
46
|
+
const list = async (key: string): Promise<Placed[]> => {
|
|
47
|
+
const placed: Placed[] = [];
|
|
48
|
+
let cursor: string | undefined;
|
|
49
|
+
for (;;) {
|
|
50
|
+
const found = await retryOnce(() => bucket.list({ prefix: prefix + key, cursor }));
|
|
51
|
+
for (const o of found.objects)
|
|
52
|
+
placed.push({ key: o.key.slice(prefix.length), uploaded: o.uploaded });
|
|
53
|
+
if (!found.truncated) return placed;
|
|
54
|
+
cursor = found.cursor;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
return {
|
|
58
|
+
...reader,
|
|
59
|
+
list,
|
|
60
|
+
put: async (key, text) => {
|
|
61
|
+
await retryOnce(() => bucket.put(prefix + key, text));
|
|
62
|
+
},
|
|
63
|
+
delete: async (keys) => {
|
|
64
|
+
for (const batch of chunks(keys, DELETE_BATCH))
|
|
65
|
+
await retryOnce(() => bucket.delete(batch.map((key) => prefix + key)));
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
type Head = {
|
|
71
|
+
id: string;
|
|
72
|
+
parents: string[];
|
|
73
|
+
created_at: string;
|
|
74
|
+
files?: { path: string };
|
|
75
|
+
tree?: { path: string };
|
|
76
|
+
dataset?: { path: string };
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const record = async (store: Store, id: string): Promise<Head | null> => {
|
|
80
|
+
const text = await store.text(`versions/${id}/version.json`);
|
|
81
|
+
return text === null ? null : (JSON.parse(text) as Head);
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
const isExpired = async (store: Store, id: string) =>
|
|
85
|
+
(await store.text(`versions/${id}/expired.json`)) !== null;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The versions some ref points at. A ref that does not hold a version ID is skipped; a failed
|
|
89
|
+
* read is not, since a version a ref points at must not be taken out
|
|
90
|
+
*/
|
|
91
|
+
const held = async (store: MaintainedStore): Promise<Set<string>> => {
|
|
92
|
+
const keys = (await store.list("refs/")).map((p) => p.key);
|
|
93
|
+
const targets = await Promise.all(
|
|
94
|
+
keys.map((key) =>
|
|
95
|
+
resolve(store, key).catch((e: unknown) => {
|
|
96
|
+
if (e instanceof FdlError) return null;
|
|
97
|
+
throw e;
|
|
98
|
+
}),
|
|
99
|
+
),
|
|
100
|
+
);
|
|
101
|
+
return new Set(targets.filter((id): id is string => id !== null));
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
/** Takes one version out of retention: the mark first, then its catalogs and indexes */
|
|
105
|
+
const expireOne = async (store: MaintainedStore, id: string, now: Date) => {
|
|
106
|
+
const dir = `versions/${id}/`;
|
|
107
|
+
if (!(await isExpired(store, id)))
|
|
108
|
+
await store.put(
|
|
109
|
+
`${dir}expired.json`,
|
|
110
|
+
`${JSON.stringify({ expired_at: now.toISOString(), reason: "retention" })}\n`,
|
|
111
|
+
);
|
|
112
|
+
const indexed = (await store.list(dir))
|
|
113
|
+
.map((p) => p.key)
|
|
114
|
+
.filter(
|
|
115
|
+
(key) =>
|
|
116
|
+
key === `${dir}ducklake.duckdb` ||
|
|
117
|
+
key === `${dir}files.json` ||
|
|
118
|
+
key === `${dir}tree.json` ||
|
|
119
|
+
key.startsWith(`${dir}catalogs/`) ||
|
|
120
|
+
key.startsWith(`${dir}profiles/`),
|
|
121
|
+
);
|
|
122
|
+
await store.delete(indexed);
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
export type ExpireOptions = {
|
|
126
|
+
/** How many of the newest versions on the ref's first-parent line to keep (at least 1) */
|
|
127
|
+
keep: number;
|
|
128
|
+
ref?: string;
|
|
129
|
+
/** Also keep the versions created at or after this time */
|
|
130
|
+
since?: Date;
|
|
131
|
+
now?: Date;
|
|
132
|
+
/** List only. Default false */
|
|
133
|
+
dryRun?: boolean;
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Takes the versions behind the newest `keep` on the ref's first-parent line out of retention,
|
|
138
|
+
* as `fdl expire` does. Versions a ref points at, and with `since` those created at or after
|
|
139
|
+
* it, are kept. Stops at the first version already
|
|
140
|
+
* expired. Returns the IDs expired, newest first
|
|
141
|
+
*/
|
|
142
|
+
export const expire = async (
|
|
143
|
+
store: MaintainedStore,
|
|
144
|
+
{ keep, ref = "refs/heads/main", since, now = new Date(), dryRun = false }: ExpireOptions,
|
|
145
|
+
): Promise<string[]> => {
|
|
146
|
+
if (keep < 1) throw new RangeError(`keep must be at least 1, got ${keep}`);
|
|
147
|
+
const newest = await resolve(store, ref).catch(() => null);
|
|
148
|
+
if (newest === null) return [];
|
|
149
|
+
const kept = await held(store);
|
|
150
|
+
const walk = async (id: string | undefined, position: number): Promise<string[]> => {
|
|
151
|
+
if (id === undefined || (position > 0 && (await isExpired(store, id)))) return [];
|
|
152
|
+
const current = await record(store, id);
|
|
153
|
+
if (current === null) return [];
|
|
154
|
+
const recent = since !== undefined && new Date(current.created_at) >= since;
|
|
155
|
+
const expired = position >= keep && !recent && !kept.has(id);
|
|
156
|
+
if (expired && !dryRun) await expireOne(store, id, now);
|
|
157
|
+
// A parent in any other form is not followed: the ID becomes part of a path
|
|
158
|
+
const parent = current.parents[0];
|
|
159
|
+
const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
|
|
160
|
+
return expired ? [id, ...rest] : rest;
|
|
161
|
+
};
|
|
162
|
+
return walk(newest, 0);
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
type FileList = {
|
|
166
|
+
tables: { [table: string]: { files: { path: string; deletes?: { path: string } }[] } };
|
|
167
|
+
/** Every data file the catalog names, at any snapshot. Absent in files.json written before it */
|
|
168
|
+
named?: string[];
|
|
169
|
+
};
|
|
170
|
+
type Described = { tables?: { kind?: string; fields?: { asset?: boolean }[] }[] };
|
|
171
|
+
type Tree = { entries: { sha256: string }[] };
|
|
172
|
+
|
|
173
|
+
const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
174
|
+
/** A data or delete file as DuckLake names it, under data_dir */
|
|
175
|
+
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* What a live version reaches: the data files its catalog names, its tree's assets, whether rows
|
|
179
|
+
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
180
|
+
* files in `tables` are only those at the version's snapshot, and the catalog may name more
|
|
181
|
+
*/
|
|
182
|
+
const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
183
|
+
const dir = `versions/${id}/`;
|
|
184
|
+
const current = await record(store, id);
|
|
185
|
+
const name = (path: string | undefined, fallback: string) => {
|
|
186
|
+
const value = path ?? fallback;
|
|
187
|
+
if (!isFileName(value)) throw new FdlError(`version ${id} names a file outside it: ${value}`);
|
|
188
|
+
return dir + value;
|
|
189
|
+
};
|
|
190
|
+
const [files, described, tree] = await Promise.all([
|
|
191
|
+
store.text(name(current?.files?.path, "files.json")),
|
|
192
|
+
store.text(name(current?.dataset?.path, "dataset.json")),
|
|
193
|
+
store.text(name(current?.tree?.path, "tree.json")),
|
|
194
|
+
]);
|
|
195
|
+
if (files === null)
|
|
196
|
+
throw new Error(`version ${id} is live but has no files.json to say what it reads`);
|
|
197
|
+
const list = JSON.parse(files) as FileList;
|
|
198
|
+
const data = [
|
|
199
|
+
...(list.named ?? []),
|
|
200
|
+
...Object.values(list.tables).flatMap((t) =>
|
|
201
|
+
t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])]),
|
|
202
|
+
),
|
|
203
|
+
];
|
|
204
|
+
const fields = ((described ? (JSON.parse(described) as Described) : {}).tables ?? [])
|
|
205
|
+
.filter((t) => t.kind !== "view")
|
|
206
|
+
.flatMap((t) => t.fields ?? []);
|
|
207
|
+
return {
|
|
208
|
+
keys: [
|
|
209
|
+
...data.map((path) => dataDir + path),
|
|
210
|
+
...(tree ? (JSON.parse(tree) as Tree).entries.map((e) => `assets/${e.sha256}`) : []),
|
|
211
|
+
],
|
|
212
|
+
assetRows: fields.some((f) => f.asset === true),
|
|
213
|
+
named: list.named !== undefined,
|
|
214
|
+
};
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
/** The versions without expired.json, from one listing of versions/ */
|
|
218
|
+
const liveVersions = async (store: MaintainedStore): Promise<string[]> => {
|
|
219
|
+
const keys = new Set((await store.list("versions/")).map((p) => p.key));
|
|
220
|
+
return [...keys]
|
|
221
|
+
.map((key) => key.match(/^versions\/([^/]+)\/version\.json$/)?.[1])
|
|
222
|
+
.filter((id): id is string => isVersionId(id) && !keys.has(`versions/${id}/expired.json`));
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
export type CollectOptions = {
|
|
226
|
+
/** Objects written more recently than this are kept */
|
|
227
|
+
graceMs: number;
|
|
228
|
+
now?: Date;
|
|
229
|
+
/** List only. Default false */
|
|
230
|
+
dryRun?: boolean;
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
235
|
+
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
236
|
+
* catalogs wrote and never pushed. A version reaches every data file its catalog names, listed
|
|
237
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
238
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
239
|
+
* since reading those rows needs the catalog. The objects are listed before the versions are
|
|
240
|
+
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
241
|
+
* keeps what it reaches. Returns the keys deleted
|
|
242
|
+
*/
|
|
243
|
+
export const collect = async (
|
|
244
|
+
store: MaintainedStore,
|
|
245
|
+
{ graceMs, now = new Date(), dryRun = false }: CollectOptions,
|
|
246
|
+
): Promise<string[]> => {
|
|
247
|
+
const { data_dir: dataDir } = await readConfig(store);
|
|
248
|
+
const before = now.getTime() - graceMs;
|
|
249
|
+
const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
|
|
250
|
+
const kept = new Set<string>();
|
|
251
|
+
let assetRows = false;
|
|
252
|
+
let unnamed = false;
|
|
253
|
+
const keep = async (ids: string[]) => {
|
|
254
|
+
for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
|
|
255
|
+
for (const key of r.keys) kept.add(key);
|
|
256
|
+
assetRows ||= r.assetRows;
|
|
257
|
+
unnamed ||= !r.named;
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
const read = await liveVersions(store);
|
|
261
|
+
await keep(read);
|
|
262
|
+
const old = (placed: Placed[]) =>
|
|
263
|
+
placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
|
|
264
|
+
const planned = () =>
|
|
265
|
+
[
|
|
266
|
+
// A live version whose files.json lacks `named` may reach data files no files.json lists
|
|
267
|
+
...(unnamed ? [] : old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length)))),
|
|
268
|
+
...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
|
|
269
|
+
].sort();
|
|
270
|
+
if (dryRun) return planned();
|
|
271
|
+
const seen = new Set(read);
|
|
272
|
+
await keep((await liveVersions(store)).filter((id) => !seen.has(id)));
|
|
273
|
+
const deleted = planned();
|
|
274
|
+
await store.delete(deleted);
|
|
275
|
+
return deleted;
|
|
276
|
+
};
|
package/src/repo.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Resolve a ref to a version and list the data files of its tables, following
|
|
2
|
-
// https://fdl.
|
|
2
|
+
// https://fdl.queria.io/spec/reading. Only files.json is read: the catalog is never opened
|
|
3
3
|
import { NotFoundError, type Store } from "./store.js";
|
|
4
4
|
|
|
5
5
|
/** The FDL major versions this reader knows */
|
|
@@ -38,7 +38,20 @@ export type Version = { config: Config; record: VersionRecord; files: FileList }
|
|
|
38
38
|
/** A data file to read: its key in the store, rows and sizes from files.json */
|
|
39
39
|
export type DataFile = { key: string; rows: number; bytes: number; footerBytes?: number };
|
|
40
40
|
|
|
41
|
-
const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{
|
|
41
|
+
const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
42
|
+
|
|
43
|
+
/** Whether a value is a version ID: a lower-case UUIDv7 */
|
|
44
|
+
export const isVersionId = (value: unknown): value is string =>
|
|
45
|
+
typeof value === "string" && VERSION_ID.test(value);
|
|
46
|
+
|
|
47
|
+
/** Whether a path in version.json is a plain file name in the version's folder */
|
|
48
|
+
export const isFileName = (value: unknown): value is string =>
|
|
49
|
+
typeof value === "string" &&
|
|
50
|
+
value !== "" &&
|
|
51
|
+
value !== "." &&
|
|
52
|
+
value !== ".." &&
|
|
53
|
+
!value.includes("/") &&
|
|
54
|
+
!value.includes("\\");
|
|
42
55
|
|
|
43
56
|
const required = async (store: Store, key: string): Promise<string> => {
|
|
44
57
|
const text = await store.text(key);
|
|
@@ -124,7 +137,7 @@ export const readVersion = async (store: Store, id: string, config?: Config): Pr
|
|
|
124
137
|
if (record.id !== id) throw new FdlError(`${dir}version.json holds ${record.id}`);
|
|
125
138
|
checkMajor(String(record.fdl), `version ${id}`);
|
|
126
139
|
const filesPath = record.files?.path ?? "files.json";
|
|
127
|
-
if (!
|
|
140
|
+
if (!isFileName(filesPath))
|
|
128
141
|
throw new FdlError(`version ${id} names files.json outside it: ${filesPath}`);
|
|
129
142
|
const files = await store.text(dir + filesPath);
|
|
130
143
|
if (files === null) throw new FdlError(`version ${id} has no files.json; read it with DuckDB`);
|