@queria-io/fdl-js 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/maintain.d.ts +4 -2
- package/dist/maintain.js +35 -11
- package/dist/maintain.js.map +1 -1
- package/dist/repo.js +1 -1
- package/dist/repo.js.map +1 -1
- package/package.json +14 -3
- package/src/maintain.ts +43 -13
- package/src/repo.ts +1 -1
package/README.md
CHANGED
|
@@ -69,6 +69,6 @@ pnpm check
|
|
|
69
69
|
pnpm build # dist/
|
|
70
70
|
```
|
|
71
71
|
|
|
72
|
-
Releases are made from the commit messages ([Conventional Commits](https://www.conventionalcommits.org/)). Each push to `main` runs the Release workflow: when the commits since the last tag include a `fix:` it publishes a patch version, a `feat:` a minor one, and a breaking change (`!`
|
|
72
|
+
Releases are made from the commit messages ([Conventional Commits](https://www.conventionalcommits.org/)). Each push to `main` runs the Release workflow: when the commits since the last tag include a `fix:` it publishes a patch version, a `feat:` a minor one, and a breaking change (`!` after the type, or a `BREAKING CHANGE:` footer) a minor one. It publishes to npm, tags the commit `v<version>` and makes a GitHub release. `docs:`, `chore:`, `test:` and the like release nothing.
|
|
73
73
|
|
|
74
74
|
`test/fixture/repo` is a small repository made by the `fdl` command: `FDL=fdl test/make-fixture.sh` remakes it.
|
package/dist/maintain.d.ts
CHANGED
|
@@ -58,8 +58,10 @@ export type CollectOptions = {
|
|
|
58
58
|
/**
|
|
59
59
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
60
60
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
61
|
-
*
|
|
62
|
-
*
|
|
61
|
+
* metadata files wrote and never pushed. A version reaches every data file its metadata names, listed
|
|
62
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
63
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
64
|
+
* since reading those rows needs the metadata. The objects are listed before the versions are
|
|
63
65
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
64
66
|
* keeps what it reaches. Returns the keys deleted
|
|
65
67
|
*/
|
package/dist/maintain.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
2
|
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
|
-
// live version reaches once past the grace period (as `fdl gc` does).
|
|
4
|
-
// opened: what a version
|
|
3
|
+
// live version reaches once past the grace period (as `fdl gc` does). Metadata files are never
|
|
4
|
+
// opened: what a version reaches comes from `named` in its files.json
|
|
5
5
|
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
6
|
import { r2Store, retryOnce } from "./store.js";
|
|
7
7
|
/** Up to this many keys go in one R2 delete */
|
|
@@ -54,16 +54,27 @@ const held = async (store) => {
|
|
|
54
54
|
})));
|
|
55
55
|
return new Set(targets.filter((id) => id !== null));
|
|
56
56
|
};
|
|
57
|
-
/**
|
|
58
|
-
|
|
57
|
+
/**
|
|
58
|
+
* The metadata files to delete: the one the record names (`metadata`, or `catalog` in records
|
|
59
|
+
* written by fdl 0.26 and earlier), or both default names when the record names none it can use
|
|
60
|
+
*/
|
|
61
|
+
const metadataPaths = (head) => {
|
|
62
|
+
const path = (head.metadata ?? head.catalog)?.path;
|
|
63
|
+
return isFileName(path) ? [path] : ["metadata.db", "ducklake.duckdb"];
|
|
64
|
+
};
|
|
65
|
+
/** Takes one version out of retention: the mark first, then its metadata and indexes */
|
|
66
|
+
const expireOne = async (store, id, head, now) => {
|
|
59
67
|
const dir = `versions/${id}/`;
|
|
68
|
+
const metadata = new Set(metadataPaths(head).map((path) => dir + path));
|
|
60
69
|
if (!(await isExpired(store, id)))
|
|
61
70
|
await store.put(`${dir}expired.json`, `${JSON.stringify({ expired_at: now.toISOString(), reason: "retention" })}\n`);
|
|
62
71
|
const indexed = (await store.list(dir))
|
|
63
72
|
.map((p) => p.key)
|
|
64
|
-
.filter((key) => key
|
|
73
|
+
.filter((key) => metadata.has(key) ||
|
|
65
74
|
key === `${dir}files.json` ||
|
|
66
75
|
key === `${dir}tree.json` ||
|
|
76
|
+
key.startsWith(`${dir}metadata/`) ||
|
|
77
|
+
// Other DuckLake formats, as fdl 0.26 and earlier lay them out
|
|
67
78
|
key.startsWith(`${dir}catalogs/`) ||
|
|
68
79
|
key.startsWith(`${dir}profiles/`));
|
|
69
80
|
await store.delete(indexed);
|
|
@@ -90,7 +101,7 @@ export const expire = async (store, { keep, ref = "refs/heads/main", since, now
|
|
|
90
101
|
const recent = since !== undefined && new Date(current.created_at) >= since;
|
|
91
102
|
const expired = position >= keep && !recent && !kept.has(id);
|
|
92
103
|
if (expired && !dryRun)
|
|
93
|
-
await expireOne(store, id, now);
|
|
104
|
+
await expireOne(store, id, current, now);
|
|
94
105
|
// A parent in any other form is not followed: the ID becomes part of a path
|
|
95
106
|
const parent = current.parents[0];
|
|
96
107
|
const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
|
|
@@ -101,7 +112,11 @@ export const expire = async (store, { keep, ref = "refs/heads/main", since, now
|
|
|
101
112
|
const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
102
113
|
/** A data or delete file as DuckLake names it, under data_dir */
|
|
103
114
|
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
104
|
-
/**
|
|
115
|
+
/**
|
|
116
|
+
* What a live version reaches: the data files its metadata names, its tree's assets, whether rows
|
|
117
|
+
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
118
|
+
* files in `tables` are only those at the version's snapshot, and the metadata may name more
|
|
119
|
+
*/
|
|
105
120
|
const reachedBy = async (store, dataDir, id) => {
|
|
106
121
|
const dir = `versions/${id}/`;
|
|
107
122
|
const current = await record(store, id);
|
|
@@ -119,7 +134,10 @@ const reachedBy = async (store, dataDir, id) => {
|
|
|
119
134
|
if (files === null)
|
|
120
135
|
throw new Error(`version ${id} is live but has no files.json to say what it reads`);
|
|
121
136
|
const list = JSON.parse(files);
|
|
122
|
-
const data =
|
|
137
|
+
const data = [
|
|
138
|
+
...(list.named ?? []),
|
|
139
|
+
...Object.values(list.tables).flatMap((t) => t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])])),
|
|
140
|
+
];
|
|
123
141
|
const fields = ((described ? JSON.parse(described) : {}).tables ?? [])
|
|
124
142
|
.filter((t) => t.kind !== "view")
|
|
125
143
|
.flatMap((t) => t.fields ?? []);
|
|
@@ -129,6 +147,7 @@ const reachedBy = async (store, dataDir, id) => {
|
|
|
129
147
|
...(tree ? JSON.parse(tree).entries.map((e) => `assets/${e.sha256}`) : []),
|
|
130
148
|
],
|
|
131
149
|
assetRows: fields.some((f) => f.asset === true),
|
|
150
|
+
named: list.named !== undefined,
|
|
132
151
|
};
|
|
133
152
|
};
|
|
134
153
|
/** The versions without expired.json, from one listing of versions/ */
|
|
@@ -141,8 +160,10 @@ const liveVersions = async (store) => {
|
|
|
141
160
|
/**
|
|
142
161
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
143
162
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
144
|
-
*
|
|
145
|
-
*
|
|
163
|
+
* metadata files wrote and never pushed. A version reaches every data file its metadata names, listed
|
|
164
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
165
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
166
|
+
* since reading those rows needs the metadata. The objects are listed before the versions are
|
|
146
167
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
147
168
|
* keeps what it reaches. Returns the keys deleted
|
|
148
169
|
*/
|
|
@@ -152,18 +173,21 @@ export const collect = async (store, { graceMs, now = new Date(), dryRun = false
|
|
|
152
173
|
const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
|
|
153
174
|
const kept = new Set();
|
|
154
175
|
let assetRows = false;
|
|
176
|
+
let unnamed = false;
|
|
155
177
|
const keep = async (ids) => {
|
|
156
178
|
for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
|
|
157
179
|
for (const key of r.keys)
|
|
158
180
|
kept.add(key);
|
|
159
181
|
assetRows ||= r.assetRows;
|
|
182
|
+
unnamed ||= !r.named;
|
|
160
183
|
}
|
|
161
184
|
};
|
|
162
185
|
const read = await liveVersions(store);
|
|
163
186
|
await keep(read);
|
|
164
187
|
const old = (placed) => placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
|
|
165
188
|
const planned = () => [
|
|
166
|
-
|
|
189
|
+
// A live version whose files.json lacks `named` may reach data files no files.json lists
|
|
190
|
+
...(unnamed ? [] : old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length)))),
|
|
167
191
|
...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
|
|
168
192
|
].sort();
|
|
169
193
|
if (dryRun)
|
package/dist/maintain.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F
|
|
1
|
+
{"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F,+FAA+F;AAC/F,sEAAsE;AACtE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnF,OAAO,EAA2B,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAuBzE,+CAA+C;AAC/C,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,MAAM,MAAM,GAAG,CAAI,KAAU,EAAE,IAAY,EAAS,EAAE,CACpD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9D,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CACtC,CAAC;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,MAAkB,EAClB,MAAc,EACd,SAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EACtB,EAAE;IACnB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAqB,EAAE;QACpD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAA0B,CAAC;QAC/B,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO;gBAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAAE,OAAO,MAAM,CAAC;YACpC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACxB,CAAC;IACH,CAAC,CAAC;IACF,OAAO;QACL,GAAG,MAAM;QACT,IAAI;QACJ,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;YACvB,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;gBAC5C,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAcF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAwB,EAAE;IACtE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC7D,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,EAAE,CACnD,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;AAE7D;;;GAGG;AACH,MAAM,IAAI,GAAG,KAAK,EAAE,KAAsB,EAAwB,EAAE;IAClE,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACf,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;QACvC,IAAI,CAAC,YAAY,QAAQ;YAAE,OAAO,IAAI,CAAC;QACvC,MAAM,CAAC,CAAC;IACV,CAAC,CAAC,CACH,CACF,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG,CAAC,IAAU,EAAY,EAAE;IAC7C,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC;IACnD,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAsB,EAAE,EAAU,EAAE,IAAU,EAAE,GAAS,EAAE,EAAE;IACpF,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC;IACxE,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC,GAAG,CACb,GAAG,GAAG,cAAc,EACpB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,IAAI,CAC9E,CAAC;IACJ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,MAAM,CACL,CAAC,GAAG,EAAE,EAAE,CACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;QACjB,GAAG,KAAK,GAAG,GAAG,YAAY;QAC1B,GAAG,KAAK,GAAG,GAAG,WAAW;QACzB,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC;QACjC,+DAA+D;QAC/D,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC;QACjC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC,CACpC,CAAC;IACJ,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC,CAAC;AAaF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EACzB,KAAsB,EACtB,EAAE,IAAI,EAAE,GAAG,GAAG,iBAAiB,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAiB,EACtE,EAAE;IACrB,IAAI,IAAI,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,EAAE,EAAsB,EAAE,QAAgB,EAAqB,EAAE;QACjF,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;QAC5E,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,MAAM;YAAE,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;QACjE,4EAA4E;QAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACzB,CAAC,CAAC;AAUF,MAAM,MAAM,GAAG,wBAAwB,CAAC;AACxC,iEAAiE;AACjE,MAAM,SAAS,GAAG,gCAAgC,CAAC;AAEnD;;;;GAIG;AACH,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,OAAe,EAAE,EAAU,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,IAAwB,EAAE,QAAgB,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,IAAI,QAAQ,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,6BAA6B,KAAK,EAAE,CAAC,CAAC;QAC9F,OAAO,GAAG,GAAG,KAAK,CAAC;IACrB,CAAC,CAAC;IACF,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;KACnD,CAAC,CAAC;IACH,IAAI,KAAK,KAAK,IAAI;QAChB,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,qDAAqD,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,CAAC;IAC3C,MAAM,IAAI,GAAG;QACX,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzE;KACF,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;SAClF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;SAChC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAClC,OAAO;QACL,IAAI,EAAE;YACJ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrF;QACD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC;QAC/C,KAAK,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS;KAChC,CAAC;AACJ,CAAC,CAAC;AAEF,uEAAuE;AACvE,MAAM,YAAY,GAAG,KAAK,EAAE,KAAsB,EAAqB,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SAClE,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,CAAC;AAC/F,CAAC,CAAC;AAUF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAsB,EACtB,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAkB,EAC1C,EAAE;IACrB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;QACnC,KAAK,MAAM,CAAC,IAAI,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC;YAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB;QACE,yFAAyF;QACzF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxF,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;KACpE,CAAC,IAAI,EAAE,CAAC;IACX,IAAI,MAAM;QAAE,OAAO,OAAO,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}
|
package/dist/repo.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Resolve a ref to a version and list the data files of its tables, following
|
|
2
|
-
// https://fdl.queria.io/spec/reading. Only files.json is read: the
|
|
2
|
+
// https://fdl.queria.io/spec/reading. Only files.json is read: the metadata is never opened
|
|
3
3
|
import { NotFoundError } from "./store.js";
|
|
4
4
|
/** The FDL major versions this reader knows */
|
|
5
5
|
const KNOWN_MAJOR = "0";
|
package/dist/repo.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,
|
|
1
|
+
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4FAA4F;AAC5F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiCtC,MAAM,UAAU,GAAG,uEAAuE,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC7D,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,KAAK,EAAE;IACZ,KAAK,KAAK,GAAG;IACb,KAAK,KAAK,IAAI;IACd,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACpB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExB,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO;QACL,GAAG,GAAG;QACN,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACpB,QAAQ,EAAE,OAAO,KAAK,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG;KAC5E,CAAC;AACJ,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACxB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACf,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@queria-io/fdl-js",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.2",
|
|
4
4
|
"description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare-workers",
|
|
@@ -62,6 +62,7 @@
|
|
|
62
62
|
[
|
|
63
63
|
"@semantic-release/commit-analyzer",
|
|
64
64
|
{
|
|
65
|
+
"preset": "conventionalcommits",
|
|
65
66
|
"releaseRules": [
|
|
66
67
|
{
|
|
67
68
|
"breaking": true,
|
|
@@ -70,8 +71,18 @@
|
|
|
70
71
|
]
|
|
71
72
|
}
|
|
72
73
|
],
|
|
73
|
-
|
|
74
|
-
|
|
74
|
+
[
|
|
75
|
+
"@semantic-release/release-notes-generator",
|
|
76
|
+
{
|
|
77
|
+
"preset": "conventionalcommits"
|
|
78
|
+
}
|
|
79
|
+
],
|
|
80
|
+
[
|
|
81
|
+
"@semantic-release/npm",
|
|
82
|
+
{
|
|
83
|
+
"npmPublish": false
|
|
84
|
+
}
|
|
85
|
+
],
|
|
75
86
|
[
|
|
76
87
|
"@semantic-release/github",
|
|
77
88
|
{
|
package/src/maintain.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
2
|
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
|
-
// live version reaches once past the grace period (as `fdl gc` does).
|
|
4
|
-
// opened: what a version
|
|
3
|
+
// live version reaches once past the grace period (as `fdl gc` does). Metadata files are never
|
|
4
|
+
// opened: what a version reaches comes from `named` in its files.json
|
|
5
5
|
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
6
|
import { type R2Like, type Store, r2Store, retryOnce } from "./store.js";
|
|
7
7
|
|
|
@@ -74,6 +74,9 @@ type Head = {
|
|
|
74
74
|
files?: { path: string };
|
|
75
75
|
tree?: { path: string };
|
|
76
76
|
dataset?: { path: string };
|
|
77
|
+
/** The metadata file. Records written by fdl 0.26 and earlier name it `catalog` */
|
|
78
|
+
metadata?: { path?: unknown };
|
|
79
|
+
catalog?: { path?: unknown };
|
|
77
80
|
};
|
|
78
81
|
|
|
79
82
|
const record = async (store: Store, id: string): Promise<Head | null> => {
|
|
@@ -101,9 +104,19 @@ const held = async (store: MaintainedStore): Promise<Set<string>> => {
|
|
|
101
104
|
return new Set(targets.filter((id): id is string => id !== null));
|
|
102
105
|
};
|
|
103
106
|
|
|
104
|
-
/**
|
|
105
|
-
|
|
107
|
+
/**
|
|
108
|
+
* The metadata files to delete: the one the record names (`metadata`, or `catalog` in records
|
|
109
|
+
* written by fdl 0.26 and earlier), or both default names when the record names none it can use
|
|
110
|
+
*/
|
|
111
|
+
const metadataPaths = (head: Head): string[] => {
|
|
112
|
+
const path = (head.metadata ?? head.catalog)?.path;
|
|
113
|
+
return isFileName(path) ? [path] : ["metadata.db", "ducklake.duckdb"];
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/** Takes one version out of retention: the mark first, then its metadata and indexes */
|
|
117
|
+
const expireOne = async (store: MaintainedStore, id: string, head: Head, now: Date) => {
|
|
106
118
|
const dir = `versions/${id}/`;
|
|
119
|
+
const metadata = new Set(metadataPaths(head).map((path) => dir + path));
|
|
107
120
|
if (!(await isExpired(store, id)))
|
|
108
121
|
await store.put(
|
|
109
122
|
`${dir}expired.json`,
|
|
@@ -113,9 +126,11 @@ const expireOne = async (store: MaintainedStore, id: string, now: Date) => {
|
|
|
113
126
|
.map((p) => p.key)
|
|
114
127
|
.filter(
|
|
115
128
|
(key) =>
|
|
116
|
-
key
|
|
129
|
+
metadata.has(key) ||
|
|
117
130
|
key === `${dir}files.json` ||
|
|
118
131
|
key === `${dir}tree.json` ||
|
|
132
|
+
key.startsWith(`${dir}metadata/`) ||
|
|
133
|
+
// Other DuckLake formats, as fdl 0.26 and earlier lay them out
|
|
119
134
|
key.startsWith(`${dir}catalogs/`) ||
|
|
120
135
|
key.startsWith(`${dir}profiles/`),
|
|
121
136
|
);
|
|
@@ -153,7 +168,7 @@ export const expire = async (
|
|
|
153
168
|
if (current === null) return [];
|
|
154
169
|
const recent = since !== undefined && new Date(current.created_at) >= since;
|
|
155
170
|
const expired = position >= keep && !recent && !kept.has(id);
|
|
156
|
-
if (expired && !dryRun) await expireOne(store, id, now);
|
|
171
|
+
if (expired && !dryRun) await expireOne(store, id, current, now);
|
|
157
172
|
// A parent in any other form is not followed: the ID becomes part of a path
|
|
158
173
|
const parent = current.parents[0];
|
|
159
174
|
const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
|
|
@@ -164,6 +179,8 @@ export const expire = async (
|
|
|
164
179
|
|
|
165
180
|
type FileList = {
|
|
166
181
|
tables: { [table: string]: { files: { path: string; deletes?: { path: string } }[] } };
|
|
182
|
+
/** Every data file the metadata names, at any snapshot. Absent in files.json written before it */
|
|
183
|
+
named?: string[];
|
|
167
184
|
};
|
|
168
185
|
type Described = { tables?: { kind?: string; fields?: { asset?: boolean }[] }[] };
|
|
169
186
|
type Tree = { entries: { sha256: string }[] };
|
|
@@ -172,7 +189,11 @@ const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
|
172
189
|
/** A data or delete file as DuckLake names it, under data_dir */
|
|
173
190
|
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
174
191
|
|
|
175
|
-
/**
|
|
192
|
+
/**
|
|
193
|
+
* What a live version reaches: the data files its metadata names, its tree's assets, whether rows
|
|
194
|
+
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
195
|
+
* files in `tables` are only those at the version's snapshot, and the metadata may name more
|
|
196
|
+
*/
|
|
176
197
|
const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
177
198
|
const dir = `versions/${id}/`;
|
|
178
199
|
const current = await record(store, id);
|
|
@@ -189,9 +210,12 @@ const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
|
189
210
|
if (files === null)
|
|
190
211
|
throw new Error(`version ${id} is live but has no files.json to say what it reads`);
|
|
191
212
|
const list = JSON.parse(files) as FileList;
|
|
192
|
-
const data =
|
|
193
|
-
|
|
194
|
-
|
|
213
|
+
const data = [
|
|
214
|
+
...(list.named ?? []),
|
|
215
|
+
...Object.values(list.tables).flatMap((t) =>
|
|
216
|
+
t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])]),
|
|
217
|
+
),
|
|
218
|
+
];
|
|
195
219
|
const fields = ((described ? (JSON.parse(described) as Described) : {}).tables ?? [])
|
|
196
220
|
.filter((t) => t.kind !== "view")
|
|
197
221
|
.flatMap((t) => t.fields ?? []);
|
|
@@ -201,6 +225,7 @@ const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
|
201
225
|
...(tree ? (JSON.parse(tree) as Tree).entries.map((e) => `assets/${e.sha256}`) : []),
|
|
202
226
|
],
|
|
203
227
|
assetRows: fields.some((f) => f.asset === true),
|
|
228
|
+
named: list.named !== undefined,
|
|
204
229
|
};
|
|
205
230
|
};
|
|
206
231
|
|
|
@@ -223,8 +248,10 @@ export type CollectOptions = {
|
|
|
223
248
|
/**
|
|
224
249
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
225
250
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
226
|
-
*
|
|
227
|
-
*
|
|
251
|
+
* metadata files wrote and never pushed. A version reaches every data file its metadata names, listed
|
|
252
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
253
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
254
|
+
* since reading those rows needs the metadata. The objects are listed before the versions are
|
|
228
255
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
229
256
|
* keeps what it reaches. Returns the keys deleted
|
|
230
257
|
*/
|
|
@@ -237,10 +264,12 @@ export const collect = async (
|
|
|
237
264
|
const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
|
|
238
265
|
const kept = new Set<string>();
|
|
239
266
|
let assetRows = false;
|
|
267
|
+
let unnamed = false;
|
|
240
268
|
const keep = async (ids: string[]) => {
|
|
241
269
|
for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
|
|
242
270
|
for (const key of r.keys) kept.add(key);
|
|
243
271
|
assetRows ||= r.assetRows;
|
|
272
|
+
unnamed ||= !r.named;
|
|
244
273
|
}
|
|
245
274
|
};
|
|
246
275
|
const read = await liveVersions(store);
|
|
@@ -249,7 +278,8 @@ export const collect = async (
|
|
|
249
278
|
placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
|
|
250
279
|
const planned = () =>
|
|
251
280
|
[
|
|
252
|
-
|
|
281
|
+
// A live version whose files.json lacks `named` may reach data files no files.json lists
|
|
282
|
+
...(unnamed ? [] : old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length)))),
|
|
253
283
|
...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
|
|
254
284
|
].sort();
|
|
255
285
|
if (dryRun) return planned();
|
package/src/repo.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Resolve a ref to a version and list the data files of its tables, following
|
|
2
|
-
// https://fdl.queria.io/spec/reading. Only files.json is read: the
|
|
2
|
+
// https://fdl.queria.io/spec/reading. Only files.json is read: the metadata is never opened
|
|
3
3
|
import { NotFoundError, type Store } from "./store.js";
|
|
4
4
|
|
|
5
5
|
/** The FDL major versions this reader knows */
|