@queria-io/fdl-js 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/maintain.d.ts +2 -2
- package/dist/maintain.js +20 -9
- package/dist/maintain.js.map +1 -1
- package/dist/repo.js +1 -1
- package/dist/repo.js.map +1 -1
- package/package.json +1 -1
- package/src/maintain.ts +25 -10
- package/src/repo.ts +1 -1
package/dist/maintain.d.ts
CHANGED
|
@@ -58,10 +58,10 @@ export type CollectOptions = {
|
|
|
58
58
|
/**
|
|
59
59
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
60
60
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
61
|
-
*
|
|
61
|
+
* metadata files wrote and never pushed. A version reaches every data file its metadata names, listed
|
|
62
62
|
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
63
63
|
* deleted. Assets are kept while a live version's rows hold assets,
|
|
64
|
-
* since reading those rows needs the
|
|
64
|
+
* since reading those rows needs the metadata. The objects are listed before the versions are
|
|
65
65
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
66
66
|
* keeps what it reaches. Returns the keys deleted
|
|
67
67
|
*/
|
package/dist/maintain.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
2
|
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
|
-
// live version reaches once past the grace period (as `fdl gc` does).
|
|
3
|
+
// live version reaches once past the grace period (as `fdl gc` does). Metadata files are never
|
|
4
4
|
// opened: what a version reaches comes from `named` in its files.json
|
|
5
5
|
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
6
|
import { r2Store, retryOnce } from "./store.js";
|
|
@@ -54,16 +54,27 @@ const held = async (store) => {
|
|
|
54
54
|
})));
|
|
55
55
|
return new Set(targets.filter((id) => id !== null));
|
|
56
56
|
};
|
|
57
|
-
/**
|
|
58
|
-
|
|
57
|
+
/**
|
|
58
|
+
* The metadata files to delete: the one the record names (`metadata`, or `catalog` in records
|
|
59
|
+
* written by fdl 0.26 and earlier), or both default names when the record names none it can use
|
|
60
|
+
*/
|
|
61
|
+
const metadataPaths = (head) => {
|
|
62
|
+
const path = (head.metadata ?? head.catalog)?.path;
|
|
63
|
+
return isFileName(path) ? [path] : ["metadata.db", "ducklake.duckdb"];
|
|
64
|
+
};
|
|
65
|
+
/** Takes one version out of retention: the mark first, then its metadata and indexes */
|
|
66
|
+
const expireOne = async (store, id, head, now) => {
|
|
59
67
|
const dir = `versions/${id}/`;
|
|
68
|
+
const metadata = new Set(metadataPaths(head).map((path) => dir + path));
|
|
60
69
|
if (!(await isExpired(store, id)))
|
|
61
70
|
await store.put(`${dir}expired.json`, `${JSON.stringify({ expired_at: now.toISOString(), reason: "retention" })}\n`);
|
|
62
71
|
const indexed = (await store.list(dir))
|
|
63
72
|
.map((p) => p.key)
|
|
64
|
-
.filter((key) => key
|
|
73
|
+
.filter((key) => metadata.has(key) ||
|
|
65
74
|
key === `${dir}files.json` ||
|
|
66
75
|
key === `${dir}tree.json` ||
|
|
76
|
+
key.startsWith(`${dir}metadata/`) ||
|
|
77
|
+
// Other DuckLake formats, as fdl 0.26 and earlier lay them out
|
|
67
78
|
key.startsWith(`${dir}catalogs/`) ||
|
|
68
79
|
key.startsWith(`${dir}profiles/`));
|
|
69
80
|
await store.delete(indexed);
|
|
@@ -90,7 +101,7 @@ export const expire = async (store, { keep, ref = "refs/heads/main", since, now
|
|
|
90
101
|
const recent = since !== undefined && new Date(current.created_at) >= since;
|
|
91
102
|
const expired = position >= keep && !recent && !kept.has(id);
|
|
92
103
|
if (expired && !dryRun)
|
|
93
|
-
await expireOne(store, id, now);
|
|
104
|
+
await expireOne(store, id, current, now);
|
|
94
105
|
// A parent in any other form is not followed: the ID becomes part of a path
|
|
95
106
|
const parent = current.parents[0];
|
|
96
107
|
const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
|
|
@@ -102,9 +113,9 @@ const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
|
102
113
|
/** A data or delete file as DuckLake names it, under data_dir */
|
|
103
114
|
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
104
115
|
/**
|
|
105
|
-
* What a live version reaches: the data files its
|
|
116
|
+
* What a live version reaches: the data files its metadata names, its tree's assets, whether rows
|
|
106
117
|
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
107
|
-
* files in `tables` are only those at the version's snapshot, and the
|
|
118
|
+
* files in `tables` are only those at the version's snapshot, and the metadata may name more
|
|
108
119
|
*/
|
|
109
120
|
const reachedBy = async (store, dataDir, id) => {
|
|
110
121
|
const dir = `versions/${id}/`;
|
|
@@ -149,10 +160,10 @@ const liveVersions = async (store) => {
|
|
|
149
160
|
/**
|
|
150
161
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
151
162
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
152
|
-
*
|
|
163
|
+
* metadata files wrote and never pushed. A version reaches every data file its metadata names, listed
|
|
153
164
|
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
154
165
|
* deleted. Assets are kept while a live version's rows hold assets,
|
|
155
|
-
* since reading those rows needs the
|
|
166
|
+
* since reading those rows needs the metadata. The objects are listed before the versions are
|
|
156
167
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
157
168
|
* keeps what it reaches. Returns the keys deleted
|
|
158
169
|
*/
|
package/dist/maintain.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F
|
|
1
|
+
{"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F,+FAA+F;AAC/F,sEAAsE;AACtE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnF,OAAO,EAA2B,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAuBzE,+CAA+C;AAC/C,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,MAAM,MAAM,GAAG,CAAI,KAAU,EAAE,IAAY,EAAS,EAAE,CACpD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9D,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CACtC,CAAC;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,MAAkB,EAClB,MAAc,EACd,SAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EACtB,EAAE;IACnB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAqB,EAAE;QACpD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAA0B,CAAC;QAC/B,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO;gBAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAAE,OAAO,MAAM,CAAC;YACpC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACxB,CAAC;IACH,CAAC,CAAC;IACF,OAAO;QACL,GAAG,MAAM;QACT,IAAI;QACJ,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;YACvB,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;gBAC5C,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAcF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAwB,EAAE;IACtE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC7D,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,EAAE,CACnD,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;AAE7D;;;GAGG;AACH,MAAM,IAAI,GAAG,KAAK,EAAE,KAAsB,EAAwB,EAAE;IAClE,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACf,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;QACvC,IAAI,CAAC,YAAY,QAAQ;YAAE,OAAO,IAAI,CAAC;QACvC,MAAM,CAAC,CAAC;IACV,CAAC,CAAC,CACH,CACF,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG,CAAC,IAAU,EAAY,EAAE;IAC7C,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC;IACnD,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAsB,EAAE,EAAU,EAAE,IAAU,EAAE,GAAS,EAAE,EAAE;IACpF,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC;IACxE,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC,GAAG,CACb,GAAG,GAAG,cAAc,EACpB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,IAAI,CAC9E,CAAC;IACJ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,MAAM,CACL,CAAC,GAAG,EAAE,EAAE,CACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;QACjB,GAAG,KAAK,GAAG,GAAG,YAAY;QAC1B,GAAG,KAAK,GAAG,GAAG,WAAW;QACzB,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC;QACjC,+DAA+D;QAC/D,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC;QACjC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC,CACpC,CAAC;IACJ,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC,CAAC;AAaF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EACzB,KAAsB,EACtB,EAAE,IAAI,EAAE,GAAG,GAAG,iBAAiB,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAiB,EACtE,EAAE;IACrB,IAAI,IAAI,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,EAAE,EAAsB,EAAE,QAAgB,EAAqB,EAAE;QACjF,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;QAC5E,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,MAAM;YAAE,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;QACjE,4EAA4E;QAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACzB,CAAC,CAAC;AAUF,MAAM,MAAM,GAAG,wBAAwB,CAAC;AACxC,iEAAiE;AACjE,MAAM,SAAS,GAAG,gCAAgC,CAAC;AAEnD;;;;GAIG;AACH,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,OAAe,EAAE,EAAU,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,IAAwB,EAAE,QAAgB,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,IAAI,QAAQ,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,6BAA6B,KAAK,EAAE,CAAC,CAAC;QAC9F,OAAO,GAAG,GAAG,KAAK,CAAC;IACrB,CAAC,CAAC;IACF,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;KACnD,CAAC,CAAC;IACH,IAAI,KAAK,KAAK,IAAI;QAChB,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,qDAAqD,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,CAAC;IAC3C,MAAM,IAAI,GAAG;QACX,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzE;KACF,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;SAClF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;SAChC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAClC,OAAO;QACL,IAAI,EAAE;YACJ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrF;QACD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC;QAC/C,KAAK,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS;KAChC,CAAC;AACJ,CAAC,CAAC;AAEF,uEAAuE;AACvE,MAAM,YAAY,GAAG,KAAK,EAAE,KAAsB,EAAqB,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SAClE,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,CAAC;AAC/F,CAAC,CAAC;AAUF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAsB,EACtB,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAkB,EAC1C,EAAE;IACrB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;QACnC,KAAK,MAAM,CAAC,IAAI,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC;YAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB;QACE,yFAAyF;QACzF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxF,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;KACpE,CAAC,IAAI,EAAE,CAAC;IACX,IAAI,MAAM;QAAE,OAAO,OAAO,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}
|
package/dist/repo.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Resolve a ref to a version and list the data files of its tables, following
|
|
2
|
-
// https://fdl.queria.io/spec/reading. Only files.json is read: the
|
|
2
|
+
// https://fdl.queria.io/spec/reading. Only files.json is read: the metadata is never opened
|
|
3
3
|
import { NotFoundError } from "./store.js";
|
|
4
4
|
/** The FDL major versions this reader knows */
|
|
5
5
|
const KNOWN_MAJOR = "0";
|
package/dist/repo.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,
|
|
1
|
+
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4FAA4F;AAC5F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiCtC,MAAM,UAAU,GAAG,uEAAuE,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC7D,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,KAAK,EAAE;IACZ,KAAK,KAAK,GAAG;IACb,KAAK,KAAK,IAAI;IACd,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACpB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExB,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO;QACL,GAAG,GAAG;QACN,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACpB,QAAQ,EAAE,OAAO,KAAK,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG;KAC5E,CAAC;AACJ,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACxB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACf,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
|
package/package.json
CHANGED
package/src/maintain.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
2
|
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
|
-
// live version reaches once past the grace period (as `fdl gc` does).
|
|
3
|
+
// live version reaches once past the grace period (as `fdl gc` does). Metadata files are never
|
|
4
4
|
// opened: what a version reaches comes from `named` in its files.json
|
|
5
5
|
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
6
|
import { type R2Like, type Store, r2Store, retryOnce } from "./store.js";
|
|
@@ -74,6 +74,9 @@ type Head = {
|
|
|
74
74
|
files?: { path: string };
|
|
75
75
|
tree?: { path: string };
|
|
76
76
|
dataset?: { path: string };
|
|
77
|
+
/** The metadata file. Records written by fdl 0.26 and earlier name it `catalog` */
|
|
78
|
+
metadata?: { path?: unknown };
|
|
79
|
+
catalog?: { path?: unknown };
|
|
77
80
|
};
|
|
78
81
|
|
|
79
82
|
const record = async (store: Store, id: string): Promise<Head | null> => {
|
|
@@ -101,9 +104,19 @@ const held = async (store: MaintainedStore): Promise<Set<string>> => {
|
|
|
101
104
|
return new Set(targets.filter((id): id is string => id !== null));
|
|
102
105
|
};
|
|
103
106
|
|
|
104
|
-
/**
|
|
105
|
-
|
|
107
|
+
/**
|
|
108
|
+
* The metadata files to delete: the one the record names (`metadata`, or `catalog` in records
|
|
109
|
+
* written by fdl 0.26 and earlier), or both default names when the record names none it can use
|
|
110
|
+
*/
|
|
111
|
+
const metadataPaths = (head: Head): string[] => {
|
|
112
|
+
const path = (head.metadata ?? head.catalog)?.path;
|
|
113
|
+
return isFileName(path) ? [path] : ["metadata.db", "ducklake.duckdb"];
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/** Takes one version out of retention: the mark first, then its metadata and indexes */
|
|
117
|
+
const expireOne = async (store: MaintainedStore, id: string, head: Head, now: Date) => {
|
|
106
118
|
const dir = `versions/${id}/`;
|
|
119
|
+
const metadata = new Set(metadataPaths(head).map((path) => dir + path));
|
|
107
120
|
if (!(await isExpired(store, id)))
|
|
108
121
|
await store.put(
|
|
109
122
|
`${dir}expired.json`,
|
|
@@ -113,9 +126,11 @@ const expireOne = async (store: MaintainedStore, id: string, now: Date) => {
|
|
|
113
126
|
.map((p) => p.key)
|
|
114
127
|
.filter(
|
|
115
128
|
(key) =>
|
|
116
|
-
key
|
|
129
|
+
metadata.has(key) ||
|
|
117
130
|
key === `${dir}files.json` ||
|
|
118
131
|
key === `${dir}tree.json` ||
|
|
132
|
+
key.startsWith(`${dir}metadata/`) ||
|
|
133
|
+
// Other DuckLake formats, as fdl 0.26 and earlier lay them out
|
|
119
134
|
key.startsWith(`${dir}catalogs/`) ||
|
|
120
135
|
key.startsWith(`${dir}profiles/`),
|
|
121
136
|
);
|
|
@@ -153,7 +168,7 @@ export const expire = async (
|
|
|
153
168
|
if (current === null) return [];
|
|
154
169
|
const recent = since !== undefined && new Date(current.created_at) >= since;
|
|
155
170
|
const expired = position >= keep && !recent && !kept.has(id);
|
|
156
|
-
if (expired && !dryRun) await expireOne(store, id, now);
|
|
171
|
+
if (expired && !dryRun) await expireOne(store, id, current, now);
|
|
157
172
|
// A parent in any other form is not followed: the ID becomes part of a path
|
|
158
173
|
const parent = current.parents[0];
|
|
159
174
|
const rest = await walk(isVersionId(parent) ? parent : undefined, position + 1);
|
|
@@ -164,7 +179,7 @@ export const expire = async (
|
|
|
164
179
|
|
|
165
180
|
type FileList = {
|
|
166
181
|
tables: { [table: string]: { files: { path: string; deletes?: { path: string } }[] } };
|
|
167
|
-
/** Every data file the
|
|
182
|
+
/** Every data file the metadata names, at any snapshot. Absent in files.json written before it */
|
|
168
183
|
named?: string[];
|
|
169
184
|
};
|
|
170
185
|
type Described = { tables?: { kind?: string; fields?: { asset?: boolean }[] }[] };
|
|
@@ -175,9 +190,9 @@ const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
|
175
190
|
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
176
191
|
|
|
177
192
|
/**
|
|
178
|
-
* What a live version reaches: the data files its
|
|
193
|
+
* What a live version reaches: the data files its metadata names, its tree's assets, whether rows
|
|
179
194
|
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
180
|
-
* files in `tables` are only those at the version's snapshot, and the
|
|
195
|
+
* files in `tables` are only those at the version's snapshot, and the metadata may name more
|
|
181
196
|
*/
|
|
182
197
|
const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
183
198
|
const dir = `versions/${id}/`;
|
|
@@ -233,10 +248,10 @@ export type CollectOptions = {
|
|
|
233
248
|
/**
|
|
234
249
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
235
250
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
236
|
-
*
|
|
251
|
+
* metadata files wrote and never pushed. A version reaches every data file its metadata names, listed
|
|
237
252
|
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
238
253
|
* deleted. Assets are kept while a live version's rows hold assets,
|
|
239
|
-
* since reading those rows needs the
|
|
254
|
+
* since reading those rows needs the metadata. The objects are listed before the versions are
|
|
240
255
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
241
256
|
* keeps what it reaches. Returns the keys deleted
|
|
242
257
|
*/
|
package/src/repo.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Resolve a ref to a version and list the data files of its tables, following
|
|
2
|
-
// https://fdl.queria.io/spec/reading. Only files.json is read: the
|
|
2
|
+
// https://fdl.queria.io/spec/reading. Only files.json is read: the metadata is never opened
|
|
3
3
|
import { NotFoundError, type Store } from "./store.js";
|
|
4
4
|
|
|
5
5
|
/** The FDL major versions this reader knows */
|