@queria-io/fdl-js 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/maintain.d.ts +3 -1
- package/dist/maintain.js +18 -5
- package/dist/maintain.js.map +1 -1
- package/package.json +14 -3
- package/src/maintain.ts +22 -7
package/README.md
CHANGED
|
@@ -69,6 +69,6 @@ pnpm check
|
|
|
69
69
|
pnpm build # dist/
|
|
70
70
|
```
|
|
71
71
|
|
|
72
|
-
Releases are made from the commit messages ([Conventional Commits](https://www.conventionalcommits.org/)). Each push to `main` runs the Release workflow: when the commits since the last tag include a `fix:` it publishes a patch version, a `feat:` a minor one, and a breaking change (`!`
|
|
72
|
+
Releases are made from the commit messages ([Conventional Commits](https://www.conventionalcommits.org/)). Each push to `main` runs the Release workflow: when the commits since the last tag include a `fix:` it publishes a patch version, a `feat:` a minor one, and a breaking change (`!` after the type, or a `BREAKING CHANGE:` footer) a minor one. It publishes to npm, tags the commit `v<version>` and makes a GitHub release. `docs:`, `chore:`, `test:` and the like release nothing.
|
|
73
73
|
|
|
74
74
|
`test/fixture/repo` is a small repository made by the `fdl` command: `FDL=fdl test/make-fixture.sh` remakes it.
|
package/dist/maintain.d.ts
CHANGED
|
@@ -58,7 +58,9 @@ export type CollectOptions = {
|
|
|
58
58
|
/**
|
|
59
59
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
60
60
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
61
|
-
* catalogs wrote and never pushed.
|
|
61
|
+
* catalogs wrote and never pushed. A version reaches every data file its catalog names, listed
|
|
62
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
63
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
62
64
|
* since reading those rows needs the catalog. The objects are listed before the versions are
|
|
63
65
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
64
66
|
* keeps what it reaches. Returns the keys deleted
|
package/dist/maintain.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
2
|
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
3
|
// live version reaches once past the grace period (as `fdl gc` does). Catalogs are never
|
|
4
|
-
// opened: what a version
|
|
4
|
+
// opened: what a version reaches comes from `named` in its files.json
|
|
5
5
|
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
6
|
import { r2Store, retryOnce } from "./store.js";
|
|
7
7
|
/** Up to this many keys go in one R2 delete */
|
|
@@ -101,7 +101,11 @@ export const expire = async (store, { keep, ref = "refs/heads/main", since, now
|
|
|
101
101
|
const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
102
102
|
/** A data or delete file as DuckLake names it, under data_dir */
|
|
103
103
|
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
104
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* What a live version reaches: the data files its catalog names, its tree's assets, whether rows
|
|
106
|
+
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
107
|
+
* files in `tables` are only those at the version's snapshot, and the catalog may name more
|
|
108
|
+
*/
|
|
105
109
|
const reachedBy = async (store, dataDir, id) => {
|
|
106
110
|
const dir = `versions/${id}/`;
|
|
107
111
|
const current = await record(store, id);
|
|
@@ -119,7 +123,10 @@ const reachedBy = async (store, dataDir, id) => {
|
|
|
119
123
|
if (files === null)
|
|
120
124
|
throw new Error(`version ${id} is live but has no files.json to say what it reads`);
|
|
121
125
|
const list = JSON.parse(files);
|
|
122
|
-
const data =
|
|
126
|
+
const data = [
|
|
127
|
+
...(list.named ?? []),
|
|
128
|
+
...Object.values(list.tables).flatMap((t) => t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])])),
|
|
129
|
+
];
|
|
123
130
|
const fields = ((described ? JSON.parse(described) : {}).tables ?? [])
|
|
124
131
|
.filter((t) => t.kind !== "view")
|
|
125
132
|
.flatMap((t) => t.fields ?? []);
|
|
@@ -129,6 +136,7 @@ const reachedBy = async (store, dataDir, id) => {
|
|
|
129
136
|
...(tree ? JSON.parse(tree).entries.map((e) => `assets/${e.sha256}`) : []),
|
|
130
137
|
],
|
|
131
138
|
assetRows: fields.some((f) => f.asset === true),
|
|
139
|
+
named: list.named !== undefined,
|
|
132
140
|
};
|
|
133
141
|
};
|
|
134
142
|
/** The versions without expired.json, from one listing of versions/ */
|
|
@@ -141,7 +149,9 @@ const liveVersions = async (store) => {
|
|
|
141
149
|
/**
|
|
142
150
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
143
151
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
144
|
-
* catalogs wrote and never pushed.
|
|
152
|
+
* catalogs wrote and never pushed. A version reaches every data file its catalog names, listed
|
|
153
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
154
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
145
155
|
* since reading those rows needs the catalog. The objects are listed before the versions are
|
|
146
156
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
147
157
|
* keeps what it reaches. Returns the keys deleted
|
|
@@ -152,18 +162,21 @@ export const collect = async (store, { graceMs, now = new Date(), dryRun = false
|
|
|
152
162
|
const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
|
|
153
163
|
const kept = new Set();
|
|
154
164
|
let assetRows = false;
|
|
165
|
+
let unnamed = false;
|
|
155
166
|
const keep = async (ids) => {
|
|
156
167
|
for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
|
|
157
168
|
for (const key of r.keys)
|
|
158
169
|
kept.add(key);
|
|
159
170
|
assetRows ||= r.assetRows;
|
|
171
|
+
unnamed ||= !r.named;
|
|
160
172
|
}
|
|
161
173
|
};
|
|
162
174
|
const read = await liveVersions(store);
|
|
163
175
|
await keep(read);
|
|
164
176
|
const old = (placed) => placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
|
|
165
177
|
const planned = () => [
|
|
166
|
-
|
|
178
|
+
// A live version whose files.json lacks `named` may reach data files no files.json lists
|
|
179
|
+
...(unnamed ? [] : old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length)))),
|
|
167
180
|
...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
|
|
168
181
|
].sort();
|
|
169
182
|
if (dryRun)
|
package/dist/maintain.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F,yFAAyF;AACzF,
|
|
1
|
+
{"version":3,"file":"maintain.js","sourceRoot":"","sources":["../src/maintain.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAC3F,yFAAyF;AACzF,sEAAsE;AACtE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnF,OAAO,EAA2B,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAuBzE,+CAA+C;AAC/C,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,MAAM,MAAM,GAAG,CAAI,KAAU,EAAE,IAAY,EAAS,EAAE,CACpD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9D,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CACtC,CAAC;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,MAAkB,EAClB,MAAc,EACd,SAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EACtB,EAAE;IACnB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAqB,EAAE;QACpD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAA0B,CAAC;QAC/B,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO;gBAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAAE,OAAO,MAAM,CAAC;YACpC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACxB,CAAC;IACH,CAAC,CAAC;IACF,OAAO;QACL,GAAG,MAAM;QACT,IAAI;QACJ,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;YACvB,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;gBAC5C,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAwB,EAAE;IACtE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC7D,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,EAAE,CACnD,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;AAE7D;;;GAGG;AACH,MAAM,IAAI,GAAG,KAAK,EAAE,KAAsB,EAAwB,EAAE;IAClE,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACf,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;QACvC,IAAI,CAAC,YAAY,QAAQ;YAAE,OAAO,IAAI,CAAC;QACvC,MAAM,CAAC,CAAC;IACV,CAAC,CAAC,CACH,CACF,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,SAAS,GAAG,KAAK,EAAE,KAAsB,EAAE,EAAU,EAAE,GAAS,EAAE,EAAE;IACxE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC,GAAG,CACb,GAAG,GAAG,cAAc,EACpB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,IAAI,CAC9E,CAAC;IACJ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,MAAM,CACL,CAAC,GAAG,EAAE,EAAE,CACN,GAAG,KAAK,GAAG,GAAG,iBAAiB;QAC/B,GAAG,KAAK,GAAG,GAAG,YAAY;QAC1B,GAAG,KAAK,GAAG,GAAG,WAAW;QACzB,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC;QACjC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,WAAW,CAAC,CACpC,CAAC;IACJ,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC,CAAC;AAaF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EACzB,KAAsB,EACtB,EAAE,IAAI,EAAE,GAAG,GAAG,iBAAiB,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAiB,EACtE,EAAE;IACrB,IAAI,IAAI,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,EAAE,EAAsB,EAAE,QAAgB,EAAqB,EAAE;QACjF,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;QAC5E,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,MAAM;YAAE,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACxD,4EAA4E;QAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACzB,CAAC,CAAC;AAUF,MAAM,MAAM,GAAG,wBAAwB,CAAC;AACxC,iEAAiE;AACjE,MAAM,SAAS,GAAG,gCAAgC,CAAC;AAEnD;;;;GAIG;AACH,MAAM,SAAS,GAAG,KAAK,EAAE,KAAY,EAAE,OAAe,EAAE,EAAU,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,IAAwB,EAAE,QAAgB,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,IAAI,QAAQ,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,6BAA6B,KAAK,EAAE,CAAC,CAAC;QAC9F,OAAO,GAAG,GAAG,KAAK,CAAC;IACrB,CAAC,CAAC;IACF,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;KACnD,CAAC,CAAC;IACH,IAAI,KAAK,KAAK,IAAI;QAChB,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,qDAAqD,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,CAAC;IAC3C,MAAM,IAAI,GAAG;QACX,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzE;KACF,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;SAClF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;SAChC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAClC,OAAO;QACL,IAAI,EAAE;YACJ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrF;QACD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC;QAC/C,KAAK,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS;KAChC,CAAC;AACJ,CAAC,CAAC;AAEF,uEAAuE;AACvE,MAAM,YAAY,GAAG,KAAK,EAAE,KAAsB,EAAqB,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SAClE,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,CAAC;AAC/F,CAAC,CAAC;AAUF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAsB,EACtB,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,MAAM,GAAG,KAAK,EAAkB,EAC1C,EAAE;IACrB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;QACnC,KAAK,MAAM,CAAC,IAAI,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC;YAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB;QACE,yFAAyF;QACzF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxF,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;KACpE,CAAC,IAAI,EAAE,CAAC;IACX,IAAI,MAAM;QAAE,OAAO,OAAO,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@queria-io/fdl-js",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare-workers",
|
|
@@ -62,6 +62,7 @@
|
|
|
62
62
|
[
|
|
63
63
|
"@semantic-release/commit-analyzer",
|
|
64
64
|
{
|
|
65
|
+
"preset": "conventionalcommits",
|
|
65
66
|
"releaseRules": [
|
|
66
67
|
{
|
|
67
68
|
"breaking": true,
|
|
@@ -70,8 +71,18 @@
|
|
|
70
71
|
]
|
|
71
72
|
}
|
|
72
73
|
],
|
|
73
|
-
|
|
74
|
-
|
|
74
|
+
[
|
|
75
|
+
"@semantic-release/release-notes-generator",
|
|
76
|
+
{
|
|
77
|
+
"preset": "conventionalcommits"
|
|
78
|
+
}
|
|
79
|
+
],
|
|
80
|
+
[
|
|
81
|
+
"@semantic-release/npm",
|
|
82
|
+
{
|
|
83
|
+
"npmPublish": false
|
|
84
|
+
}
|
|
85
|
+
],
|
|
75
86
|
[
|
|
76
87
|
"@semantic-release/github",
|
|
77
88
|
{
|
package/src/maintain.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Retention for a repository's host, following https://fdl.queria.io/spec/data-files:
|
|
2
2
|
// take older versions out of retention (as `fdl expire` does) and delete the data files no
|
|
3
3
|
// live version reaches once past the grace period (as `fdl gc` does). Catalogs are never
|
|
4
|
-
// opened: what a version
|
|
4
|
+
// opened: what a version reaches comes from `named` in its files.json
|
|
5
5
|
import { FdlError, isFileName, isVersionId, readConfig, resolve } from "./repo.js";
|
|
6
6
|
import { type R2Like, type Store, r2Store, retryOnce } from "./store.js";
|
|
7
7
|
|
|
@@ -164,6 +164,8 @@ export const expire = async (
|
|
|
164
164
|
|
|
165
165
|
type FileList = {
|
|
166
166
|
tables: { [table: string]: { files: { path: string; deletes?: { path: string } }[] } };
|
|
167
|
+
/** Every data file the catalog names, at any snapshot. Absent in files.json written before it */
|
|
168
|
+
named?: string[];
|
|
167
169
|
};
|
|
168
170
|
type Described = { tables?: { kind?: string; fields?: { asset?: boolean }[] }[] };
|
|
169
171
|
type Tree = { entries: { sha256: string }[] };
|
|
@@ -172,7 +174,11 @@ const SHA256 = /^assets\/[0-9a-f]{64}$/;
|
|
|
172
174
|
/** A data or delete file as DuckLake names it, under data_dir */
|
|
173
175
|
const DATA_FILE = /(^|\/)ducklake-[^/]+\.parquet$/;
|
|
174
176
|
|
|
175
|
-
/**
|
|
177
|
+
/**
|
|
178
|
+
* What a live version reaches: the data files its catalog names, its tree's assets, whether rows
|
|
179
|
+
* hold assets, and whether files.json names every data file (`named`). Without `named`, the
|
|
180
|
+
* files in `tables` are only those at the version's snapshot, and the catalog may name more
|
|
181
|
+
*/
|
|
176
182
|
const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
177
183
|
const dir = `versions/${id}/`;
|
|
178
184
|
const current = await record(store, id);
|
|
@@ -189,9 +195,12 @@ const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
|
189
195
|
if (files === null)
|
|
190
196
|
throw new Error(`version ${id} is live but has no files.json to say what it reads`);
|
|
191
197
|
const list = JSON.parse(files) as FileList;
|
|
192
|
-
const data =
|
|
193
|
-
|
|
194
|
-
|
|
198
|
+
const data = [
|
|
199
|
+
...(list.named ?? []),
|
|
200
|
+
...Object.values(list.tables).flatMap((t) =>
|
|
201
|
+
t.files.flatMap((f) => [f.path, ...(f.deletes ? [f.deletes.path] : [])]),
|
|
202
|
+
),
|
|
203
|
+
];
|
|
195
204
|
const fields = ((described ? (JSON.parse(described) as Described) : {}).tables ?? [])
|
|
196
205
|
.filter((t) => t.kind !== "view")
|
|
197
206
|
.flatMap((t) => t.fields ?? []);
|
|
@@ -201,6 +210,7 @@ const reachedBy = async (store: Store, dataDir: string, id: string) => {
|
|
|
201
210
|
...(tree ? (JSON.parse(tree) as Tree).entries.map((e) => `assets/${e.sha256}`) : []),
|
|
202
211
|
],
|
|
203
212
|
assetRows: fields.some((f) => f.asset === true),
|
|
213
|
+
named: list.named !== undefined,
|
|
204
214
|
};
|
|
205
215
|
};
|
|
206
216
|
|
|
@@ -223,7 +233,9 @@ export type CollectOptions = {
|
|
|
223
233
|
/**
|
|
224
234
|
* Deletes the data files and assets no live version reaches, written longer ago than the grace
|
|
225
235
|
* period, as `fdl gc` does. These are files only expired versions reached, and files working
|
|
226
|
-
* catalogs wrote and never pushed.
|
|
236
|
+
* catalogs wrote and never pushed. A version reaches every data file its catalog names, listed
|
|
237
|
+
* in `named` of its files.json; while a live version's files.json lacks `named`, no data file is
|
|
238
|
+
* deleted. Assets are kept while a live version's rows hold assets,
|
|
227
239
|
* since reading those rows needs the catalog. The objects are listed before the versions are
|
|
228
240
|
* read, and the versions are listed again just before deleting, so a version pushed meanwhile
|
|
229
241
|
* keeps what it reaches. Returns the keys deleted
|
|
@@ -237,10 +249,12 @@ export const collect = async (
|
|
|
237
249
|
const [data, assets] = await Promise.all([store.list(dataDir), store.list("assets/")]);
|
|
238
250
|
const kept = new Set<string>();
|
|
239
251
|
let assetRows = false;
|
|
252
|
+
let unnamed = false;
|
|
240
253
|
const keep = async (ids: string[]) => {
|
|
241
254
|
for (const r of await Promise.all(ids.map((id) => reachedBy(store, dataDir, id)))) {
|
|
242
255
|
for (const key of r.keys) kept.add(key);
|
|
243
256
|
assetRows ||= r.assetRows;
|
|
257
|
+
unnamed ||= !r.named;
|
|
244
258
|
}
|
|
245
259
|
};
|
|
246
260
|
const read = await liveVersions(store);
|
|
@@ -249,7 +263,8 @@ export const collect = async (
|
|
|
249
263
|
placed.filter((p) => p.uploaded.getTime() < before && !kept.has(p.key)).map((p) => p.key);
|
|
250
264
|
const planned = () =>
|
|
251
265
|
[
|
|
252
|
-
|
|
266
|
+
// A live version whose files.json lacks `named` may reach data files no files.json lists
|
|
267
|
+
...(unnamed ? [] : old(data).filter((key) => DATA_FILE.test(key.slice(dataDir.length)))),
|
|
253
268
|
...(assetRows ? [] : old(assets).filter((key) => SHA256.test(key))),
|
|
254
269
|
].sort();
|
|
255
270
|
if (dryRun) return planned();
|