@gscdump/engine 0.20.0 → 0.20.2

package/dist/rollups.d.mts

@@ -127,10 +127,13 @@ declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number, s
 interface RollupBucket {
   list: (opts: {
     prefix: string;
+    cursor?: string;
   }) => Promise<{
     objects: Array<{
       key: string;
     }>;
+    truncated?: boolean;
+    cursor?: string;
   }>;
   get: (key: string) => Promise<{
     text: () => Promise<string>;

package/dist/rollups.mjs

@@ -15,18 +15,25 @@ function rollupParquetKey(ctx, id, builtAt, searchType) {
 const ROLLUP_FILE_RE = /^(?<id>[a-z0-9_]+)__v(?<ts>\d+)\.json$/;
 async function readLatestRollup(bucket, ctx, id, searchType) {
 	const prefix = `${rollupPrefix(ctx, searchType)}/`;
-	const listing = await bucket.list({ prefix }).catch(() => null);
-	if (!listing) return null;
 	let newest = null;
-	for (const obj of listing.objects) {
-		const m = ROLLUP_FILE_RE.exec(obj.key.slice(prefix.length));
-		if (!m?.groups || m.groups.id !== id) continue;
-		const ts = Number(m.groups.ts);
-		if (!newest || ts > newest.ts) newest = {
-			ts,
-			key: obj.key
-		};
-	}
+	let cursor;
+	do {
+		const listing = await bucket.list({
+			prefix,
+			cursor
+		}).catch(() => null);
+		if (!listing) return null;
+		for (const obj of listing.objects) {
+			const m = ROLLUP_FILE_RE.exec(obj.key.slice(prefix.length));
+			if (!m?.groups || m.groups.id !== id) continue;
+			const ts = Number(m.groups.ts);
+			if (!newest || ts > newest.ts) newest = {
+				ts,
+				key: obj.key
+			};
+		}
+		cursor = listing.truncated ? listing.cursor : void 0;
+	} while (cursor !== void 0);
 	if (!newest) return null;
 	const obj = await bucket.get(newest.key).catch(() => null);
 	if (!obj) return null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.20.0",
+  "version": "0.20.2",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.20.0",
-    "gscdump": "0.20.0"
+    "@gscdump/contracts": "0.20.2",
+    "gscdump": "0.20.2"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
@@ -178,7 +178,7 @@
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.25.8",
     "hyparquet-writer": "^0.15.1",
-    "tsx": "^4.22.2",
+    "tsx": "^4.22.3",
     "vitest": "^4.1.6"
   },
   "scripts": {