@gscdump/engine 0.20.1 → 0.20.3

package/dist/rollups.d.mts

@@ -90,13 +90,14 @@ interface RollupDef {
      */
     dataSource: DataSource;
     /**
-     * Wall-clock millis when the runner started this rollup. Use for
-     * derived window cutoffs (e.g. trailing-28d boundary) so the SQL can
-     * inline a date literal and stay portable across DuckDB builds that
-     * don't bundle the ICU extension (Workers DuckDB, for one — CURRENT_DATE
-     * lives in ICU).
+     * UTC millis the trailing window anchors to — its inclusive END. Equals
+     * the newest synced/finalized data date when the runner is given
+     * `dataEndDate`, otherwise wall-clock build time. Builders derive window
+     * cutoffs from this (e.g. the trailing-28d boundary) and inline a date
+     * literal so the SQL stays portable across DuckDB builds without the ICU
+     * extension (Workers DuckDB — `CURRENT_DATE` lives in ICU).
      */
-    builtAt: number;
+    windowAnchorMs: number;
     /**
      * GSC search-type slice the runner was invoked for. Builders forward
      * this to every `engine.runSQL` call so the aggregated facts come
@@ -127,10 +128,13 @@ declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number, s
 interface RollupBucket {
   list: (opts: {
     prefix: string;
+    cursor?: string;
   }) => Promise<{
     objects: Array<{
       key: string;
     }>;
+    truncated?: boolean;
+    cursor?: string;
   }>;
   get: (key: string) => Promise<{
     text: () => Promise<string>;
@@ -153,6 +157,15 @@ interface RebuildRollupsOptions {
    * only tenants and explicit cross-type admin views.
    */
   searchType?: SearchType;
+  /**
+   * ISO date (`YYYY-MM-DD`) of the newest synced/finalized day. Trailing-
+   * window rollups (28d/90d) anchor their window END here instead of
+   * wall-clock build time, so a "last 28 days" rollup covers the 28 days of
+   * data that actually exist — not 28 days back from whenever the job ran,
+   * which would include GSC's 2-3 day empty tail. Omit for the legacy
+   * wall-clock behaviour.
+   */
+  dataEndDate?: string;
 }
 interface RebuildRollupResult {
   id: string;

package/dist/rollups.mjs

@@ -15,18 +15,25 @@ function rollupParquetKey(ctx, id, builtAt, searchType) {
 const ROLLUP_FILE_RE = /^(?<id>[a-z0-9_]+)__v(?<ts>\d+)\.json$/;
 async function readLatestRollup(bucket, ctx, id, searchType) {
 	const prefix = `${rollupPrefix(ctx, searchType)}/`;
-	const listing = await bucket.list({ prefix }).catch(() => null);
-	if (!listing) return null;
 	let newest = null;
-	for (const obj of listing.objects) {
-		const m = ROLLUP_FILE_RE.exec(obj.key.slice(prefix.length));
-		if (!m?.groups || m.groups.id !== id) continue;
-		const ts = Number(m.groups.ts);
-		if (!newest || ts > newest.ts) newest = {
-			ts,
-			key: obj.key
-		};
-	}
+	let cursor;
+	do {
+		const listing = await bucket.list({
+			prefix,
+			cursor
+		}).catch(() => null);
+		if (!listing) return null;
+		for (const obj of listing.objects) {
+			const m = ROLLUP_FILE_RE.exec(obj.key.slice(prefix.length));
+			if (!m?.groups || m.groups.id !== id) continue;
+			const ts = Number(m.groups.ts);
+			if (!newest || ts > newest.ts) newest = {
+				ts,
+				key: obj.key
+			};
+		}
+		cursor = listing.truncated ? listing.cursor : void 0;
+	} while (cursor !== void 0);
 	if (!newest) return null;
 	const obj = await bucket.get(newest.key).catch(() => null);
 	if (!obj) return null;
@@ -34,16 +41,18 @@ async function readLatestRollup(bucket, ctx, id, searchType) {
 }
 async function rebuildRollups(opts) {
 	const now = opts.now ?? (() => Date.now());
+	const dataEndMs = opts.dataEndDate !== void 0 ? isoDateToUtcMs(opts.dataEndDate) : null;
 	const results = [];
 	for (const def of opts.defs) {
 		const builtAt = now();
+		const windowAnchorMs = dataEndMs ?? builtAt;
 		const defSearchType = def.sliceOrthogonal === true ? void 0 : opts.searchType;
 		try {
 			const payload = await def.build({
 				engine: opts.engine,
 				ctx: opts.ctx,
 				dataSource: opts.dataSource,
-				builtAt,
+				windowAnchorMs,
 				...defSearchType !== void 0 ? { searchType: defSearchType } : {}
 			});
 			if (def.format === "parquet") {
@@ -108,6 +117,11 @@ async function rebuildRollups(opts) {
 	}
 	return results;
 }
+function isoDateToUtcMs(iso) {
+	const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(iso);
+	if (!m) throw new Error(`dataEndDate must be ISO YYYY-MM-DD, got: ${iso}`);
+	return Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
+}
 function utcDateMinusDays(at, days) {
 	const d = new Date(at - days * MS_PER_DAY);
 	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
@@ -343,13 +357,13 @@ const weeklyTotalsRollup = {
 const topPages28dRollup = {
 	id: "top_pages_28d",
 	windowDays: 28,
-	async build({ engine, ctx, builtAt, searchType }) {
-		const cutoff = utcDateMinusDays(builtAt, 28);
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
 		const partitions = partitionsInRange(await engine.listPartitions({
 			ctx,
 			table: "pages",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(builtAt, 0));
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
@@ -382,13 +396,13 @@ const topPages28dRollup = {
 const topCountries28dRollup = {
 	id: "top_countries_28d",
 	windowDays: 28,
-	async build({ engine, ctx, builtAt, searchType }) {
-		const cutoff = utcDateMinusDays(builtAt, 28);
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
 		const partitions = partitionsInRange(await engine.listPartitions({
 			ctx,
 			table: "countries",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(builtAt, 0));
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
@@ -421,13 +435,13 @@ const topCountries28dRollup = {
 const topKeywords28dRollup = {
 	id: "top_keywords_28d",
 	windowDays: 28,
-	async build({ engine, ctx, builtAt, searchType }) {
-		const cutoff = utcDateMinusDays(builtAt, 28);
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
 		const partitions = partitionsInRange(await engine.listPartitions({
 			ctx,
 			table: "keywords",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(builtAt, 0));
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
@@ -484,13 +498,13 @@ const topKeywords28dParquetRollup = {
 		}
 	],
 	parquetSortKey: ["clicks"],
-	async build({ engine, ctx, builtAt, searchType }) {
-		const cutoff = utcDateMinusDays(builtAt, 28);
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
 		const partitions = partitionsInRange(await engine.listPartitions({
 			ctx,
 			table: "keywords",
 			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(builtAt, 0));
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
@@ -568,7 +582,7 @@ const indexingHealthRollup = {
 	id: "indexing_health",
 	windowDays: 90,
 	sliceOrthogonal: true,
-	async build({ engine, ctx, dataSource, builtAt }) {
+	async build({ engine, ctx, dataSource, windowAnchorMs }) {
 		const key = inspectionParquetKey(ctx);
 		if (!await dataSource.head?.(key)) return { days: [] };
 		const sql = `
@@ -583,7 +597,7 @@ const indexingHealthRollup = {
         SUM(CASE WHEN CAST(richResultsVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
         SUM(CASE WHEN userCanonical IS NOT NULL AND googleCanonical IS NOT NULL AND CAST(userCanonical AS VARCHAR) <> CAST(googleCanonical AS VARCHAR) THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
       FROM read_parquet({{INSPECTIONS}}, union_by_name = true)
-      WHERE substr(CAST(inspectedAt AS VARCHAR), 1, 10) >= '${utcDateMinusDays(builtAt, 90)}'
+      WHERE substr(CAST(inspectedAt AS VARCHAR), 1, 10) >= '${utcDateMinusDays(windowAnchorMs, 90)}'
       GROUP BY 1
       ORDER BY 1
     `;
@@ -612,19 +626,19 @@ const indexPercentRollup = {
 	id: "index_percent",
 	windowDays: 90,
 	sliceOrthogonal: true,
-	async build({ engine, ctx, dataSource, builtAt, searchType }) {
+	async build({ engine, ctx, dataSource, windowAnchorMs, searchType }) {
 		const urlsKeys = await dataSource.list(sitemapUrlsIndexPrefix(ctx));
 		if (urlsKeys.length === 0) return {
 			totalSitemapUrls: 0,
 			days: []
 		};
-		const cutoff = utcDateMinusDays(builtAt, 90);
+		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
 		const factSearchType = searchType ?? "web";
 		const pagesPartitions = partitionsInRange(await engine.listPartitions({
 			ctx,
 			table: "pages",
 			searchType: factSearchType
-		}), cutoff, utcDateMinusDays(builtAt, 0));
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		const numerator = await engine.runSQL({
 			ctx,
 			table: "pages",
@@ -683,10 +697,10 @@ const sitemapHealthRollup = {
 	id: "sitemap_health",
 	windowDays: 90,
 	sliceOrthogonal: true,
-	async build({ dataSource, ctx, builtAt }) {
+	async build({ dataSource, ctx, windowAnchorMs }) {
 		const index = await createSitemapStore({ dataSource }).loadIndex(ctx);
 		const records = Object.values(index.records);
-		const cutoff = utcDateMinusDays(builtAt, 90);
+		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
 		const byDay = /* @__PURE__ */ new Map();
 		const feeds = [];
 		for (const r of records) {
@@ -727,10 +741,10 @@ const sitemapChanges28dRollup = {
 	id: "sitemap_changes_28d",
 	windowDays: 28,
 	sliceOrthogonal: true,
-	async build({ dataSource, ctx, builtAt }) {
+	async build({ dataSource, ctx, windowAnchorMs }) {
 		const store = createSitemapStore({ dataSource });
-		const from = utcDateMinusDays(builtAt, 28);
-		const to = utcDateMinusDays(builtAt, 0);
+		const from = utcDateMinusDays(windowAnchorMs, 28);
+		const to = utcDateMinusDays(windowAnchorMs, 0);
 		const counts = /* @__PURE__ */ new Map();
 		const addedTop = [];
 		const removedTop = [];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.20.1",
+  "version": "0.20.3",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.20.1",
-    "@gscdump/contracts": "0.20.1"
+    "gscdump": "0.20.3",
+    "@gscdump/contracts": "0.20.3"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",