@gscdump/engine 3.8.0 → 4.0.1

package/dist/THIRD-PARTY-LICENSES.md

@@ -35,11 +35,11 @@ Repository: https://github.com/101arrowz/fzstd
 
 ---------------------------------------
 
-## hyparquet, hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
+## hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
 
 License: MIT
 By: Hyperparam
-Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
+Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
 
 > The MIT License (MIT)
 > 

package/dist/adapters/filesystem.mjs

@@ -130,10 +130,48 @@ function lockFileFor(locksDir, scope) {
 	const safe = `${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`.replace(/[^\w.-]/g, "_");
 	return join(locksDir, `${safe}.lock`);
 }
+const inProcessLockTails = /* @__PURE__ */ new Map();
+async function withInProcessQueue(path, fn) {
+	const previous = inProcessLockTails.get(path) ?? Promise.resolve();
+	let release;
+	const current = new Promise((resolve) => {
+		release = resolve;
+	});
+	const tail = previous.then(() => current);
+	inProcessLockTails.set(path, tail);
+	await previous;
+	try {
+		return await fn();
+	} finally {
+		release();
+		if (inProcessLockTails.get(path) === tail) inProcessLockTails.delete(path);
+	}
+}
 function createFilesystemManifestStore(opts) {
 	const manifestPath = resolve(opts.path);
 	const locksDir = join(dirname(manifestPath), "locks");
 	const readyDirectories = /* @__PURE__ */ new Map();
+	async function withFileLock(path, fn) {
+		await writeFile(path, "", { flag: "a" }).catch(async (error) => {
+			if (error.code !== "ENOENT") throw error;
+			readyDirectories.delete(locksDir);
+			await ensureDirectory(locksDir);
+			await writeFile(path, "", { flag: "a" });
+		});
+		const release = await lock(path, {
+			realpath: false,
+			stale: 3e4,
+			retries: {
+				retries: 20,
+				minTimeout: 50,
+				maxTimeout: 500,
+				factor: 1.5
+			}
+		});
+		return await fn().finally(() => release().catch((releaseErr) => {
+			console.warn(`[gscdump/engine] failed to release lock ${path}; it will go stale after 30000ms`, releaseErr);
+		}));
+	}
 	async function ensureDirectory(path) {
 		let pending = readyDirectories.get(path);
 		if (!pending) {
@@ -247,28 +285,23 @@ function createFilesystemManifestStore(opts) {
 				await save(data);
 			});
 		},
+		async setSyncStates(scopes, state, detail) {
+			if (scopes.length === 0) return;
+			return enqueue(async () => {
+				const data = await load();
+				const byKey = new Map((data.syncStates ?? []).map((s) => [syncStateKey(s), s]));
+				for (const scope of scopes) {
+					const key = syncStateKey(scope);
+					byKey.set(key, mergeSyncState(byKey.get(key), scope, state, detail));
+				}
+				data.syncStates = Array.from(byKey.values());
+				await save(data);
+			});
+		},
 		async withLock(scope, fn) {
 			await ensureDirectory(locksDir);
 			const path = lockFileFor(locksDir, scope);
-			await writeFile(path, "", { flag: "a" }).catch(async (error) => {
-				if (error.code !== "ENOENT") throw error;
-				readyDirectories.delete(locksDir);
-				await ensureDirectory(locksDir);
-				await writeFile(path, "", { flag: "a" });
-			});
-			const release = await lock(path, {
-				realpath: false,
-				stale: 3e4,
-				retries: {
-					retries: 20,
-					minTimeout: 50,
-					maxTimeout: 500,
-					factor: 1.5
-				}
-			});
-			return await fn().finally(() => release().catch((releaseErr) => {
-				console.warn(`[gscdump/engine] failed to release lock ${path}; it will go stale after 30000ms`, releaseErr);
-			}));
+			return withInProcessQueue(path, () => withFileLock(path, fn));
 		},
 		async purgeTenant(filter) {
 			return enqueue(async () => {

package/dist/adapters/hyparquet.d.mts

@@ -1,5 +1,5 @@
-import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/index.mjs";
-import "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/node.mjs";
+import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/types/index.mjs";
+import "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
 import { CodecCtx, DataSource, ParquetCodec, Row, TableName } from "../storage.mjs";
 import { ColumnDef } from "../schema.mjs";
 export declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;

package/dist/adapters/hyparquet.mjs

@@ -1,11 +1,11 @@
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import { SCHEMAS } from "../schema.mjs";
-import { parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/metadata.mjs";
-import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/read.mjs";
-import "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/node.mjs";
-import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/write-rows.mjs";
-import "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
+import { parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
+import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
+import "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
+import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/write-rows.mjs";
+import "../node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 const ROW_GROUP_SIZE = 25e3;
 const DEFAULT_COMPACTION_READ_CONCURRENCY = 8;
 const MAX_COMPACTION_READ_CONCURRENCY = 32;

package/dist/adapters/node.d.mts

@@ -1,5 +1,5 @@
 import { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { NodeHarness, NodeHarnessOptions, createNodeHarness } from "./node-harness.mjs";
 import { AttachSnapshotOptions, AttachSnapshotResult, SnapshotQueryRunner, attachSnapshotIndex, snapshotAlias } from "./snapshot-attach.mjs";
-import { AttachParquetIndexOptions, AttachParquetIndexResult, attachParquetIndex } from "./parquet-attach.mjs";
-export { type AttachParquetIndexOptions, type AttachParquetIndexResult, type AttachSnapshotOptions, type AttachSnapshotResult, type NodeDuckDBOptions, type NodeHarness, type NodeHarnessOptions, type SnapshotQueryRunner, attachParquetIndex, attachSnapshotIndex, createNodeDuckDBHandle, createNodeHarness, resetNodeDuckDB, snapshotAlias };
+import { AttachParquetIndexOptions, AttachParquetIndexResult, ParquetFileGroup, attachParquetIndex } from "./parquet-attach.mjs";
+export { type AttachParquetIndexOptions, type AttachParquetIndexResult, type AttachSnapshotOptions, type AttachSnapshotResult, type NodeDuckDBOptions, type NodeHarness, type NodeHarnessOptions, type ParquetFileGroup, type SnapshotQueryRunner, attachParquetIndex, attachSnapshotIndex, createNodeDuckDBHandle, createNodeHarness, resetNodeDuckDB, snapshotAlias };

package/dist/adapters/parquet-attach.d.mts

@@ -1,11 +1,18 @@
 import { SnapshotQueryRunner } from "./snapshot-attach.mjs";
+/** Files that share constant column values, such as one Site and search type. */
+export interface ParquetFileGroup {
+  urls: string[];
+  /** Column name → VARCHAR value added to every row of this group. */
+  constants?: Record<string, string>;
+}
 export interface AttachParquetIndexOptions {
   /**
-   * Map of table name → list of Parquet URLs or local paths. The list may mix monthly
-   * compacted files and per-day files — DuckDB will scan all of them with
-   * `union_by_name = true`. Empty lists are skipped (no view created).
+   * Map of table name → list of Parquet URLs or local paths, or groups of
+   * them. A list may mix monthly compacted files and per-day files — DuckDB
+   * scans all of them with `union_by_name = true`. Empty lists and groups
+   * are skipped; a table with no files gets no view.
    */
-  tables: Record<string, string[]>;
+  tables: Record<string, string[] | ParquetFileGroup[]>;
   /** Schema the views land under. Default `main`. */
   schema?: string;
   /**

package/dist/adapters/parquet-attach.mjs

@@ -1,11 +1,24 @@
 const IDENT_RE = /^[A-Z_][\w$]*$/i;
 const REMOTE_URL_RE = /^(?:https?|s3):\/\//i;
+function toGroups(value) {
+	return value.every((item) => typeof item === "string") ? [{ urls: value }] : value;
+}
+function quote(value) {
+	return `'${value.replace(/'/g, "''")}'`;
+}
+function groupSelect(group) {
+	return `SELECT ${Object.entries(group.constants ?? {}).map(([name, value]) => `${quote(value)}::VARCHAR AS ${name}, `).join("")}* FROM read_parquet([${group.urls.map(quote).join(", ")}], union_by_name = true)`;
+}
 async function attachParquetIndex(runner, opts) {
 	const schema = opts.schema ?? "main";
 	const forceDownload = opts.forceDownload !== false;
 	if (!IDENT_RE.test(schema)) throw new TypeError(`attachParquetIndex: invalid schema identifier ${JSON.stringify(schema)}`);
-	for (const table of Object.keys(opts.tables)) if (!IDENT_RE.test(table)) throw new TypeError(`attachParquetIndex: invalid table identifier ${JSON.stringify(table)}`);
-	if (Object.values(opts.tables).some((urls) => urls.some((url) => REMOTE_URL_RE.test(url)))) {
+	const tables = Object.entries(opts.tables).map(([table, value]) => [table, toGroups(value).filter((group) => group.urls.length > 0)]);
+	for (const [table, groups] of tables) {
+		if (!IDENT_RE.test(table)) throw new TypeError(`attachParquetIndex: invalid table identifier ${JSON.stringify(table)}`);
+		for (const name of groups.flatMap((group) => Object.keys(group.constants ?? {}))) if (!IDENT_RE.test(name)) throw new TypeError(`attachParquetIndex: invalid column identifier ${JSON.stringify(name)}`);
+	}
+	if (tables.some(([, groups]) => groups.some((group) => group.urls.some((url) => REMOTE_URL_RE.test(url))))) {
 		await runner("LOAD httpfs").catch((cause) => {
 			throw new Error("Remote files require the DuckDB httpfs extension. Install httpfs before attaching remote files.", { cause });
 		});
@@ -13,9 +26,9 @@ async function attachParquetIndex(runner, opts) {
 	}
 	await runner(`CREATE SCHEMA IF NOT EXISTS ${schema}`);
 	const created = [];
-	for (const [table, urls] of Object.entries(opts.tables)) {
-		if (urls.length === 0) continue;
-		await runner(`CREATE OR REPLACE VIEW ${schema}.${table} AS SELECT * FROM read_parquet([${urls.map((u) => `'${u.replace(/'/g, "''")}'`).join(", ")}], union_by_name = true)`);
+	for (const [table, groups] of tables) {
+		if (groups.length === 0) continue;
+		await runner(`CREATE OR REPLACE VIEW ${schema}.${table} AS ${groups.map(groupSelect).join(" UNION ALL BY NAME ")}`);
 		created.push(table);
 	}
 	return {

package/dist/adapters/r2-manifest.mjs

@@ -54,6 +54,34 @@ function shardScopesFromEntries(entries) {
 	}
 	return out;
 }
+function applySyncState(snap, scope, state, at, detail) {
+	const scopeSearchType = inferSearchType(scope);
+	const idx = snap.syncStates.findIndex((s) => s.userId === scope.userId && s.siteId === scope.siteId && s.table === scope.table && s.date === scope.date && inferSearchType(s) === scopeSearchType);
+	if (idx === -1) {
+		snap.syncStates.push({
+			userId: scope.userId,
+			siteId: scope.siteId,
+			table: scope.table,
+			date: scope.date,
+			state,
+			updatedAt: at,
+			attempts: 1,
+			error: detail?.error,
+			...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
+		});
+		return;
+	}
+	const prev = snap.syncStates[idx];
+	const attempts = state === "inflight" && prev.state !== "inflight" ? prev.attempts + 1 : prev.attempts;
+	const error = state === "done" ? void 0 : detail?.error ?? prev.error;
+	snap.syncStates[idx] = {
+		...prev,
+		state,
+		updatedAt: at,
+		attempts,
+		error
+	};
+}
 function createR2ManifestStore(opts) {
 	const { bucket, userId } = opts;
 	const newSnapshotId = opts.newSnapshotId ?? defaultSnapshotId;
@@ -298,34 +326,26 @@ function createR2ManifestStore(opts) {
 			assertScopedUser(scope.userId, "setSyncState");
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");
 			const at = detail?.at ?? now();
-			const scopeSearchType = inferSearchType(scope);
-			await mutateShard(scope.siteId, scope.table, (snap) => {
-				const idx = snap.syncStates.findIndex((s) => s.userId === userId && s.siteId === scope.siteId && s.table === scope.table && s.date === scope.date && inferSearchType(s) === scopeSearchType);
-				if (idx === -1) {
-					snap.syncStates.push({
-						userId,
-						siteId: scope.siteId,
-						table: scope.table,
-						date: scope.date,
-						state,
-						updatedAt: at,
-						attempts: 1,
-						error: detail?.error,
-						...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
-					});
-					return;
-				}
-				const prev = snap.syncStates[idx];
-				const attempts = state === "inflight" && prev.state !== "inflight" ? prev.attempts + 1 : prev.attempts;
-				const error = state === "done" ? void 0 : detail?.error ?? prev.error;
-				snap.syncStates[idx] = {
-					...prev,
-					state,
-					updatedAt: at,
-					attempts,
-					error
+			await mutateShard(scope.siteId, scope.table, (snap) => applySyncState(snap, scope, state, at, detail));
+		},
+		async setSyncStates(scopes, state, detail) {
+			const byShard = /* @__PURE__ */ new Map();
+			for (const scope of scopes) {
+				assertScopedUser(scope.userId, "setSyncStates");
+				if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");
+				const key = `${scope.siteId}|${scope.table}`;
+				const shard = byShard.get(key) ?? {
+					siteId: scope.siteId,
+					table: scope.table,
+					scopes: []
 				};
-			});
+				shard.scopes.push(scope);
+				byShard.set(key, shard);
+			}
+			const at = detail?.at ?? now();
+			await mapWithConcurrency([...byShard.values()], SHARD_IO_CONCURRENCY, async (shard) => mutateShard(shard.siteId, shard.table, (snap) => {
+				for (const scope of shard.scopes) applySyncState(snap, scope, state, at, detail);
+			}));
 		},
 		async withLock(_scope, fn) {
 			return fn();

package/dist/analysis-range.d.mts

@@ -1,6 +1,63 @@
 import { AnalysisParams } from "./analysis-types.mjs";
+import { SearchType } from "gscdump/query";
 export interface AnalysisCoverageRange {
   start: string;
   end: string;
 }
-export declare function analysisRequestedRange(params: AnalysisParams, onInvalidFilter?: (error: unknown) => void): AnalysisCoverageRange | null;
+export declare function analysisRequestedRange(params: AnalysisParams, onInvalidFilter?: (error: unknown) => void): AnalysisCoverageRange | null;
+export interface DateSpan {
+  start: string;
+  end: string;
+}
+/**
+ * Where a host reads synced dates from. gscdump.com reads its ingest
+ * ledger; the CLI reads the Store's sync states. `table` is a string so a
+ * host can name its own derived tables.
+ */
+export interface CoverageReader {
+  datesForTable: (opts: {
+    table: string;
+    searchType: SearchType;
+    start: string;
+    end: string;
+  }) => Promise<readonly string[]>;
+}
+export interface CoveragePlanTable {
+  table: string;
+  required: boolean;
+  coveredSpans: DateSpan[];
+  gaps: DateSpan[];
+}
+export interface CoveragePlan {
+  searchType: SearchType;
+  requested: DateSpan;
+  tables: CoveragePlanTable[];
+  /** Every required table covers the requested range (less the tail grace). */
+  complete: boolean;
+  /** Hash of the covered spans. It changes when coverage changes. */
+  coverageVersion: string;
+}
+export interface BuildCoveragePlanOptions {
+  searchType: SearchType;
+  requested: DateSpan;
+  tables: readonly (string | {
+    table: string;
+    required?: boolean;
+  })[];
+  reader: CoverageReader;
+  /** Ignore the still-restating tail when deciding whether coverage is complete. */
+  tailGraceDays?: number;
+  /** Tables read at once. Default 3. */
+  concurrency?: number;
+}
+/** Sorted, merged spans of consecutive dates. */
+export declare function consolidateDatesToSpans(dates: readonly string[]): DateSpan[];
+/** The parts of `requested` that no span covers. Spans must be sorted and disjoint. */
+export declare function gapsForRange(coveredSpans: readonly DateSpan[], requested: DateSpan): DateSpan[];
+/** Days in a span, both ends included. */
+export declare function spanDays(span: DateSpan): number;
+/**
+ * Per-table coverage of one requested range. Pure over the reader: the
+ * host decides where synced dates live.
+ */
+export declare function buildCoveragePlan(opts: BuildCoveragePlanOptions): Promise<CoveragePlan>;

package/dist/analysis-range.mjs

@@ -1,3 +1,4 @@
+import { addDays } from "gscdump/dates";
 import { extractDateRange, normalizeFilter } from "gscdump/query";
 const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
 function addRange(ranges, start, end) {
@@ -31,4 +32,104 @@ function analysisRequestedRange(params, onInvalidFilter) {
 		end: next.end > range.end ? next.end : range.end
 	}));
 }
-export { analysisRequestedRange };
+function consolidateDatesToSpans(dates) {
+	if (dates.length === 0) return [];
+	const sorted = [...new Set(dates)].sort();
+	const out = [];
+	let start = sorted[0];
+	let end = sorted[0];
+	for (const date of sorted.slice(1)) {
+		if (date === addDays(end, 1)) {
+			end = date;
+			continue;
+		}
+		out.push({
+			start,
+			end
+		});
+		start = date;
+		end = date;
+	}
+	out.push({
+		start,
+		end
+	});
+	return out;
+}
+function gapsForRange(coveredSpans, requested) {
+	const gaps = [];
+	let cursor = requested.start;
+	for (const span of coveredSpans) {
+		if (span.end < cursor) continue;
+		if (span.start > requested.end) break;
+		if (span.start > cursor) gaps.push({
+			start: cursor,
+			end: addDays(span.start, -1)
+		});
+		if (span.end >= requested.end) return gaps;
+		cursor = addDays(span.end, 1);
+	}
+	if (cursor <= requested.end) gaps.push({
+		start: cursor,
+		end: requested.end
+	});
+	return gaps;
+}
+function spanDays(span) {
+	return Math.round((Date.parse(`${span.end}T00:00:00Z`) - Date.parse(`${span.start}T00:00:00Z`)) / 864e5) + 1;
+}
+async function mapLimit(items, limit, fn) {
+	const out = Array.from({ length: items.length });
+	let next = 0;
+	const worker = async () => {
+		while (next < items.length) {
+			const index = next++;
+			out[index] = await fn(items[index]);
+		}
+	};
+	await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
+	return out;
+}
+async function hashCoverage(parts) {
+	const data = new TextEncoder().encode(parts.sort().join("\n"));
+	const digest = await crypto.subtle.digest("SHA-256", data);
+	return [...new Uint8Array(digest).slice(0, 16)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+async function buildCoveragePlan(opts) {
+	const { requested, searchType } = opts;
+	const graceEnd = opts.tailGraceDays && opts.tailGraceDays > 0 ? addDays(requested.end, -opts.tailGraceDays) : requested.end;
+	const gate = {
+		start: requested.start,
+		end: graceEnd < requested.start ? requested.start : graceEnd
+	};
+	const tables = await mapLimit(opts.tables, opts.concurrency ?? 3, async (raw) => {
+		const input = typeof raw === "string" ? {
+			table: raw,
+			required: true
+		} : {
+			table: raw.table,
+			required: raw.required ?? true
+		};
+		const coveredSpans = consolidateDatesToSpans((await opts.reader.datesForTable({
+			table: input.table,
+			searchType,
+			start: requested.start,
+			end: requested.end
+		})).filter((date) => date >= requested.start && date <= requested.end));
+		return {
+			table: input.table,
+			required: input.required,
+			coveredSpans,
+			gaps: gapsForRange(coveredSpans, requested)
+		};
+	});
+	const required = tables.filter((table) => table.required);
+	return {
+		searchType,
+		requested: { ...requested },
+		tables,
+		complete: required.length > 0 && required.every((table) => gapsForRange(table.coveredSpans, gate).length === 0),
+		coverageVersion: await hashCoverage(tables.flatMap((table) => table.coveredSpans.map((span) => `${table.table}\0${span.start}\0${span.end}`)))
+	};
+}
+export { analysisRequestedRange, buildCoveragePlan, consolidateDatesToSpans, gapsForRange, spanDays };

package/dist/analysis-types.d.mts

@@ -7,8 +7,18 @@ export interface AnalysisParams {
   prevStartDate?: string;
   prevEndDate?: string;
   brandTerms?: string[];
+  /**
+   * Output cap: how many result rows the analyzer returns. It never bounds
+   * the rows an analyzer reads. Use `fetchBudget` for that.
+   */
   limit?: number;
   offset?: number;
+  /**
+   * Row-plan fetch cap per query, in rows. Defaults to
+   * `DEFAULT_FETCH_BUDGET` (one GSC page). SQL plans ignore it. When a fetch
+   * reaches the budget, the result meta reports `coverage.kind: 'truncated'`.
+   */
+  fetchBudget?: number;
   /**
    * movers: select a single direction and paginate within it. When set, the
    * analyzer filters to that direction, reports `meta.total` as the full
@@ -48,9 +58,40 @@ export interface AnalysisParams {
   /** GSC slice the analysis is scoped to. Undefined = analyzer runs cross-type (today's behaviour for web-only sites). */
   searchType?: SearchType;
 }
+/**
+ * Whether an analyzer read every matching row. `truncated` means at least one
+ * row query stopped at its fetch budget, so totals and rankings can be wrong.
+ * `fetched` is the row count of the largest truncated query.
+ */
+export type AnalyzerCoverage = {
+  kind: 'complete';
+} | {
+  kind: 'truncated';
+  fetched: number;
+};
 export interface AnalysisResult {
   results: Record<string, unknown>[];
-  meta: Record<string, unknown>;
+  meta: Record<string, unknown> & {
+    coverage?: AnalyzerCoverage;
+  };
 }
+/** One GSC Search Analytics page. The default row-plan fetch cap. */
+export declare const DEFAULT_FETCH_BUDGET = 25000;
+/** Upper bound for an opt-in fetch budget. Larger pulls exhaust GSC quota. */
+export declare const MAX_FETCH_BUDGET = 100000;
+declare const fetchBudgetBrand: unique symbol;
+/**
+ * Rows a row plan may fetch per query. Branded so an output `limit` cannot
+ * be passed where a fetch cap is expected. Build one with `fetchBudgetOf`.
+ */
+export type FetchBudget = number & {
+  readonly [fetchBudgetBrand]: true;
+};
+/**
+ * Resolve the fetch budget for a run: `params.fetchBudget` clamped to
+ * `[1, MAX_FETCH_BUDGET]`, else `DEFAULT_FETCH_BUDGET`. It never reads
+ * `params.limit`.
+ */
+export declare function fetchBudgetOf(params: Pick<AnalysisParams, 'fetchBudget'>): FetchBudget;
 /** Coerce arbitrary value (number, bigint, string, null) to number, defaulting to 0. */
 export declare function num(v: unknown): number;

package/dist/analysis-types.mjs

@@ -1,7 +1,14 @@
+const DEFAULT_FETCH_BUDGET = 25e3;
+const MAX_FETCH_BUDGET = 1e5;
+function fetchBudgetOf(params) {
+	const raw = params.fetchBudget;
+	if (raw == null || !Number.isFinite(raw) || raw < 1) return DEFAULT_FETCH_BUDGET;
+	return Math.min(Math.floor(raw), MAX_FETCH_BUDGET);
+}
 function num(v) {
 	if (typeof v === "number") return v;
 	if (typeof v === "bigint") return Number(v);
 	if (v == null) return 0;
 	return Number(v);
 }
-export { num };
+export { DEFAULT_FETCH_BUDGET, MAX_FETCH_BUDGET, fetchBudgetOf, num };

package/dist/analyzer/dispatch.mjs

@@ -47,16 +47,32 @@ async function runAnalyzerFromSource(source, params, registry) {
 	if (plan.kind === "rows") return runRowsPlanAgainstSource(source, analyzer, plan, params);
 	return runSqlPlanAgainstSource(source, analyzer, plan, params);
 }
+async function fetchPlanRows(source, state) {
+	const rows = await source.queryRows(state);
+	return {
+		rows,
+		truncated: state.rowLimit != null && rows.length >= state.rowLimit
+	};
+}
+function coverageOf(fetches) {
+	const truncated = fetches.filter((f) => f.truncated);
+	if (truncated.length === 0) return { kind: "complete" };
+	return {
+		kind: "truncated",
+		fetched: Math.max(...truncated.map((f) => f.rows.length))
+	};
+}
 async function runRowsPlanAgainstSource(source, analyzer, plan, params) {
 	const entries = Object.entries(plan.queries);
-	const resolved = await Promise.all(entries.map(async ([k, q]) => [k, await source.queryRows(q.state)]));
-	const rowMap = Object.fromEntries(resolved);
+	const fetched = await Promise.all(entries.map(async ([k, q]) => [k, await fetchPlanRows(source, q.state)]));
+	const rowMap = Object.fromEntries(fetched.map(([k, f]) => [k, f.rows]));
 	const { results, meta } = analyzer.reduce(rowMap, { params });
 	return {
 		results,
 		meta: {
 			tool: params.type,
-			...meta
+			...meta,
+			coverage: coverageOf(fetched.map(([, f]) => f))
 		}
 	};
 }
@@ -85,7 +101,8 @@ async function runSqlPlanAgainstSource(source, analyzer, plan, params) {
 		meta: {
 			tool: params.type,
 			...sourceMeta,
-			...meta
+			...meta,
+			coverage: { kind: "complete" }
 		}
 	};
 }

package/dist/engine.mjs

@@ -1,8 +1,9 @@
 import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
-import { SCHEMAS, currentSchemaVersion, dedupeByNaturalKey } from "./schema.mjs";
+import { SCHEMAS, currentSchemaVersion } from "./schema.mjs";
 import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from "./compaction.mjs";
 import { compileLogicalQueryPlan } from "./parquet-plan.mjs";
 import { gcOrphansImpl } from "./gc.mjs";
+import { sumByStoredKey } from "./ingest.mjs";
 import { extractParquetPushdown } from "./parquet-pushdown.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
@@ -60,7 +61,7 @@ function createStorageEngine(opts) {
 				partitions: [partition],
 				searchType: inferSearchType({ searchType })
 			});
-			const normalizedRows = dedupeByNaturalKey(ctx.table, rows.map((r) => normalizeRow(ctx.table, r)));
+			const normalizedRows = sumByStoredKey(ctx.table, rows.map((r) => normalizeRow(ctx.table, r)));
 			const key = objectKey(ctx, ctx.table, partition, now, searchType);
 			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, normalizedRows, key, dataSource);
 			let bytes = writtenBytes;
@@ -128,8 +129,7 @@ function createStorageEngine(opts) {
 				const k = `${String(r.url ?? "")}\0${String(r.hour ?? "")}`;
 				dedup.set(k, r);
 			}
-			for (const r of rows) {
-				const normalized = normalizeRow(ctx.table, r);
+			for (const normalized of sumByStoredKey(ctx.table, rows.map((r) => normalizeRow(ctx.table, r)))) {
 				const k = `${String(normalized.url ?? "")}\0${String(normalized.hour ?? "")}`;
 				dedup.set(k, normalized);
 			}
@@ -381,6 +381,7 @@ function createStorageEngine(opts) {
 		getWatermarks: (filter) => manifestStore.getWatermarks(filter),
 		getSyncStates: (filter) => manifestStore.getSyncStates(filter),
 		setSyncState: (scope, state, detail) => manifestStore.setSyncState(scope, state, detail),
+		setSyncStates: (scopes, state, detail) => manifestStore.setSyncStates(scopes, state, detail),
 		readObject: (key) => dataSource.read(key)
 	};
 }

package/dist/entities/inspection.d.mts

@@ -155,6 +155,12 @@ export interface InspectionStore {
    * `undefined` if the month has no shards.
    */
   loadHistory: (ctx: TenantCtx, yearMonth: string) => Promise<InspectionHistoryShard | undefined>;
+  /**
+   * Month buckets (`YYYY-MM`, or `unknown`) that hold at least one history
+   * shard, sorted ascending. Pair with {@link InspectionStore.loadHistory} to
+   * read the full history.
+   */
+  listHistoryMonths: (ctx: TenantCtx) => Promise<string[]>;
   /**
    * Encode caller-provided rows into the inspections parquet sidecar at
    * `entities/inspections/index.parquet`. Sorted by `urlHash` so DuckDB
@@ -171,6 +177,11 @@ export interface InspectionStore {
     rowCount: number;
     bytes: number;
   }>;
+  /**
+   * Rows of the inspections parquet sidecar that {@link InspectionStore.materialize}
+   * wrote last, or `undefined` if it was never written.
+   */
+  loadMaterialized: (ctx: TenantCtx) => Promise<InspectionParquetRow[] | undefined>;
   /**
    * Append a batch of inspection results as an immutable per-batch parquet
    * under `events/<YYYY-MM>/<batchId>.parquet`, partitioned by the `YYYY-MM`