@gscdump/engine-gsc-api 4.0.1 → 4.1.0

package/README.md

@@ -50,6 +50,37 @@ Create a Source per request if your host manages token refresh between requests.
 | `fetchGscDaily` | Read daily metrics |
 | `runGscSyncSlice` | Read a bounded Search Analytics sync slice |
 | `runGscSearchAppearanceContextSlice` | Read a Search Appearance context slice |
+| `planGscSyncWork` | Find ledger gaps and size Sync windows |
+| `gscSyncFanout` | Build the standard Search Analytics table fanout |
+| `tablesCoveredByGscSync` | List the tables a Sync slice must write |
+| `ledgerTablesForGscSync` | List the ledger tables needed for one fanout |
+| `planGscBackfillDates` | Select interior repair or backward Backfill dates |
+
+## Plan stored Sync work
+
+Pass the dates you want to cover, the planned `(table, searchType)` fanout, and
+the dates already committed to your ledger. The plan returns only missing
+windows. Record zero-row slices in the ledger too, so they do not repeat.
+
+```ts
+import { gscSyncFanout, planGscSyncWork } from '@gscdump/engine-gsc-api'
+
+const plan = planGscSyncWork({
+  dates: ['2026-09-01', '2026-09-02'],
+  fanout: gscSyncFanout(['web'], { omitWebDates: true }),
+  ledger: [{ table: 'pages', searchType: 'web', date: '2026-09-01' }],
+  pagesPerDayEstimate: 100,
+})
+
+for (const window of plan.windows) {
+  // Run runGscSyncSlice for this table and date range, then commit its ledger rows.
+  console.log(window)
+}
+```
+
+The defaults match the hosted Worker's tested window budget. Set `rowBudget`,
+`maxSpanDays`, and `maxWindowsPerRun` for your own runtime. The window count
+limit can widen a span beyond `maxSpanDays` during a long catch-up.
 
 ## Limits and fallback
 

package/dist/index.d.mts

@@ -1,5 +1,6 @@
 import { CreateLiveGscSourceOptions, canProxyToGsc, createLiveGscSource } from "./live.mjs";
 import { FetchTopNOptions, GscDailyRow, GscRange, GscTopNRow, fetchGscDaily, fetchGscTopN } from "./rollup-synth.mjs";
 import { GSC_API_CAPABILITIES, GscApiQuerySourceOptions, createGscApiQuerySource } from "./source.mjs";
+import { GscSyncEntryWork, GscSyncFanoutEntry, GscSyncLedgerRow, GscSyncWindow, GscSyncWorkPlan, PlanGscBackfillDatesOptions, PlanGscBackwardDatesOptions, PlanGscSyncWorkOptions, gscSyncFanout, ledgerTablesForGscSync, planGscBackfillDates, planGscSyncWork, tablesCoveredByGscSync } from "./sync-plan.mjs";
 import { GscApiRow, RunGscSearchAppearanceContextSliceOptions, RunGscSearchAppearanceContextSliceResult, RunGscSyncSliceOptions, RunGscSyncSliceResult, SearchAppearanceContextGrain, SearchAppearanceContextTable, SyncSliceDimensionFilter, SyncSliceDomainFilter, runGscSearchAppearanceContextSlice, runGscSyncSlice } from "./sync-slice.mjs";
-export { type CreateLiveGscSourceOptions, type FetchTopNOptions, GSC_API_CAPABILITIES, type GscApiQuerySourceOptions, type GscApiRow, type GscDailyRow, type GscRange, type GscTopNRow, type RunGscSearchAppearanceContextSliceOptions, type RunGscSearchAppearanceContextSliceResult, type RunGscSyncSliceOptions, type RunGscSyncSliceResult, type SearchAppearanceContextGrain, type SearchAppearanceContextTable, type SyncSliceDimensionFilter, type SyncSliceDomainFilter, canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN, runGscSearchAppearanceContextSlice, runGscSyncSlice };
+export { type CreateLiveGscSourceOptions, type FetchTopNOptions, GSC_API_CAPABILITIES, type GscApiQuerySourceOptions, type GscApiRow, type GscDailyRow, type GscRange, type GscSyncEntryWork, type GscSyncFanoutEntry, type GscSyncLedgerRow, type GscSyncWindow, type GscSyncWorkPlan, type GscTopNRow, type PlanGscBackfillDatesOptions, type PlanGscBackwardDatesOptions, type PlanGscSyncWorkOptions, type RunGscSearchAppearanceContextSliceOptions, type RunGscSearchAppearanceContextSliceResult, type RunGscSyncSliceOptions, type RunGscSyncSliceResult, type SearchAppearanceContextGrain, type SearchAppearanceContextTable, type SyncSliceDimensionFilter, type SyncSliceDomainFilter, canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN, gscSyncFanout, ledgerTablesForGscSync, planGscBackfillDates, planGscSyncWork, runGscSearchAppearanceContextSlice, runGscSyncSlice, tablesCoveredByGscSync };

package/dist/index.mjs

@@ -1,5 +1,6 @@
 import { fetchGscDaily, fetchGscTopN } from "./rollup-synth.mjs";
 import { GSC_API_CAPABILITIES, createGscApiQuerySource } from "./source.mjs";
 import { canProxyToGsc, createLiveGscSource } from "./live.mjs";
+import { gscSyncFanout, ledgerTablesForGscSync, planGscBackfillDates, planGscSyncWork, tablesCoveredByGscSync } from "./sync-plan.mjs";
 import { runGscSearchAppearanceContextSlice, runGscSyncSlice } from "./sync-slice.mjs";
-export { GSC_API_CAPABILITIES, canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN, runGscSearchAppearanceContextSlice, runGscSyncSlice };
+export { GSC_API_CAPABILITIES, canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN, gscSyncFanout, ledgerTablesForGscSync, planGscBackfillDates, planGscSyncWork, runGscSearchAppearanceContextSlice, runGscSyncSlice, tablesCoveredByGscSync };

package/dist/sync-plan.d.mts

@@ -0,0 +1,63 @@
+import { SearchType, TableName } from "@gscdump/engine/contracts";
+export interface GscSyncFanoutEntry {
+  table: string;
+  searchType?: SearchType;
+}
+export interface GscSyncLedgerRow {
+  table: string;
+  searchType: string;
+  date: string;
+}
+export interface GscSyncWindow extends GscSyncFanoutEntry {
+  startDate: string;
+  endDate: string;
+}
+export interface PlanGscSyncWorkOptions {
+  dates: readonly string[];
+  fanout: readonly GscSyncFanoutEntry[];
+  ledger: readonly GscSyncLedgerRow[];
+  pagesPerDayEstimate?: number | null;
+  /** Target rows per window. The default is the hosted Worker's tested budget. */
+  rowBudget?: number;
+  /** Target window cap. A maxWindowsPerRun limit can widen it. */
+  maxSpanDays?: number;
+  /** Largest number of windows per contiguous run. */
+  maxWindowsPerRun?: number;
+}
+export interface GscSyncEntryWork {
+  entry: GscSyncFanoutEntry;
+  missingDates: string[];
+  windows: GscSyncWindow[];
+}
+export interface GscSyncWorkPlan {
+  entries: GscSyncEntryWork[];
+  missingDates: string[];
+  windows: GscSyncWindow[];
+}
+/** Build the standard Search Analytics fanout. Web keeps its legacy absent searchType field. */
+export declare function gscSyncFanout(searchTypes: readonly SearchType[], options?: {
+  omitWebDates?: boolean;
+}): GscSyncFanoutEntry[];
+/** The web queries slice also writes the dates table. Other search types use a dates slice. */
+export declare function tablesCoveredByGscSync(entry: GscSyncFanoutEntry): TableName[];
+/** The minimal set of ledger tables needed to inspect this fanout. */
+export declare function ledgerTablesForGscSync(fanout: readonly GscSyncFanoutEntry[]): TableName[];
+/** Plan only slices missing at least one table that their job must write. */
+export declare function planGscSyncWork(options: PlanGscSyncWorkOptions): GscSyncWorkPlan;
+export interface PlanGscBackfillDatesOptions {
+  _tag: 'repair';
+  /** Interior repair never probes before this date. */
+  oldestCoveredDate: string;
+  newestAvailableDate: string;
+  coveredDates: Iterable<string>;
+  targetDays: number;
+}
+export interface PlanGscBackwardDatesOptions {
+  _tag: 'backfill';
+  startFromDate: string;
+  oldestAvailableDate: string;
+  targetDays: number;
+  maxDays: number;
+}
+/** Pick interior gaps or extend history backwards from the oldest covered date. */
+export declare function planGscBackfillDates(options: PlanGscBackfillDatesOptions | PlanGscBackwardDatesOptions): string[];

package/dist/sync-plan.mjs

@@ -0,0 +1,112 @@
+import { TABLES_BY_SEARCH_TYPE } from "@gscdump/engine/sync-config";
+import { addDays } from "gscdump/dates";
+const DEFAULT_ROW_BUDGET = 25e3;
+const DEFAULT_MAX_SPAN_DAYS = 31;
+const DEFAULT_MAX_WINDOWS_PER_RUN = 14;
+const TABLE_DENSITY = {
+	pages: 1,
+	countries: .3,
+	dates: 1,
+	queries: 3,
+	page_queries: 8
+};
+function gscSyncFanout(searchTypes, options = {}) {
+	return searchTypes.flatMap((searchType) => TABLES_BY_SEARCH_TYPE[searchType].filter((table) => !(searchType === "web" && options.omitWebDates && table === "dates")).map((table) => ({
+		table,
+		...searchType === "web" ? {} : { searchType }
+	})));
+}
+function tablesCoveredByGscSync(entry) {
+	switch (entry.table) {
+		case "pages": return ["pages"];
+		case "countries": return ["countries"];
+		case "page_queries": return ["page_queries"];
+		case "search_appearance": return ["search_appearance"];
+		case "search_appearance_pages": return ["search_appearance_pages"];
+		case "search_appearance_queries": return ["search_appearance_queries"];
+		case "search_appearance_page_queries": return ["search_appearance_page_queries"];
+		case "hourly_pages": return ["hourly_pages"];
+		case "queries": return entry.searchType === void 0 || entry.searchType === "web" ? ["queries", "dates"] : ["queries"];
+		case "dates": return entry.searchType === void 0 || entry.searchType === "web" ? [] : ["dates"];
+		default: throw new RangeError(`Unknown GSC sync fanout table: '${entry.table}'`);
+	}
+}
+function ledgerTablesForGscSync(fanout) {
+	return [...new Set(fanout.flatMap(tablesCoveredByGscSync))];
+}
+function positiveInteger(value, fallback) {
+	if (value === void 0) return fallback;
+	if (!Number.isFinite(value)) throw new RangeError("Sync window limits must be finite numbers");
+	return Math.max(1, Math.floor(value));
+}
+function windowDays(estimate, table, totalDays, options) {
+	const maxSpan = positiveInteger(options.maxSpanDays, DEFAULT_MAX_SPAN_DAYS);
+	const maxWindows = positiveInteger(options.maxWindowsPerRun, DEFAULT_MAX_WINDOWS_PER_RUN);
+	const rowBudget = positiveInteger(options.rowBudget, DEFAULT_ROW_BUDGET);
+	const density = estimate && estimate > 0 ? Math.max(1, Math.floor(rowBudget / Math.max(1, estimate * (TABLE_DENSITY[table] ?? 1)))) : 1;
+	return Math.max(Math.min(maxSpan, density), Math.ceil(totalDays / maxWindows));
+}
+function contiguousRuns(dates) {
+	const runs = [];
+	for (const date of dates) {
+		const last = runs.at(-1);
+		if (last && addDays(last.end, 1) === date) {
+			last.end = date;
+			last.days++;
+		} else runs.push({
+			start: date,
+			end: date,
+			days: 1
+		});
+	}
+	return runs;
+}
+function windowsForRun(entry, run, spanDays) {
+	const windows = [];
+	for (let startDate = run.start; startDate <= run.end; startDate = addDays(windows.at(-1).endDate, 1)) {
+		const endDate = addDays(startDate, spanDays - 1);
+		windows.push({
+			...entry,
+			startDate,
+			endDate: endDate < run.end ? endDate : run.end
+		});
+	}
+	return windows;
+}
+function planGscSyncWork(options) {
+	const dates = [...new Set(options.dates)].sort();
+	const covered = new Set(options.ledger.map((row) => `${row.searchType}|${row.table}|${row.date}`));
+	const entries = [];
+	for (const entry of options.fanout) {
+		const tables = tablesCoveredByGscSync(entry);
+		if (tables.length === 0) continue;
+		const searchType = entry.searchType ?? "web";
+		const missingDates = dates.filter((date) => tables.some((table) => !covered.has(`${searchType}|${table}|${date}`)));
+		if (missingDates.length === 0) continue;
+		const windows = contiguousRuns(missingDates).flatMap((run) => windowsForRun(entry, run, windowDays(options.pagesPerDayEstimate, entry.table, run.days, options)));
+		entries.push({
+			entry,
+			missingDates,
+			windows
+		});
+	}
+	return {
+		entries,
+		missingDates: [...new Set(entries.flatMap((entry) => entry.missingDates))].sort(),
+		windows: entries.flatMap((entry) => entry.windows)
+	};
+}
+function planGscBackfillDates(options) {
+	if (!Number.isFinite(options.targetDays) || options._tag === "backfill" && !Number.isFinite(options.maxDays)) throw new RangeError("Backfill limits must be finite numbers");
+	const limit = Math.max(0, Math.floor(Math.min(options.targetDays, options._tag === "backfill" ? options.maxDays : options.targetDays)));
+	if (options._tag === "repair") {
+		const covered = new Set(options.coveredDates);
+		const dates = [];
+		for (let date = options.oldestCoveredDate; date <= options.newestAvailableDate && dates.length < limit; date = addDays(date, 1)) if (!covered.has(date)) dates.push(date);
+		return dates;
+	}
+	const dates = [];
+	for (let date = options.startFromDate; date >= options.oldestAvailableDate && dates.length < limit; date = addDays(date, -1)) dates.push(date);
+	return dates;
+}
+export { gscSyncFanout, ledgerTablesForGscSync, planGscBackfillDates, planGscSyncWork, tablesCoveredByGscSync };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-gsc-api",
   "type": "module",
-  "version": "4.0.1",
+  "version": "4.1.0",
   "description": "GSC live-API engine adapter — wraps the Search Analytics REST API as an AnalysisQuerySource for typed analyzer dispatch.",
   "author": {
     "name": "Harlan Wilton",
@@ -36,8 +36,8 @@
     "node": ">=22.13.0"
   },
   "dependencies": {
-    "@gscdump/engine": "^4.0.1",
-    "gscdump": "^4.0.1"
+    "@gscdump/engine": "^4.1.0",
+    "gscdump": "^4.1.0"
   },
   "devDependencies": {
     "vitest": "^5.0.1"