@gscdump/engine-gsc-api 0.17.2 → 0.17.4

package/dist/index.d.mts

@@ -64,4 +64,58 @@ interface GscApiQuerySourceOptions {
   siteUrl: string;
 }
 declare function createGscApiQuerySource(options: GscApiQuerySourceOptions): AnalysisQuerySource;
-export { type CreateLiveGscSourceOptions, type FetchTopNOptions, type GscApiQuerySourceOptions, type GscDailyRow, type GscRange, type GscTopNRow, canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN };
+interface GscApiRow {
+  keys: string[];
+  clicks: number;
+  impressions: number;
+  ctr: number;
+  position: number;
+}
+interface SyncSliceDomainFilter {
+  /**
+   * Domain (eTLD+1 + subdomain) to scope the slice to — matches both
+   *  `www.` and bare variants. Strip the protocol; the regex is built here.
+   */
+  domain?: string;
+}
+interface RunGscSyncSliceOptions {
+  client: GoogleSearchConsoleClient;
+  siteUrl: string;
+  /** One of the engine sync-fan tables. Drives the dimension list. */
+  table: 'pages' | 'keywords' | 'countries' | 'devices' | 'page_keywords';
+  startDate: string;
+  endDate: string;
+  domainFilter?: SyncSliceDomainFilter | null;
+  /**
+   * Invoked per GSC API page with the rows fetched. Return a promise; the
+   *  loop awaits it before paging further. Throw inside to abort; AbortError /
+   *  timeout messages are swallowed so the continuation can re-process.
+   */
+  onBatch: (rows: GscApiRow[]) => Promise<void>;
+  initialStartRow?: number;
+  /**
+   * Max GSC API pages per call. `Infinity` for unbounded; hosts cap this
+   *  based on path (D1 vs R2) and table.
+   */
+  maxPages?: number;
+  /** GSC page size. 500 is the D1-safe default; 10k is the R2 path. */
+  rowLimit?: number;
+  /**
+   * Soft CPU budget for the loop itself. Returns `hasMore` when crossed so
+   *  the continuation resumes from `nextStartRow`.
+   */
+  cpuBudgetMs?: number;
+  searchType?: SearchType;
+  /** Invoked once per successful GSC API page. Hosts wire telemetry here. */
+  onPage?: (info: {
+    searchType: SearchType;
+    rowsThisPage: number;
+  }) => void;
+}
+interface RunGscSyncSliceResult {
+  totalRows: number;
+  hasMore: boolean;
+  nextStartRow: number;
+}
+declare function runGscSyncSlice(opts: RunGscSyncSliceOptions): Promise<RunGscSyncSliceResult>;
+export { type CreateLiveGscSourceOptions, type FetchTopNOptions, type GscApiQuerySourceOptions, type GscApiRow, type GscDailyRow, type GscRange, type GscTopNRow, type RunGscSyncSliceOptions, type RunGscSyncSliceResult, type SyncSliceDomainFilter, canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN, runGscSyncSlice };

package/dist/index.mjs

@@ -159,4 +159,88 @@ function createLiveGscSource(opts) {
 		}
 	};
 }
-export { canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN };
+const DIMENSIONS_BY_TABLE = {
+	pages: ["page", "date"],
+	keywords: ["query", "date"],
+	countries: ["country", "date"],
+	devices: ["device", "date"],
+	page_keywords: [
+		"page",
+		"query",
+		"date"
+	]
+};
+function isTimeoutLike(err) {
+	if (!(err instanceof Error)) return false;
+	return err.name === "AbortError" || err.message?.includes("timeout") || err.message?.includes("aborted");
+}
+function buildDomainFilterGroups(filter) {
+	if (!filter?.domain) return void 0;
+	return [{ filters: [{
+		dimension: "page",
+		operator: "includingRegex",
+		expression: `^https?://(www\\.)?${filter.domain.replace(/^www\./, "").replace(/\./g, "\\.")}/`
+	}] }];
+}
+async function runGscSyncSlice(opts) {
+	const rowLimit = opts.rowLimit ?? 500;
+	const cpuBudgetMs = opts.cpuBudgetMs ?? 2e4;
+	const maxPages = opts.maxPages ?? Infinity;
+	const searchType = opts.searchType ?? "web";
+	const dimensions = [...DIMENSIONS_BY_TABLE[opts.table]];
+	const dimensionFilterGroups = buildDomainFilterGroups(opts.domainFilter);
+	const loopStart = Date.now();
+	let startRow = opts.initialStartRow ?? 0;
+	let totalRows = 0;
+	let pageCount = 0;
+	while (true) {
+		if (pageCount >= maxPages) return {
+			totalRows,
+			hasMore: true,
+			nextStartRow: startRow
+		};
+		if (Date.now() - loopStart >= cpuBudgetMs) return {
+			totalRows,
+			hasMore: true,
+			nextStartRow: startRow
+		};
+		const query = {
+			startDate: opts.startDate,
+			endDate: opts.endDate,
+			dimensions,
+			rowLimit,
+			startRow,
+			dataState: "all",
+			searchType,
+			...dimensionFilterGroups ? { dimensionFilterGroups } : {}
+		};
+		const response = await opts.client._rawQuery(opts.siteUrl, query).catch((err) => {
+			if (isTimeoutLike(err)) return null;
+			throw err;
+		});
+		if (!response) return {
+			totalRows,
+			hasMore: true,
+			nextStartRow: startRow
+		};
+		const rows = response.rows ?? [];
+		totalRows += rows.length;
+		pageCount++;
+		opts.onPage?.({
+			searchType,
+			rowsThisPage: rows.length
+		});
+		if (rows.length > 0) await opts.onBatch(rows).catch((err) => {
+			if (isTimeoutLike(err)) return;
+			throw err;
+		});
+		if (rows.length < rowLimit) break;
+		startRow += rows.length;
+	}
+	return {
+		totalRows,
+		hasMore: false,
+		nextStartRow: startRow
+	};
+}
+export { canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN, runGscSyncSlice };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-gsc-api",
   "type": "module",
-  "version": "0.17.2",
+  "version": "0.17.4",
   "description": "GSC live-API engine adapter — wraps the Search Analytics REST API as an AnalysisQuerySource for typed analyzer dispatch.",
   "author": {
     "name": "Harlan Wilton",
@@ -36,8 +36,8 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@gscdump/engine": "0.17.2",
-    "gscdump": "0.17.2"
+    "@gscdump/engine": "0.17.4",
+    "gscdump": "0.17.4"
   },
   "devDependencies": {
     "vitest": "^4.1.6"