@gscdump/engine-gsc-api 1.4.0 → 1.4.1

package/dist/sync-slice.mjs

@@ -0,0 +1,248 @@
+const DIMENSIONS_BY_TABLE = {
+	pages: ["page", "date"],
+	queries: ["query", "date"],
+	countries: ["country", "date"],
+	dates: ["device", "date"],
+	page_queries: [
+		"page",
+		"query",
+		"date"
+	],
+	search_appearance: ["searchAppearance"],
+	search_appearance_pages: ["page", "date"],
+	search_appearance_queries: ["query", "date"],
+	search_appearance_page_queries: [
+		"page",
+		"query",
+		"date"
+	],
+	hourly_pages: ["hour", "page"]
+};
+function isTimeoutLike(err) {
+	if (!(err instanceof Error)) return false;
+	return err.name === "AbortError" || err.message?.includes("timeout") || err.message?.includes("aborted");
+}
+function buildDimensionFilterGroups(domainFilter, filters = []) {
+	const out = filters.map((f) => ({
+		dimension: f.dimension,
+		operator: f.operator ?? "equals",
+		expression: f.expression
+	}));
+	if (domainFilter?.domain) {
+		const pattern = `^https?://(www\\.)?${domainFilter.domain.replace(/^www\./, "").replace(/\./g, "\\.")}/`;
+		out.push({
+			dimension: "page",
+			operator: "includingRegex",
+			expression: pattern
+		});
+	}
+	return out.length > 0 ? [{ filters: out }] : void 0;
+}
+async function runGscSyncSlice(opts) {
+	const rowLimit = opts.rowLimit ?? 500;
+	const cpuBudgetMs = opts.cpuBudgetMs ?? 2e4;
+	const maxPages = opts.maxPages ?? Infinity;
+	const searchType = opts.searchType ?? "web";
+	const dimensions = opts.dimensions ? [...opts.dimensions] : [...DIMENSIONS_BY_TABLE[opts.table]];
+	const dataState = opts.dataState ?? (dimensions.includes("hour") ? "hourly_all" : "all");
+	const dimensionFilterGroups = buildDimensionFilterGroups(opts.domainFilter, opts.dimensionFilters);
+	const loopStart = Date.now();
+	let startRow = opts.initialStartRow ?? 0;
+	let totalRows = 0;
+	let pageCount = 0;
+	let metadata;
+	const fetchPage = async (row) => {
+		const query = {
+			startDate: opts.startDate,
+			endDate: opts.endDate,
+			dimensions,
+			rowLimit,
+			startRow: row,
+			dataState,
+			type: searchType,
+			...dimensionFilterGroups ? { dimensionFilterGroups } : {}
+		};
+		try {
+			const response = await opts.client.searchAnalytics.query(opts.siteUrl, query);
+			return {
+				kind: "ok",
+				startRow: row,
+				rows: response.rows ?? [],
+				metadata: response.metadata
+			};
+		} catch (err) {
+			if (isTimeoutLike(err)) return {
+				kind: "timeout",
+				startRow: row
+			};
+			return {
+				kind: "error",
+				error: err
+			};
+		}
+	};
+	const startFetchIfAllowed = (row) => {
+		if (pageCount >= maxPages) return null;
+		if (Date.now() - loopStart >= cpuBudgetMs) return null;
+		return fetchPage(row);
+	};
+	let pending = startFetchIfAllowed(startRow);
+	if (pending === null) return {
+		totalRows,
+		hasMore: true,
+		nextStartRow: startRow,
+		metadata
+	};
+	while (pending !== null) {
+		const page = await pending;
+		pending = null;
+		if (page.kind === "error") throw page.error;
+		if (page.kind === "timeout") return {
+			totalRows,
+			hasMore: true,
+			nextStartRow: page.startRow,
+			metadata
+		};
+		const rows = page.rows;
+		totalRows += rows.length;
+		pageCount++;
+		if (page.metadata) metadata = page.metadata;
+		opts.onPage?.({
+			searchType,
+			rowsThisPage: rows.length
+		});
+		const isLastPage = rows.length === 0;
+		const nextStartRow = page.startRow + rows.length;
+		const prefetch = isLastPage ? null : startFetchIfAllowed(nextStartRow);
+		if (rows.length > 0) {
+			if (await opts.onBatch(rows).then(() => false).catch((err) => {
+				if (isTimeoutLike(err)) return true;
+				throw err;
+			})) return {
+				totalRows: totalRows - rows.length,
+				hasMore: true,
+				nextStartRow: page.startRow,
+				metadata
+			};
+		}
+		if (isLastPage) return {
+			totalRows,
+			hasMore: false,
+			nextStartRow,
+			metadata
+		};
+		if (prefetch === null) return {
+			totalRows,
+			hasMore: true,
+			nextStartRow,
+			metadata
+		};
+		startRow = nextStartRow;
+		pending = prefetch;
+	}
+	return {
+		totalRows,
+		hasMore: false,
+		nextStartRow: startRow,
+		metadata
+	};
+}
+function contextTableForGrain(grain) {
+	switch (grain) {
+		case "page": return "search_appearance_pages";
+		case "query": return "search_appearance_queries";
+		case "page_query": return "search_appearance_page_queries";
+	}
+}
+async function runGscSearchAppearanceContextSlice(opts) {
+	const table = opts.table ?? contextTableForGrain(opts.grain ?? "page_query");
+	const appearances = opts.continuation?.appearances?.slice() ?? opts.appearances?.slice() ?? [];
+	let totalRows = 0;
+	let hasMore = false;
+	if (!opts.appearances && opts.continuation?.phase !== "context") {
+		const discovered = new Set(appearances);
+		const discovery = await runGscSyncSlice({
+			client: opts.client,
+			siteUrl: opts.siteUrl,
+			table: "search_appearance",
+			startDate: opts.startDate,
+			endDate: opts.endDate,
+			domainFilter: opts.domainFilter,
+			dataState: opts.dataState,
+			rowLimit: opts.rowLimit,
+			maxPages: opts.maxPages,
+			cpuBudgetMs: opts.cpuBudgetMs,
+			searchType: opts.searchType,
+			initialStartRow: opts.continuation?.phase === "discovery" ? opts.continuation.nextStartRow : void 0,
+			onPage: opts.onPage,
+			onBatch: async (rows) => {
+				for (const row of rows) {
+					const value = String(row.keys?.[0] ?? "");
+					if (value) discovered.add(value);
+				}
+				await opts.onTotalBatch?.(rows);
+			}
+		});
+		totalRows += discovery.totalRows;
+		if (discovery.hasMore) return {
+			appearances: [...discovered],
+			totalRows,
+			hasMore: true,
+			continuation: {
+				phase: "discovery",
+				appearances: [...discovered],
+				nextStartRow: discovery.nextStartRow
+			}
+		};
+		hasMore ||= discovery.hasMore;
+		appearances.splice(0, appearances.length, ...discovered);
+	}
+	const startIndex = opts.continuation?.phase === "context" ? opts.continuation.appearanceIndex : 0;
+	const startRow = opts.continuation?.phase === "context" ? opts.continuation.nextStartRow : 0;
+	for (let i = startIndex; i < appearances.length; i++) {
+		const searchAppearance = appearances[i];
+		const context = await runGscSyncSlice({
+			client: opts.client,
+			siteUrl: opts.siteUrl,
+			table,
+			startDate: opts.startDate,
+			endDate: opts.endDate,
+			domainFilter: opts.domainFilter,
+			dimensionFilters: [{
+				dimension: "searchAppearance",
+				expression: searchAppearance
+			}],
+			dataState: opts.dataState,
+			rowLimit: opts.rowLimit,
+			maxPages: opts.maxPages,
+			cpuBudgetMs: opts.cpuBudgetMs,
+			searchType: opts.searchType,
+			initialStartRow: i === startIndex ? startRow : void 0,
+			onPage: opts.onPage,
+			onBatch: (rows) => opts.onContextBatch({
+				searchAppearance,
+				table,
+				rows
+			})
+		});
+		totalRows += context.totalRows;
+		if (context.hasMore) return {
+			appearances,
+			totalRows,
+			hasMore: true,
+			continuation: {
+				phase: "context",
+				appearances,
+				appearanceIndex: i,
+				nextStartRow: context.nextStartRow
+			}
+		};
+		hasMore ||= context.hasMore;
+	}
+	return {
+		appearances,
+		totalRows,
+		hasMore
+	};
+}
+export { runGscSearchAppearanceContextSlice, runGscSyncSlice };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-gsc-api",
   "type": "module",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "GSC live-API engine adapter — wraps the Search Analytics REST API as an AnalysisQuerySource for typed analyzer dispatch.",
   "author": {
     "name": "Harlan Wilton",
@@ -36,8 +36,8 @@
     "node": ">=22"
   },
   "dependencies": {
-    "@gscdump/engine": "^1.4.0",
-    "gscdump": "^1.4.0"
+    "@gscdump/engine": "^1.4.1",
+    "gscdump": "^1.4.1"
   },
   "devDependencies": {
     "vitest": "^4.1.10"