@gscdump/engine-gsc-api 1.3.2 → 1.4.1

package/dist/post-process.mjs

@@ -0,0 +1,146 @@
+import { dimensionValue, matchesTopLevelPage, metricValue } from "@gscdump/engine/resolver";
+import { extractMetricFilters, extractSpecialOperatorFilters } from "gscdump/query";
+import { normalizeUrl } from "gscdump/normalize";
+const METRIC_NAMES = [
+	"clicks",
+	"impressions",
+	"ctr",
+	"position"
+];
+function isMetricDimension(dim) {
+	return METRIC_NAMES.includes(dim);
+}
+function isLocalDimensionFilter(filter) {
+	return filter.dimension !== "date" && !isMetricDimension(filter.dimension) && filter.operator !== "topLevel" && !filter.operator.startsWith("metric");
+}
+function compileDimensionFilter(filter) {
+	const { dimension, expression } = filter;
+	switch (filter.operator) {
+		case "equals": return dimension === "page" ? (row) => normalizeUrl(dimensionValue(row, dimension)) === expression : (row) => dimensionValue(row, dimension) === expression;
+		case "notEquals": return dimension === "page" ? (row) => normalizeUrl(dimensionValue(row, dimension)) !== expression : (row) => dimensionValue(row, dimension) !== expression;
+		case "contains": return (row) => dimensionValue(row, dimension).includes(expression);
+		case "notContains": return (row) => !dimensionValue(row, dimension).includes(expression);
+		case "includingRegex": {
+			let regex;
+			return (row) => (regex ??= new RegExp(expression)).test(dimensionValue(row, dimension));
+		}
+		case "excludingRegex": {
+			let regex;
+			return (row) => !(regex ??= new RegExp(expression)).test(dimensionValue(row, dimension));
+		}
+		default: return () => true;
+	}
+}
+function compileMetricFilter(filter) {
+	const { dimension } = filter;
+	const target = Number(filter.expression);
+	switch (filter.operator) {
+		case "metricGte": return (row) => metricValue(row, dimension) >= target;
+		case "metricGt": return (row) => metricValue(row, dimension) > target;
+		case "metricLte": return (row) => metricValue(row, dimension) <= target;
+		case "metricLt": return (row) => metricValue(row, dimension) < target;
+		case "metricBetween": {
+			const target2 = Number(filter.expression2);
+			return (row) => {
+				const value = metricValue(row, dimension);
+				return value >= target && value <= target2;
+			};
+		}
+		default: return () => true;
+	}
+}
+function compileDimensionFilterTree(filter) {
+	if (!filter || !("_filters" in filter)) return () => true;
+	const localFilters = filter._filters;
+	const matchers = [];
+	for (const localFilter of localFilters) if (isLocalDimensionFilter(localFilter)) matchers.push(compileDimensionFilter(localFilter));
+	for (const group of filter._nestedGroups ?? []) matchers.push(compileDimensionFilterTree(group));
+	if (matchers.length === 0) return () => true;
+	if (filter._groupType === "or") return (row) => {
+		for (const matches of matchers) if (matches(row)) return true;
+		return false;
+	};
+	return (row) => {
+		for (const matches of matchers) if (!matches(row)) return false;
+		return true;
+	};
+}
+function selectTopK(rows, k, compareRows) {
+	const heap = [];
+	const compare = (a, b) => compareRows(a.row, b.row) || a.index - b.index;
+	const siftUp = (start) => {
+		let child = start;
+		while (child > 0) {
+			const parent = child - 1 >>> 1;
+			if (compare(heap[parent], heap[child]) >= 0) break;
+			const swap = heap[parent];
+			heap[parent] = heap[child];
+			heap[child] = swap;
+			child = parent;
+		}
+	};
+	const siftDown = () => {
+		let parent = 0;
+		while (true) {
+			const left = parent * 2 + 1;
+			if (left >= heap.length) return;
+			const right = left + 1;
+			const worse = right < heap.length && compare(heap[right], heap[left]) > 0 ? right : left;
+			if (compare(heap[parent], heap[worse]) >= 0) return;
+			const swap = heap[parent];
+			heap[parent] = heap[worse];
+			heap[worse] = swap;
+			parent = worse;
+		}
+	};
+	for (let index = 0; index < rows.length; index++) {
+		const candidate = {
+			row: rows[index],
+			index
+		};
+		if (heap.length < k) {
+			heap.push(candidate);
+			siftUp(heap.length - 1);
+		} else if (compare(candidate, heap[0]) < 0) {
+			heap[0] = candidate;
+			siftDown();
+		}
+	}
+	heap.sort(compare);
+	return heap.map((entry) => entry.row);
+}
+function applyBuilderStatePostProcessing(rows, state) {
+	const matchesDimensions = compileDimensionFilterTree(state.filter);
+	const metricMatchers = extractMetricFilters(state.filter).map(compileMetricFilter);
+	const hasTopLevelFilter = extractSpecialOperatorFilters(state.filter).some((filter) => filter.operator === "topLevel");
+	const column = state.orderBy?.column ?? "clicks";
+	const dir = state.orderBy?.dir ?? "desc";
+	const checkFiniteOrderValues = column !== "date" && state.rowLimit !== void 0;
+	let hasNonFiniteOrderValue = false;
+	const filtered = [];
+	for (const row of rows) {
+		if (!matchesDimensions(row)) continue;
+		let matchesMetrics = true;
+		for (const matchesMetric of metricMatchers) if (!matchesMetric(row)) {
+			matchesMetrics = false;
+			break;
+		}
+		if (!matchesMetrics) continue;
+		if (hasTopLevelFilter && !matchesTopLevelPage(row)) continue;
+		filtered.push(row);
+		if (checkFiniteOrderValues && !Number.isFinite(metricValue(row, column))) hasNonFiniteOrderValue = true;
+	}
+	const compareRows = (a, b) => {
+		const left = column === "date" ? String(a.date ?? "") : metricValue(a, column);
+		const right = column === "date" ? String(b.date ?? "") : metricValue(b, column);
+		if (left === right) return 0;
+		if (dir === "asc") return left < right ? -1 : 1;
+		return left > right ? -1 : 1;
+	};
+	const offset = Math.max(0, Number(state.startRow ?? 0));
+	const limit = Math.max(0, Number((state.rowLimit ?? filtered.length) || 0));
+	if (limit === 0) return [];
+	const requested = offset + limit;
+	return (Number.isInteger(requested) && requested > 0 && requested < filtered.length / 2 && !hasNonFiniteOrderValue ? selectTopK(filtered, requested, compareRows) : filtered.sort(compareRows)).slice(offset, offset + limit);
+}
+export { applyBuilderStatePostProcessing };

package/dist/rollup-synth.d.mts

@@ -0,0 +1,45 @@
+import { GoogleSearchConsoleClient } from "gscdump";
+import { Column, Dimension, SearchType } from "gscdump/query";
+interface GscRange {
+  start: string;
+  end: string;
+}
+interface GscTopNRow {
+  key: string;
+  clicks: number;
+  impressions: number;
+  sum_position: number;
+}
+interface FetchTopNOptions<D extends Dimension> {
+  client: GoogleSearchConsoleClient;
+  siteUrl: string;
+  dimension: Column<D>;
+  range: GscRange;
+  /**
+   * Ask the GSC API to order by clicks desc. Skip for dimensions where GSC
+   * already returns sensibly ranked rows (e.g. country).
+   */
+  orderByClicksDesc?: boolean;
+  /** Forwarded to the GSC builder. */
+  limit?: number;
+  /** Trim after the fact (e.g. country has no server-side limit). */
+  sliceTop?: number;
+  /** GSC search corpus; callers default API-boundary omissions to web. */
+  searchType?: SearchType;
+}
+declare function fetchGscTopN<D extends Dimension>(opts: FetchTopNOptions<D>): Promise<GscTopNRow[]>;
+interface GscDailyRow {
+  date: number;
+  clicks: number;
+  impressions: number;
+  sum_position: number;
+  anonymizedImpressionsPct: number;
+}
+declare function fetchGscDaily(opts: {
+  client: GoogleSearchConsoleClient;
+  siteUrl: string;
+  range: GscRange;
+  /** GSC search corpus; callers default API-boundary omissions to web. */
+  searchType?: SearchType;
+}): Promise<GscDailyRow[]>;
+export { FetchTopNOptions, GscDailyRow, GscRange, GscTopNRow, fetchGscDaily, fetchGscTopN };

package/dist/rollup-synth.mjs

@@ -0,0 +1,82 @@
+import { between, clicks, date, gsc } from "gscdump/query";
+async function collectRows(gen) {
+	const out = [];
+	for await (const batch of gen) out.push(...batch);
+	return out;
+}
+async function fetchGscTopN(opts) {
+	const { client, siteUrl, dimension, range, orderByClicksDesc, limit, sliceTop, searchType } = opts;
+	let builder = gsc.select(dimension).where(between(date, range.start, range.end));
+	if (searchType) builder = builder.type(searchType);
+	if (orderByClicksDesc) builder = builder.orderBy(clicks, "desc");
+	if (typeof limit === "number") builder = builder.limit(limit);
+	const rows = await collectRows(client.query(siteUrl, builder));
+	const mapRow = (raw) => {
+		const row = raw;
+		const key = row[dimension.dimension];
+		if (typeof key !== "string" || !key) return null;
+		const impressions = Number(row.impressions ?? 0);
+		const position = Math.max(1, Number(row.position ?? 0));
+		return {
+			key,
+			clicks: Number(row.clicks ?? 0),
+			impressions,
+			sum_position: (position - 1) * impressions
+		};
+	};
+	const topLimit = typeof sliceTop === "number" && Number.isInteger(sliceTop) && sliceTop >= 0 ? sliceTop : void 0;
+	const mapped = [];
+	if (topLimit === 0) return mapped;
+	if (orderByClicksDesc && topLimit !== void 0) {
+		for (const row of rows) {
+			const value = mapRow(row);
+			if (value) mapped.push(value);
+			if (mapped.length >= topLimit) break;
+		}
+		return mapped;
+	}
+	if (!orderByClicksDesc && topLimit !== void 0 && topLimit <= 128) {
+		let requiresFullSort = false;
+		for (const row of rows) {
+			const value = mapRow(row);
+			if (!value) continue;
+			if (!Number.isFinite(value.clicks)) {
+				requiresFullSort = true;
+				break;
+			}
+			let insertAt = 0;
+			while (insertAt < mapped.length && mapped[insertAt].clicks >= value.clicks) insertAt++;
+			if (insertAt < topLimit) {
+				mapped.splice(insertAt, 0, value);
+				if (mapped.length > topLimit) mapped.pop();
+			}
+		}
+		if (!requiresFullSort) return mapped;
+		mapped.length = 0;
+	}
+	for (const row of rows) {
+		const value = mapRow(row);
+		if (value) mapped.push(value);
+	}
+	if (!orderByClicksDesc) mapped.sort((a, b) => b.clicks - a.clicks);
+	return typeof sliceTop === "number" ? mapped.slice(0, sliceTop) : mapped;
+}
+async function fetchGscDaily(opts) {
+	const { client, siteUrl, range, searchType } = opts;
+	let builder = gsc.select(date).where(between(date, range.start, range.end));
+	if (searchType) builder = builder.type(searchType);
+	const query = builder;
+	return (await collectRows(client.query(siteUrl, query))).map((r) => {
+		const row = r;
+		if (!row.date) return null;
+		const impressions = row.impressions ?? 0;
+		return {
+			date: Date.parse(`${row.date}T00:00:00Z`),
+			clicks: row.clicks ?? 0,
+			impressions,
+			sum_position: (Math.max(1, row.position ?? 0) - 1) * impressions,
+			anonymizedImpressionsPct: 0
+		};
+	}).filter((x) => x != null).sort((a, b) => a.date - b.date);
+}
+export { collectRows, fetchGscDaily, fetchGscTopN };

package/dist/source.d.mts

@@ -0,0 +1,8 @@
+import { GoogleSearchConsoleClient } from "gscdump";
+import { AnalysisQuerySource } from "@gscdump/engine/source";
+interface GscApiQuerySourceOptions {
+  client: GoogleSearchConsoleClient;
+  siteUrl: string;
+}
+declare function createGscApiQuerySource(options: GscApiQuerySourceOptions): AnalysisQuerySource;
+export { GscApiQuerySourceOptions, createGscApiQuerySource };

package/dist/source.mjs

@@ -0,0 +1,50 @@
+import { applyBuilderStatePostProcessing } from "./post-process.mjs";
+import { collectRows } from "./rollup-synth.mjs";
+import { assertDimensionsSupported, getFilterDimensions } from "@gscdump/engine/resolver";
+import { extractMetricFilters, extractSpecialOperatorFilters } from "gscdump/query";
+import { buildLogicalPlan } from "gscdump/query/plan";
+const GSC_API_CAPABILITIES = {
+	regex: true,
+	multiDataset: false,
+	comparisonJoin: false,
+	windowTotals: false
+};
+function isMetricDimension(dim) {
+	return [
+		"clicks",
+		"impressions",
+		"ctr",
+		"position"
+	].includes(dim);
+}
+function builderFromState(state) {
+	return { getState: () => state };
+}
+function canPushDownPagination(state) {
+	return !state.orderBy && extractMetricFilters(state.filter).length === 0 && extractSpecialOperatorFilters(state.filter).length === 0;
+}
+function createGscApiQuerySource(options) {
+	const { client, siteUrl } = options;
+	return {
+		name: "gsc-api",
+		kind: "live",
+		capabilities: GSC_API_CAPABILITIES,
+		async queryRows(state) {
+			buildLogicalPlan(state, GSC_API_CAPABILITIES);
+			const filterDims = getFilterDimensions(state.filter, isMetricDimension);
+			assertDimensionsSupported([...state.dimensions, ...filterDims], "api", "gsc-api query source");
+			const pushDownPagination = canPushDownPagination(state);
+			const apiState = pushDownPagination ? state : {
+				...state,
+				rowLimit: void 0,
+				startRow: void 0
+			};
+			return applyBuilderStatePostProcessing(await collectRows(client.query(siteUrl, builderFromState(apiState))), pushDownPagination ? {
+				...state,
+				rowLimit: void 0,
+				startRow: void 0
+			} : state);
+		}
+	};
+}
+export { GSC_API_CAPABILITIES, createGscApiQuerySource };

package/dist/sync-slice.d.mts

@@ -0,0 +1,136 @@
+import { GoogleSearchConsoleClient } from "gscdump";
+import { SearchType } from "@gscdump/engine";
+import { GscDataState, GscSearchAnalyticsMetadata } from "gscdump/contracts";
+interface GscApiRow {
+  keys: string[];
+  clicks: number;
+  impressions: number;
+  ctr: number;
+  position: number;
+}
+interface SyncSliceDomainFilter {
+  /**
+   * Domain (eTLD+1 + subdomain) to scope the slice to — matches both
+   *  `www.` and bare variants. Strip the protocol; the regex is built here.
+   */
+  domain?: string;
+}
+interface SyncSliceDimensionFilter {
+  dimension: 'page' | 'query' | 'country' | 'device' | 'searchAppearance';
+  operator?: 'equals' | 'notEquals' | 'contains' | 'notContains' | 'includingRegex' | 'excludingRegex';
+  expression: string;
+}
+interface RunGscSyncSliceOptions {
+  client: GoogleSearchConsoleClient;
+  siteUrl: string;
+  /** One of the engine sync-fan tables. Drives the dimension list. */
+  table: 'pages' | 'queries' | 'countries' | 'dates' | 'page_queries' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries' | 'hourly_pages';
+  startDate: string;
+  endDate: string;
+  domainFilter?: SyncSliceDomainFilter | null;
+  /** Additional AND filters, e.g. `searchAppearance = AMP_BLUE_LINK`. */
+  dimensionFilters?: SyncSliceDimensionFilter[];
+  /**
+   * Override the dimension list for this slice. Defaults to
+   *  `DIMENSIONS_BY_TABLE[table]`. Hosts that need bespoke groupings (e.g.
+   *  hourly Discover variants) supply this directly.
+   */
+  dimensions?: string[];
+  /**
+   * GSC `dataState` for the query. Defaults to `'all'`. Use `'hourly_all'`
+   *  for hourly Discover slices.
+   */
+  dataState?: GscDataState;
+  /**
+   * Invoked per GSC API page with the rows fetched. Return a promise; the
+   *  loop awaits it before paging further. Throw a durable error to abort the
+   *  slice. A thrown AbortError / timeout stops the loop and returns retry
+   *  state at the current cursor so the continuation re-processes this page.
+   */
+  onBatch: (rows: GscApiRow[]) => Promise<void>;
+  initialStartRow?: number;
+  /**
+   * Max GSC API pages per call. `Infinity` for unbounded; hosts cap this
+   *  based on path (D1 vs R2) and table.
+   */
+  maxPages?: number;
+  /** GSC page size. 500 is the D1-safe default; 10k is the R2 path. */
+  rowLimit?: number;
+  /**
+   * Soft CPU budget for the loop itself. Returns `hasMore` when crossed so
+   *  the continuation resumes from `nextStartRow`.
+   */
+  cpuBudgetMs?: number;
+  searchType?: SearchType;
+  /** Invoked once per successful GSC API page. Hosts wire telemetry here. */
+  onPage?: (info: {
+    searchType: SearchType;
+    rowsThisPage: number;
+  }) => void;
+}
+interface RunGscSyncSliceResult {
+  totalRows: number;
+  hasMore: boolean;
+  nextStartRow: number;
+  /**
+   * Metadata from the LAST GSC API page seen during this slice run. When
+   *  `dataState='hourly_all'` and grouped by `hour`, this surfaces
+   *  `first_incomplete_hour` so hosts can watermark hourly progress.
+   */
+  metadata?: GscSearchAnalyticsMetadata;
+}
+type SearchAppearanceContextGrain = 'page' | 'query' | 'page_query';
+type SearchAppearanceContextTable = 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries';
+interface RunGscSearchAppearanceContextSliceOptions {
+  client: GoogleSearchConsoleClient;
+  siteUrl: string;
+  startDate: string;
+  endDate: string;
+  domainFilter?: SyncSliceDomainFilter | null;
+  /** Use a known appearance list to skip discovery. */
+  appearances?: string[];
+  /** Context grain to fetch for every discovered appearance. Defaults to page_query. */
+  grain?: SearchAppearanceContextGrain;
+  /** Context table to fetch. Overrides `grain` when provided. */
+  table?: SearchAppearanceContextTable;
+  dataState?: GscDataState;
+  rowLimit?: number;
+  maxPages?: number;
+  cpuBudgetMs?: number;
+  searchType?: SearchType;
+  onTotalBatch?: (rows: GscApiRow[]) => Promise<void>;
+  onContextBatch: (batch: {
+    searchAppearance: string;
+    table: SearchAppearanceContextTable;
+    rows: GscApiRow[];
+  }) => Promise<void>;
+  onPage?: (info: {
+    searchType: SearchType;
+    rowsThisPage: number;
+  }) => void;
+  continuation?: SearchAppearanceContinuation;
+}
+type SearchAppearanceContinuation = {
+  phase: 'discovery';
+  appearances: string[];
+  nextStartRow: number;
+} | {
+  phase: 'context';
+  appearances: string[];
+  appearanceIndex: number;
+  nextStartRow: number;
+};
+interface RunGscSearchAppearanceContextSliceResult {
+  appearances: string[];
+  totalRows: number;
+  hasMore: boolean;
+  continuation?: SearchAppearanceContinuation;
+}
+declare function runGscSyncSlice(opts: RunGscSyncSliceOptions): Promise<RunGscSyncSliceResult>;
+/**
+ * Implements GSC's required two-step search-appearance flow:
+ * 1. group by `searchAppearance` alone to discover available appearances;
+ * 2. for each appearance, filter by it and fetch page/query/date context.
+ */
+declare function runGscSearchAppearanceContextSlice(opts: RunGscSearchAppearanceContextSliceOptions): Promise<RunGscSearchAppearanceContextSliceResult>;
+export { GscApiRow, RunGscSearchAppearanceContextSliceOptions, RunGscSearchAppearanceContextSliceResult, RunGscSyncSliceOptions, RunGscSyncSliceResult, SearchAppearanceContextGrain, SearchAppearanceContextTable, SearchAppearanceContinuation, SyncSliceDimensionFilter, SyncSliceDomainFilter, runGscSearchAppearanceContextSlice, runGscSyncSlice };