@gscdump/engine-gsc-api 1.4.0 → 1.4.1

package/dist/index.mjs

@@ -1,555 +1,5 @@
-import { googleSearchConsole } from "gscdump";
-import { assertDimensionsSupported, dimensionValue, getFilterDimensions, matchesTopLevelPage, metricValue } from "@gscdump/engine/resolver";
-import { between, clicks, date, extractMetricFilters, extractSpecialOperatorFilters, gsc } from "gscdump/query";
-import { buildLogicalPlan } from "gscdump/query/plan";
-import { normalizeUrl } from "gscdump/normalize";
-const METRIC_NAMES = [
-	"clicks",
-	"impressions",
-	"ctr",
-	"position"
-];
-function isMetricDimension$1(dim) {
-	return METRIC_NAMES.includes(dim);
-}
-function isLocalDimensionFilter(filter) {
-	return filter.dimension !== "date" && !isMetricDimension$1(filter.dimension) && filter.operator !== "topLevel" && !filter.operator.startsWith("metric");
-}
-function compileDimensionFilter(filter) {
-	const { dimension, expression } = filter;
-	switch (filter.operator) {
-		case "equals": return dimension === "page" ? (row) => normalizeUrl(dimensionValue(row, dimension)) === expression : (row) => dimensionValue(row, dimension) === expression;
-		case "notEquals": return dimension === "page" ? (row) => normalizeUrl(dimensionValue(row, dimension)) !== expression : (row) => dimensionValue(row, dimension) !== expression;
-		case "contains": return (row) => dimensionValue(row, dimension).includes(expression);
-		case "notContains": return (row) => !dimensionValue(row, dimension).includes(expression);
-		case "includingRegex": {
-			let regex;
-			return (row) => (regex ??= new RegExp(expression)).test(dimensionValue(row, dimension));
-		}
-		case "excludingRegex": {
-			let regex;
-			return (row) => !(regex ??= new RegExp(expression)).test(dimensionValue(row, dimension));
-		}
-		default: return () => true;
-	}
-}
-function compileMetricFilter(filter) {
-	const { dimension } = filter;
-	const target = Number(filter.expression);
-	switch (filter.operator) {
-		case "metricGte": return (row) => metricValue(row, dimension) >= target;
-		case "metricGt": return (row) => metricValue(row, dimension) > target;
-		case "metricLte": return (row) => metricValue(row, dimension) <= target;
-		case "metricLt": return (row) => metricValue(row, dimension) < target;
-		case "metricBetween": {
-			const target2 = Number(filter.expression2);
-			return (row) => {
-				const value = metricValue(row, dimension);
-				return value >= target && value <= target2;
-			};
-		}
-		default: return () => true;
-	}
-}
-function compileDimensionFilterTree(filter) {
-	if (!filter || !("_filters" in filter)) return () => true;
-	const localFilters = filter._filters;
-	const matchers = [];
-	for (const localFilter of localFilters) if (isLocalDimensionFilter(localFilter)) matchers.push(compileDimensionFilter(localFilter));
-	for (const group of filter._nestedGroups ?? []) matchers.push(compileDimensionFilterTree(group));
-	if (matchers.length === 0) return () => true;
-	if (filter._groupType === "or") return (row) => {
-		for (const matches of matchers) if (matches(row)) return true;
-		return false;
-	};
-	return (row) => {
-		for (const matches of matchers) if (!matches(row)) return false;
-		return true;
-	};
-}
-function selectTopK(rows, k, compareRows) {
-	const heap = [];
-	const compare = (a, b) => compareRows(a.row, b.row) || a.index - b.index;
-	const siftUp = (start) => {
-		let child = start;
-		while (child > 0) {
-			const parent = child - 1 >>> 1;
-			if (compare(heap[parent], heap[child]) >= 0) break;
-			const swap = heap[parent];
-			heap[parent] = heap[child];
-			heap[child] = swap;
-			child = parent;
-		}
-	};
-	const siftDown = () => {
-		let parent = 0;
-		while (true) {
-			const left = parent * 2 + 1;
-			if (left >= heap.length) return;
-			const right = left + 1;
-			const worse = right < heap.length && compare(heap[right], heap[left]) > 0 ? right : left;
-			if (compare(heap[parent], heap[worse]) >= 0) return;
-			const swap = heap[parent];
-			heap[parent] = heap[worse];
-			heap[worse] = swap;
-			parent = worse;
-		}
-	};
-	for (let index = 0; index < rows.length; index++) {
-		const candidate = {
-			row: rows[index],
-			index
-		};
-		if (heap.length < k) {
-			heap.push(candidate);
-			siftUp(heap.length - 1);
-		} else if (compare(candidate, heap[0]) < 0) {
-			heap[0] = candidate;
-			siftDown();
-		}
-	}
-	heap.sort(compare);
-	return heap.map((entry) => entry.row);
-}
-function applyBuilderStatePostProcessing(rows, state) {
-	const matchesDimensions = compileDimensionFilterTree(state.filter);
-	const metricMatchers = extractMetricFilters(state.filter).map(compileMetricFilter);
-	const hasTopLevelFilter = extractSpecialOperatorFilters(state.filter).some((filter) => filter.operator === "topLevel");
-	const column = state.orderBy?.column ?? "clicks";
-	const dir = state.orderBy?.dir ?? "desc";
-	const checkFiniteOrderValues = column !== "date" && state.rowLimit !== void 0;
-	let hasNonFiniteOrderValue = false;
-	const filtered = [];
-	for (const row of rows) {
-		if (!matchesDimensions(row)) continue;
-		let matchesMetrics = true;
-		for (const matchesMetric of metricMatchers) if (!matchesMetric(row)) {
-			matchesMetrics = false;
-			break;
-		}
-		if (!matchesMetrics) continue;
-		if (hasTopLevelFilter && !matchesTopLevelPage(row)) continue;
-		filtered.push(row);
-		if (checkFiniteOrderValues && !Number.isFinite(metricValue(row, column))) hasNonFiniteOrderValue = true;
-	}
-	const compareRows = (a, b) => {
-		const left = column === "date" ? String(a.date ?? "") : metricValue(a, column);
-		const right = column === "date" ? String(b.date ?? "") : metricValue(b, column);
-		if (left === right) return 0;
-		if (dir === "asc") return left < right ? -1 : 1;
-		return left > right ? -1 : 1;
-	};
-	const offset = Math.max(0, Number(state.startRow ?? 0));
-	const limit = Math.max(0, Number((state.rowLimit ?? filtered.length) || 0));
-	if (limit === 0) return [];
-	const requested = offset + limit;
-	return (Number.isInteger(requested) && requested > 0 && requested < filtered.length / 2 && !hasNonFiniteOrderValue ? selectTopK(filtered, requested, compareRows) : filtered.sort(compareRows)).slice(offset, offset + limit);
-}
-async function collectRows(gen) {
-	const out = [];
-	for await (const batch of gen) out.push(...batch);
-	return out;
-}
-async function fetchGscTopN(opts) {
-	const { client, siteUrl, dimension, range, orderByClicksDesc, limit, sliceTop, searchType } = opts;
-	let builder = gsc.select(dimension).where(between(date, range.start, range.end));
-	if (searchType) builder = builder.type(searchType);
-	if (orderByClicksDesc) builder = builder.orderBy(clicks, "desc");
-	if (typeof limit === "number") builder = builder.limit(limit);
-	const rows = await collectRows(client.query(siteUrl, builder));
-	const mapRow = (raw) => {
-		const row = raw;
-		const key = row[dimension.dimension];
-		if (typeof key !== "string" || !key) return null;
-		const impressions = Number(row.impressions ?? 0);
-		const position = Math.max(1, Number(row.position ?? 0));
-		return {
-			key,
-			clicks: Number(row.clicks ?? 0),
-			impressions,
-			sum_position: (position - 1) * impressions
-		};
-	};
-	const topLimit = typeof sliceTop === "number" && Number.isInteger(sliceTop) && sliceTop >= 0 ? sliceTop : void 0;
-	const mapped = [];
-	if (topLimit === 0) return mapped;
-	if (orderByClicksDesc && topLimit !== void 0) {
-		for (const row of rows) {
-			const value = mapRow(row);
-			if (value) mapped.push(value);
-			if (mapped.length >= topLimit) break;
-		}
-		return mapped;
-	}
-	if (!orderByClicksDesc && topLimit !== void 0 && topLimit <= 128) {
-		let requiresFullSort = false;
-		for (const row of rows) {
-			const value = mapRow(row);
-			if (!value) continue;
-			if (!Number.isFinite(value.clicks)) {
-				requiresFullSort = true;
-				break;
-			}
-			let insertAt = 0;
-			while (insertAt < mapped.length && mapped[insertAt].clicks >= value.clicks) insertAt++;
-			if (insertAt < topLimit) {
-				mapped.splice(insertAt, 0, value);
-				if (mapped.length > topLimit) mapped.pop();
-			}
-		}
-		if (!requiresFullSort) return mapped;
-		mapped.length = 0;
-	}
-	for (const row of rows) {
-		const value = mapRow(row);
-		if (value) mapped.push(value);
-	}
-	if (!orderByClicksDesc) mapped.sort((a, b) => b.clicks - a.clicks);
-	return typeof sliceTop === "number" ? mapped.slice(0, sliceTop) : mapped;
-}
-async function fetchGscDaily(opts) {
-	const { client, siteUrl, range, searchType } = opts;
-	let builder = gsc.select(date).where(between(date, range.start, range.end));
-	if (searchType) builder = builder.type(searchType);
-	const query = builder;
-	return (await collectRows(client.query(siteUrl, query))).map((r) => {
-		const row = r;
-		if (!row.date) return null;
-		const impressions = row.impressions ?? 0;
-		return {
-			date: Date.parse(`${row.date}T00:00:00Z`),
-			clicks: row.clicks ?? 0,
-			impressions,
-			sum_position: (Math.max(1, row.position ?? 0) - 1) * impressions,
-			anonymizedImpressionsPct: 0
-		};
-	}).filter((x) => x != null).sort((a, b) => a.date - b.date);
-}
-const GSC_API_CAPABILITIES = {
-	regex: true,
-	multiDataset: false,
-	comparisonJoin: false,
-	windowTotals: false
-};
-function isMetricDimension(dim) {
-	return [
-		"clicks",
-		"impressions",
-		"ctr",
-		"position"
-	].includes(dim);
-}
-function builderFromState(state) {
-	return { getState: () => state };
-}
-function canPushDownPagination(state) {
-	return !state.orderBy && extractMetricFilters(state.filter).length === 0 && extractSpecialOperatorFilters(state.filter).length === 0;
-}
-function createGscApiQuerySource(options) {
-	const { client, siteUrl } = options;
-	return {
-		name: "gsc-api",
-		kind: "live",
-		capabilities: GSC_API_CAPABILITIES,
-		async queryRows(state) {
-			buildLogicalPlan(state, GSC_API_CAPABILITIES);
-			const filterDims = getFilterDimensions(state.filter, isMetricDimension);
-			assertDimensionsSupported([...state.dimensions, ...filterDims], "api", "gsc-api query source");
-			const pushDownPagination = canPushDownPagination(state);
-			const apiState = pushDownPagination ? state : {
-				...state,
-				rowLimit: void 0,
-				startRow: void 0
-			};
-			return applyBuilderStatePostProcessing(await collectRows(client.query(siteUrl, builderFromState(apiState))), pushDownPagination ? {
-				...state,
-				rowLimit: void 0,
-				startRow: void 0
-			} : state);
-		}
-	};
-}
-const PRO_ONLY_DIMENSIONS = /* @__PURE__ */ new Set(["queryCanonical", "page_keywords"]);
-function canProxyToGsc(state) {
-	if (state.dimensions.some((d) => PRO_ONLY_DIMENSIONS.has(d))) return false;
-	return true;
-}
-function withSearchType(state, searchType) {
-	return state.searchType ? state : {
-		...state,
-		searchType
-	};
-}
-function createLiveGscSource(opts) {
-	let clientPromise = null;
-	function getClient() {
-		if (!clientPromise) clientPromise = opts.getAccessToken().then((accessToken) => opts.createClient?.(accessToken) ?? googleSearchConsole({ accessToken }));
-		return clientPromise;
-	}
-	return {
-		name: "gsc-api",
-		kind: "live",
-		capabilities: {
-			regex: true,
-			multiDataset: false,
-			comparisonJoin: false,
-			windowTotals: false
-		},
-		async queryRows(state) {
-			const client = await getClient();
-			const scopedState = opts.searchType !== void 0 ? withSearchType(state, opts.searchType) : state;
-			return createGscApiQuerySource({
-				client,
-				siteUrl: opts.siteUrl
-			}).queryRows(scopedState);
-		}
-	};
-}
-const DIMENSIONS_BY_TABLE = {
-	pages: ["page", "date"],
-	queries: ["query", "date"],
-	countries: ["country", "date"],
-	dates: ["device", "date"],
-	page_queries: [
-		"page",
-		"query",
-		"date"
-	],
-	search_appearance: ["searchAppearance"],
-	search_appearance_pages: ["page", "date"],
-	search_appearance_queries: ["query", "date"],
-	search_appearance_page_queries: [
-		"page",
-		"query",
-		"date"
-	],
-	hourly_pages: ["hour", "page"]
-};
-function isTimeoutLike(err) {
-	if (!(err instanceof Error)) return false;
-	return err.name === "AbortError" || err.message?.includes("timeout") || err.message?.includes("aborted");
-}
-function buildDimensionFilterGroups(domainFilter, filters = []) {
-	const out = filters.map((f) => ({
-		dimension: f.dimension,
-		operator: f.operator ?? "equals",
-		expression: f.expression
-	}));
-	if (domainFilter?.domain) {
-		const pattern = `^https?://(www\\.)?${domainFilter.domain.replace(/^www\./, "").replace(/\./g, "\\.")}/`;
-		out.push({
-			dimension: "page",
-			operator: "includingRegex",
-			expression: pattern
-		});
-	}
-	return out.length > 0 ? [{ filters: out }] : void 0;
-}
-async function runGscSyncSlice(opts) {
-	const rowLimit = opts.rowLimit ?? 500;
-	const cpuBudgetMs = opts.cpuBudgetMs ?? 2e4;
-	const maxPages = opts.maxPages ?? Infinity;
-	const searchType = opts.searchType ?? "web";
-	const dimensions = opts.dimensions ? [...opts.dimensions] : [...DIMENSIONS_BY_TABLE[opts.table]];
-	const dataState = opts.dataState ?? (dimensions.includes("hour") ? "hourly_all" : "all");
-	const dimensionFilterGroups = buildDimensionFilterGroups(opts.domainFilter, opts.dimensionFilters);
-	const loopStart = Date.now();
-	let startRow = opts.initialStartRow ?? 0;
-	let totalRows = 0;
-	let pageCount = 0;
-	let metadata;
-	const fetchPage = async (row) => {
-		const query = {
-			startDate: opts.startDate,
-			endDate: opts.endDate,
-			dimensions,
-			rowLimit,
-			startRow: row,
-			dataState,
-			type: searchType,
-			...dimensionFilterGroups ? { dimensionFilterGroups } : {}
-		};
-		try {
-			const response = await opts.client.searchAnalytics.query(opts.siteUrl, query);
-			return {
-				kind: "ok",
-				startRow: row,
-				rows: response.rows ?? [],
-				metadata: response.metadata
-			};
-		} catch (err) {
-			if (isTimeoutLike(err)) return {
-				kind: "timeout",
-				startRow: row
-			};
-			return {
-				kind: "error",
-				error: err
-			};
-		}
-	};
-	const startFetchIfAllowed = (row) => {
-		if (pageCount >= maxPages) return null;
-		if (Date.now() - loopStart >= cpuBudgetMs) return null;
-		return fetchPage(row);
-	};
-	let pending = startFetchIfAllowed(startRow);
-	if (pending === null) return {
-		totalRows,
-		hasMore: true,
-		nextStartRow: startRow,
-		metadata
-	};
-	while (pending !== null) {
-		const page = await pending;
-		pending = null;
-		if (page.kind === "error") throw page.error;
-		if (page.kind === "timeout") return {
-			totalRows,
-			hasMore: true,
-			nextStartRow: page.startRow,
-			metadata
-		};
-		const rows = page.rows;
-		totalRows += rows.length;
-		pageCount++;
-		if (page.metadata) metadata = page.metadata;
-		opts.onPage?.({
-			searchType,
-			rowsThisPage: rows.length
-		});
-		const isLastPage = rows.length === 0;
-		const nextStartRow = page.startRow + rows.length;
-		const prefetch = isLastPage ? null : startFetchIfAllowed(nextStartRow);
-		if (rows.length > 0) {
-			if (await opts.onBatch(rows).then(() => false).catch((err) => {
-				if (isTimeoutLike(err)) return true;
-				throw err;
-			})) return {
-				totalRows: totalRows - rows.length,
-				hasMore: true,
-				nextStartRow: page.startRow,
-				metadata
-			};
-		}
-		if (isLastPage) return {
-			totalRows,
-			hasMore: false,
-			nextStartRow,
-			metadata
-		};
-		if (prefetch === null) return {
-			totalRows,
-			hasMore: true,
-			nextStartRow,
-			metadata
-		};
-		startRow = nextStartRow;
-		pending = prefetch;
-	}
-	return {
-		totalRows,
-		hasMore: false,
-		nextStartRow: startRow,
-		metadata
-	};
-}
-function contextTableForGrain(grain) {
-	switch (grain) {
-		case "page": return "search_appearance_pages";
-		case "query": return "search_appearance_queries";
-		case "page_query": return "search_appearance_page_queries";
-	}
-}
-async function runGscSearchAppearanceContextSlice(opts) {
-	const table = opts.table ?? contextTableForGrain(opts.grain ?? "page_query");
-	const appearances = opts.continuation?.appearances?.slice() ?? opts.appearances?.slice() ?? [];
-	let totalRows = 0;
-	let hasMore = false;
-	if (!opts.appearances && opts.continuation?.phase !== "context") {
-		const discovered = new Set(appearances);
-		const discovery = await runGscSyncSlice({
-			client: opts.client,
-			siteUrl: opts.siteUrl,
-			table: "search_appearance",
-			startDate: opts.startDate,
-			endDate: opts.endDate,
-			domainFilter: opts.domainFilter,
-			dataState: opts.dataState,
-			rowLimit: opts.rowLimit,
-			maxPages: opts.maxPages,
-			cpuBudgetMs: opts.cpuBudgetMs,
-			searchType: opts.searchType,
-			initialStartRow: opts.continuation?.phase === "discovery" ? opts.continuation.nextStartRow : void 0,
-			onPage: opts.onPage,
-			onBatch: async (rows) => {
-				for (const row of rows) {
-					const value = String(row.keys?.[0] ?? "");
-					if (value) discovered.add(value);
-				}
-				await opts.onTotalBatch?.(rows);
-			}
-		});
-		totalRows += discovery.totalRows;
-		if (discovery.hasMore) return {
-			appearances: [...discovered],
-			totalRows,
-			hasMore: true,
-			continuation: {
-				phase: "discovery",
-				appearances: [...discovered],
-				nextStartRow: discovery.nextStartRow
-			}
-		};
-		hasMore ||= discovery.hasMore;
-		appearances.splice(0, appearances.length, ...discovered);
-	}
-	const startIndex = opts.continuation?.phase === "context" ? opts.continuation.appearanceIndex : 0;
-	const startRow = opts.continuation?.phase === "context" ? opts.continuation.nextStartRow : 0;
-	for (let i = startIndex; i < appearances.length; i++) {
-		const searchAppearance = appearances[i];
-		const context = await runGscSyncSlice({
-			client: opts.client,
-			siteUrl: opts.siteUrl,
-			table,
-			startDate: opts.startDate,
-			endDate: opts.endDate,
-			domainFilter: opts.domainFilter,
-			dimensionFilters: [{
-				dimension: "searchAppearance",
-				expression: searchAppearance
-			}],
-			dataState: opts.dataState,
-			rowLimit: opts.rowLimit,
-			maxPages: opts.maxPages,
-			cpuBudgetMs: opts.cpuBudgetMs,
-			searchType: opts.searchType,
-			initialStartRow: i === startIndex ? startRow : void 0,
-			onPage: opts.onPage,
-			onBatch: (rows) => opts.onContextBatch({
-				searchAppearance,
-				table,
-				rows
-			})
-		});
-		totalRows += context.totalRows;
-		if (context.hasMore) return {
-			appearances,
-			totalRows,
-			hasMore: true,
-			continuation: {
-				phase: "context",
-				appearances,
-				appearanceIndex: i,
-				nextStartRow: context.nextStartRow
-			}
-		};
-		hasMore ||= context.hasMore;
-	}
-	return {
-		appearances,
-		totalRows,
-		hasMore
-	};
-}
+import { fetchGscDaily, fetchGscTopN } from "./rollup-synth.mjs";
+import { createGscApiQuerySource } from "./source.mjs";
+import { canProxyToGsc, createLiveGscSource } from "./live.mjs";
+import { runGscSearchAppearanceContextSlice, runGscSyncSlice } from "./sync-slice.mjs";
 export { canProxyToGsc, createGscApiQuerySource, createLiveGscSource, fetchGscDaily, fetchGscTopN, runGscSearchAppearanceContextSlice, runGscSyncSlice };

package/dist/live.d.mts

@@ -0,0 +1,25 @@
+import { GoogleSearchConsoleClient } from "gscdump";
+import { BuilderState } from "gscdump/query";
+import { SearchType as SearchType$1 } from "@gscdump/engine";
+import { AnalysisQuerySource } from "@gscdump/engine/source";
+declare function canProxyToGsc(state: BuilderState): boolean;
+interface CreateLiveGscSourceOptions {
+  /** GSC property URL (e.g. `sc-domain:example.com` or `https://example.com/`). */
+  siteUrl: string;
+  /**
+   * Returns a valid GSC access token. Called lazily on first query so refresh
+   * cost is paid only when the source actually runs. Host owns refresh logic.
+   */
+  getAccessToken: () => Promise<string>;
+  /** Optional host client factory (for custom timeouts, fetch, or telemetry). */
+  createClient?: (accessToken: string) => GoogleSearchConsoleClient | Promise<GoogleSearchConsoleClient>;
+  /**
+   * GSC `searchType` slice this source is scoped to (`web`, `discover`,
+   * `news`, `googleNews`, `image`, `video`). When set, the slice is injected
+   * into every outgoing builder state so the live API returns rows for that
+   * slice only. An explicit search type already present on the state wins.
+   */
+  searchType?: SearchType$1;
+}
+declare function createLiveGscSource(opts: CreateLiveGscSourceOptions): AnalysisQuerySource;
+export { CreateLiveGscSourceOptions, canProxyToGsc, createLiveGscSource };

package/dist/live.mjs

@@ -0,0 +1,39 @@
+import { createGscApiQuerySource } from "./source.mjs";
+import { googleSearchConsole } from "gscdump";
+const PRO_ONLY_DIMENSIONS = /* @__PURE__ */ new Set(["queryCanonical", "page_keywords"]);
+function canProxyToGsc(state) {
+	if (state.dimensions.some((d) => PRO_ONLY_DIMENSIONS.has(d))) return false;
+	return true;
+}
+function withSearchType(state, searchType) {
+	return state.searchType ? state : {
+		...state,
+		searchType
+	};
+}
+function createLiveGscSource(opts) {
+	let clientPromise = null;
+	function getClient() {
+		if (!clientPromise) clientPromise = opts.getAccessToken().then((accessToken) => opts.createClient?.(accessToken) ?? googleSearchConsole({ accessToken }));
+		return clientPromise;
+	}
+	return {
+		name: "gsc-api",
+		kind: "live",
+		capabilities: {
+			regex: true,
+			multiDataset: false,
+			comparisonJoin: false,
+			windowTotals: false
+		},
+		async queryRows(state) {
+			const client = await getClient();
+			const scopedState = opts.searchType !== void 0 ? withSearchType(state, opts.searchType) : state;
+			return createGscApiQuerySource({
+				client,
+				siteUrl: opts.siteUrl
+			}).queryRows(scopedState);
+		}
+	};
+}
+export { canProxyToGsc, createLiveGscSource };