gscdump 0.3.1 → 0.5.2

package/dist/query/plan.d.mts

@@ -0,0 +1,130 @@
+declare const _default: {
+  name: string;
+  'alpha-2': string;
+  'alpha-3': string;
+  'country-code': string;
+}[];
+declare const Devices: {
+  readonly MOBILE: "MOBILE";
+  readonly DESKTOP: "DESKTOP";
+  readonly TABLET: "TABLET";
+};
+type Device = typeof Devices[keyof typeof Devices];
+declare const SearchTypes: {
+  readonly WEB: "web";
+  readonly IMAGE: "image";
+  readonly VIDEO: "video";
+  readonly NEWS: "news";
+  readonly DISCOVER: "discover";
+  readonly GOOGLE_NEWS: "googleNews";
+};
+type SearchType = typeof SearchTypes[keyof typeof SearchTypes];
+declare const Countries: { [K in (typeof _default)[number]["alpha-3"]]: Lowercase<K> };
+type Country = typeof Countries[keyof typeof Countries];
+interface DimensionValueMap {
+  query: string;
+  queryCanonical: string;
+  page: string;
+  country: Country;
+  device: Device;
+  searchAppearance: string;
+  date: string;
+}
+type Dimension = keyof DimensionValueMap;
+interface QueryParamValueMap {
+  searchType: SearchType;
+}
+type QueryParamName = keyof QueryParamValueMap;
+type FilterOperator = 'equals' | 'notEquals' | 'contains' | 'notContains' | 'includingRegex' | 'excludingRegex';
+type DateOperator = 'gte' | 'gt' | 'lte' | 'lt' | 'between';
+type MetricOperator = 'metricGte' | 'metricGt' | 'metricLte' | 'metricLt' | 'metricBetween';
+type SpecialOperator = 'topLevel';
+interface InternalFilter {
+  dimension: Dimension | QueryParamName | Metric;
+  operator: FilterOperator | DateOperator | MetricOperator | SpecialOperator;
+  expression: string;
+  expression2?: string;
+}
+interface Filter<C = object> {
+  readonly __filterBrand: 'gscdump.Filter';
+  readonly _constraints: C;
+  readonly _filters: InternalFilter[];
+  readonly _nestedGroups?: Filter<any>[];
+  readonly _groupType?: 'and' | 'or';
+}
+type Metric = 'clicks' | 'impressions' | 'ctr' | 'position';
+interface BuilderState {
+  dimensions: Dimension[];
+  metrics?: Metric[];
+  filter?: Filter<any>;
+  orderBy?: {
+    column: Metric | 'date';
+    dir: 'asc' | 'desc';
+  };
+  rowLimit?: number;
+  startRow?: number;
+}
+/** Logical table / dataset identifier. Canonical across query builder + storage engine. */
+type TableName = 'pages' | 'keywords' | 'countries' | 'devices' | 'page_keywords' | 'search_appearance';
+type LogicalDataset = TableName;
+type ComparisonFilter = 'new' | 'lost' | 'improving' | 'declining';
+interface PlannerCapabilities {
+  regex?: boolean;
+  multiDataset?: boolean;
+  comparisonJoin?: boolean;
+  windowTotals?: boolean;
+}
+interface LogicalDimensionFilter {
+  dimension: Dimension;
+  operator: FilterOperator;
+  expression: string;
+  expression2?: string;
+}
+interface LogicalMetricFilter {
+  metric: Metric;
+  operator: MetricOperator;
+  expression: number;
+  expression2?: number;
+}
+interface LogicalFilterLeaf {
+  kind: 'leaf';
+  filter: LogicalDimensionFilter;
+}
+interface LogicalFilterGroup {
+  kind: 'group';
+  groupType: 'and' | 'or';
+  children: LogicalFilterNode[];
+}
+type LogicalFilterNode = LogicalFilterLeaf | LogicalFilterGroup;
+interface LogicalQueryPlan {
+  dataset: LogicalDataset;
+  dimensions: Dimension[];
+  groupByDimensions: Dimension[];
+  hasDate: boolean;
+  metrics: Metric[];
+  dateRange: {
+    startDate: string;
+    endDate: string;
+  };
+  dimensionFilters: LogicalDimensionFilter[];
+  dimensionFilterTree?: LogicalFilterNode;
+  metricFilters: LogicalMetricFilter[];
+  specialFilters: {
+    topLevel: boolean;
+  };
+  queryParams: Partial<Record<QueryParamName, string>>;
+  orderBy?: BuilderState['orderBy'];
+  rowLimit?: number;
+  startRow?: number;
+}
+interface LogicalComparisonPlan {
+  current: LogicalQueryPlan;
+  previous: LogicalQueryPlan;
+  comparisonFilter?: ComparisonFilter;
+}
+declare class UnsupportedLogicalCapabilityError extends Error {
+  constructor(capability: keyof PlannerCapabilities, context: string);
+}
+declare function buildLogicalPlan(state: BuilderState, capabilities?: PlannerCapabilities): LogicalQueryPlan;
+declare function buildLogicalComparisonPlan(current: BuilderState, previous: BuilderState, capabilities?: PlannerCapabilities, comparisonFilter?: ComparisonFilter): LogicalComparisonPlan;
+export { type ComparisonFilter, type LogicalComparisonPlan, type LogicalDataset, type LogicalDimensionFilter, LogicalFilterGroup, LogicalFilterLeaf, LogicalFilterNode, type LogicalMetricFilter, type LogicalQueryPlan, type PlannerCapabilities, type TableName, UnsupportedLogicalCapabilityError, buildLogicalComparisonPlan, buildLogicalPlan };

package/dist/query/plan.mjs

@@ -0,0 +1,296 @@
+const DATE_OPERATORS = [
+	"gte",
+	"gt",
+	"lte",
+	"lt",
+	"between"
+];
+const METRIC_OPERATORS = [
+	"metricGte",
+	"metricGt",
+	"metricLte",
+	"metricLt",
+	"metricBetween"
+];
+const SPECIAL_OPERATORS = ["topLevel"];
+const QUERY_PARAMS = ["searchType"];
+const FILTER_METRICS = [
+	"clicks",
+	"impressions",
+	"ctr",
+	"position"
+];
+function isDateOperator(op) {
+	return DATE_OPERATORS.includes(op);
+}
+function isMetricOperator(op) {
+	return METRIC_OPERATORS.includes(op);
+}
+function isSpecialOperator(op) {
+	return SPECIAL_OPERATORS.includes(op);
+}
+function isQueryParam(value) {
+	return QUERY_PARAMS.includes(value);
+}
+function isMetric(value) {
+	return FILTER_METRICS.includes(value);
+}
+function isRegexOperator(op) {
+	return op === "includingRegex" || op === "excludingRegex";
+}
+const MS_PER_DAY = 864e5;
+function toIsoDate(d) {
+	return d.toISOString().slice(0, 10);
+}
+function addDays(dateStr, n) {
+	return toIsoDate(new Date(Date.parse(`${dateStr}T00:00:00Z`) + n * MS_PER_DAY));
+}
+function isWireGroupType(type) {
+	return type === "and" || type === "or";
+}
+function convertWireLeaf(alt) {
+	if (!alt.column || !alt.type || isWireGroupType(alt.type)) return null;
+	const f = {
+		dimension: alt.column,
+		operator: alt.type,
+		expression: alt.type === "between" ? alt.from ?? "" : alt.value ?? ""
+	};
+	if (alt.type === "between" && alt.to) f.expression2 = alt.to;
+	return f;
+}
+function convertWireGroup(alt) {
+	if (!isWireGroupType(alt.type)) {
+		const leaf = convertWireLeaf(alt);
+		return leaf ? { _filters: [leaf] } : null;
+	}
+	const leaves = [];
+	const nested = [];
+	for (const child of alt.filters ?? []) if (isWireGroupType(child.type)) {
+		const sub = convertWireGroup(child);
+		if (sub) nested.push(sub);
+	} else {
+		const leaf = convertWireLeaf(child);
+		if (leaf) leaves.push(leaf);
+	}
+	if (leaves.length === 0 && nested.length === 0) return null;
+	return {
+		_filters: leaves,
+		_nestedGroups: nested.length > 0 ? nested : void 0,
+		_groupType: alt.type
+	};
+}
+function isWireFilter(input) {
+	if (!input || typeof input !== "object") return false;
+	const o = input;
+	if ("_filters" in o) return false;
+	return "type" in o && typeof o.type === "string" || "filters" in o && Array.isArray(o.filters);
+}
+function normalizeFilter(input) {
+	if (!input) return void 0;
+	if (isWireFilter(input)) return convertWireGroup(input) ?? void 0;
+	return input;
+}
+function extractSpecialFilters(filter) {
+	if (!filter) return {};
+	let startDate;
+	let endDate;
+	let searchType;
+	const otherFilters = [];
+	const cleanedNestedGroups = [];
+	for (const f of filter._filters) if (f.dimension === "date" && isDateOperator(f.operator)) switch (f.operator) {
+		case "gte":
+			startDate = f.expression;
+			break;
+		case "gt":
+			startDate = addDays(f.expression, 1);
+			break;
+		case "lte":
+			endDate = f.expression;
+			break;
+		case "lt":
+			endDate = addDays(f.expression, -1);
+			break;
+		case "between":
+			startDate = f.expression;
+			endDate = f.expression2;
+			break;
+	}
+	else if (isQueryParam(f.dimension)) {
+		if (f.dimension === "searchType") searchType = f.expression;
+	} else if (isMetricOperator(f.operator) || isSpecialOperator(f.operator)) otherFilters.push(f);
+	else otherFilters.push(f);
+	if (filter._nestedGroups) for (const nested of filter._nestedGroups) {
+		const extracted = extractSpecialFilters(nested);
+		if (extracted.startDate) startDate = extracted.startDate;
+		if (extracted.endDate) endDate = extracted.endDate;
+		if (extracted.searchType) searchType = extracted.searchType;
+		if (extracted.dimensionFilter) cleanedNestedGroups.push(extracted.dimensionFilter);
+	}
+	const dimensionFilter = otherFilters.length > 0 || cleanedNestedGroups.length > 0 ? {
+		...filter,
+		_filters: otherFilters,
+		_nestedGroups: cleanedNestedGroups.length > 0 ? cleanedNestedGroups : void 0
+	} : void 0;
+	return {
+		startDate,
+		endDate,
+		searchType,
+		dimensionFilter
+	};
+}
+function extractDateRange(input) {
+	const { startDate, endDate } = extractSpecialFilters(normalizeFilter(input));
+	return {
+		startDate,
+		endDate
+	};
+}
+function extractMetricFilters(input) {
+	const filter = normalizeFilter(input);
+	if (!filter) return [];
+	const metricFilters = filter._filters.filter((f) => isMetricOperator(f.operator));
+	const nested = filter._nestedGroups?.flatMap((g) => extractMetricFilters(g)) ?? [];
+	return [...metricFilters, ...nested];
+}
+function extractSpecialOperatorFilters(input) {
+	const filter = normalizeFilter(input);
+	if (!filter) return [];
+	const special = filter._filters.filter((f) => isSpecialOperator(f.operator));
+	const nested = filter._nestedGroups?.flatMap((g) => extractSpecialOperatorFilters(g)) ?? [];
+	return [...special, ...nested];
+}
+var UnsupportedLogicalCapabilityError = class extends Error {
+	constructor(capability, context) {
+		super(`${context} requires ${capability} capability`);
+		this.name = "UnsupportedLogicalCapabilityError";
+	}
+};
+function collectInternalFilters(filter) {
+	if (!filter || !("_filters" in filter)) return [];
+	const flat = filter._filters;
+	const nested = filter._nestedGroups?.flatMap((group) => collectInternalFilters(group)) ?? [];
+	return [...flat, ...nested];
+}
+function inferDataset(dimensions, filterDims = []) {
+	const allDims = new Set([...dimensions, ...filterDims]);
+	const has = (dimension) => allDims.has(dimension);
+	if (has("searchAppearance")) return "search_appearance";
+	if (has("page") && (has("query") || has("queryCanonical"))) return "page_keywords";
+	if (has("query") || has("queryCanonical")) return "keywords";
+	if (has("page")) return "pages";
+	if (has("country")) return "countries";
+	if (has("device")) return "devices";
+	return "keywords";
+}
+function requireCapability(capabilities, capability, enabled, context) {
+	if (enabled && !capabilities?.[capability]) throw new UnsupportedLogicalCapabilityError(capability, context);
+}
+function isDimensionLeaf(filter) {
+	if (isMetric(filter.dimension)) return false;
+	if (filter.dimension === "date" && isDateOperator(filter.operator)) return false;
+	if (filter.operator === "topLevel" || filter.operator.startsWith("metric")) return false;
+	if (isQueryParam(filter.dimension)) return false;
+	return true;
+}
+function toLogicalDimensionFilter(filter) {
+	return {
+		dimension: filter.dimension,
+		operator: filter.operator,
+		expression: filter.expression,
+		expression2: filter.expression2
+	};
+}
+function buildDimensionFilterTree(filter, capabilities) {
+	if (!filter || !("_filters" in filter)) return void 0;
+	const groupType = filter._groupType ?? "and";
+	const children = [];
+	for (const f of filter._filters) {
+		if (!isDimensionLeaf(f)) continue;
+		requireCapability(capabilities, "regex", isRegexOperator(f.operator), "logical plan");
+		children.push({
+			kind: "leaf",
+			filter: toLogicalDimensionFilter(f)
+		});
+	}
+	for (const g of filter._nestedGroups ?? []) {
+		const sub = buildDimensionFilterTree(g, capabilities);
+		if (sub) children.push(sub);
+	}
+	if (children.length === 0) return void 0;
+	if (children.length === 1) return children[0];
+	return {
+		kind: "group",
+		groupType,
+		children
+	};
+}
+function buildLogicalPlan(state, capabilities = {}) {
+	const normalizedFilter = normalizeFilter(state.filter);
+	const { startDate, endDate } = extractDateRange(normalizedFilter);
+	if (!startDate || !endDate) throw new Error("logical plan requires date range (use between(date, ...) or gte/lte)");
+	const allFilters = collectInternalFilters(normalizedFilter);
+	const metricFilters = extractMetricFilters(normalizedFilter);
+	const specialFilters = extractSpecialOperatorFilters(normalizedFilter);
+	const queryParams = {};
+	const dimensionFilters = [];
+	for (const filter of allFilters) {
+		if (isMetric(filter.dimension)) continue;
+		if (filter.dimension === "date" && isDateOperator(filter.operator)) continue;
+		if (filter.operator === "topLevel" || filter.operator.startsWith("metric")) continue;
+		if (isQueryParam(filter.dimension)) {
+			queryParams[filter.dimension] = filter.expression;
+			continue;
+		}
+		const operator = filter.operator;
+		requireCapability(capabilities, "regex", isRegexOperator(operator), "logical plan");
+		dimensionFilters.push({
+			dimension: filter.dimension,
+			operator,
+			expression: filter.expression,
+			expression2: filter.expression2
+		});
+	}
+	const dimensionFilterTree = buildDimensionFilterTree(normalizedFilter, capabilities);
+	const filterDims = dimensionFilters.map((filter) => filter.dimension);
+	return {
+		dataset: inferDataset(state.dimensions, filterDims),
+		dimensions: [...state.dimensions],
+		groupByDimensions: state.dimensions.filter((d) => d !== "date"),
+		hasDate: state.dimensions.includes("date"),
+		metrics: state.metrics ? [...state.metrics] : [
+			"clicks",
+			"impressions",
+			"ctr",
+			"position"
+		],
+		dateRange: {
+			startDate,
+			endDate
+		},
+		dimensionFilters,
+		dimensionFilterTree,
+		metricFilters: metricFilters.map((filter) => ({
+			metric: filter.dimension,
+			operator: filter.operator,
+			expression: Number(filter.expression),
+			expression2: filter.expression2 == null ? void 0 : Number(filter.expression2)
+		})),
+		specialFilters: { topLevel: specialFilters.some((filter) => filter.operator === "topLevel") },
+		queryParams,
+		orderBy: state.orderBy,
+		rowLimit: state.rowLimit,
+		startRow: state.startRow
+	};
+}
+function buildLogicalComparisonPlan(current, previous, capabilities = {}, comparisonFilter) {
+	requireCapability(capabilities, "comparisonJoin", true, "logical comparison plan");
+	const currentPlan = buildLogicalPlan(current, capabilities);
+	const previousPlan = buildLogicalPlan(previous, capabilities);
+	requireCapability(capabilities, "multiDataset", currentPlan.dataset !== previousPlan.dataset, "logical comparison plan");
+	return {
+		current: currentPlan,
+		previous: previousPlan,
+		comparisonFilter
+	};
+}
+export { UnsupportedLogicalCapabilityError, buildLogicalComparisonPlan, buildLogicalPlan };

package/dist/sitemap.d.mts

@@ -0,0 +1,13 @@
+interface DiscoverSitemapOptions {
+  /** User-Agent sent on the discovery requests. */
+  userAgent?: string;
+  /** AbortSignal threaded through fetches; defaults to a 10s timeout per call. */
+  signal?: AbortSignal;
+}
+/**
+ * Try to discover a sitemap for `domain` by checking robots.txt for a
+ * `Sitemap:` directive, then a small set of common paths. Returns the first
+ * URL that responds with a 2xx, or `null`.
+ */
+declare function discoverSitemap(domain: string, options?: DiscoverSitemapOptions): Promise<string | null>;
+export { DiscoverSitemapOptions, discoverSitemap };

package/dist/sitemap.mjs

@@ -0,0 +1,31 @@
+const FETCH_TIMEOUT_MS = 1e4;
+const COMMON_PATHS = ["/sitemap.xml", "/sitemap_index.xml"];
+const SITEMAP_DIRECTIVE_RE = /^Sitemap:\s*(\S+)/im;
+async function discoverSitemap(domain, options = {}) {
+	const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
+	const baseUrl = `https://${domain}`;
+	const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
+	const robotsRes = await fetch(`${baseUrl}/robots.txt`, {
+		headers: { "User-Agent": userAgent },
+		signal: signalFor()
+	}).catch(() => null);
+	if (robotsRes?.ok) {
+		const match = (await robotsRes.text()).match(SITEMAP_DIRECTIVE_RE);
+		if (match?.[1]) {
+			if ((await fetch(match[1], {
+				method: "HEAD",
+				signal: signalFor()
+			}).catch(() => null))?.ok) return match[1];
+		}
+	}
+	for (const path of COMMON_PATHS) {
+		const url = `${baseUrl}${path}`;
+		if ((await fetch(url, {
+			method: "HEAD",
+			headers: { "User-Agent": userAgent },
+			signal: signalFor()
+		}).catch(() => null))?.ok) return url;
+	}
+	return null;
+}
+export { discoverSitemap };

package/dist/tenant.d.mts

@@ -0,0 +1,18 @@
+declare function encodeSiteId(siteUrl: string): string;
+/**
+ * Best-effort inverse of `encodeSiteId` for the common prefixes. Lossy
+ * (`encodeSiteId` collapses non-word chars to `_` and strips trailing
+ * underscores), but round-trips domain properties + https origins cleanly —
+ * the only two shapes GSC hands out in practice.
+ *
+ * Returns the input unchanged when neither prefix is recognised, so callers
+ * can pass through canonical site URLs without branching.
+ */
+declare function decodeSiteId(encoded: string): string;
+/**
+ * Normalize a siteUrl to the form Google APIs expect: domain properties get
+ * the `sc-domain:` prefix added if missing, URL properties pass through.
+ * Idempotent — safe to call on already-prefixed values.
+ */
+declare function normalizeSiteUrl(siteUrl: string): string;
+export { decodeSiteId, encodeSiteId, normalizeSiteUrl };

package/dist/tenant.mjs

@@ -0,0 +1,18 @@
+const SC_DOMAIN_RE = /^sc-domain:/;
+const PROTOCOL_RE = /^https?:\/\//;
+const NON_ID_RE = /[^\w.-]/g;
+const TRAILING_UNDERSCORES_RE = /_+$/;
+function encodeSiteId(siteUrl) {
+	return siteUrl.replace(SC_DOMAIN_RE, "d_").replace(PROTOCOL_RE, "h_").replace(NON_ID_RE, "_").replace(TRAILING_UNDERSCORES_RE, "");
+}
+function decodeSiteId(encoded) {
+	if (encoded.startsWith("d_")) return `sc-domain:${encoded.slice(2)}`;
+	if (encoded.startsWith("h_")) return `https://${encoded.slice(2)}/`;
+	return encoded;
+}
+function normalizeSiteUrl(siteUrl) {
+	if (siteUrl.startsWith("sc-domain:")) return siteUrl;
+	if (siteUrl.startsWith("http://") || siteUrl.startsWith("https://")) return siteUrl;
+	return `sc-domain:${siteUrl}`;
+}
+export { decodeSiteId, encodeSiteId, normalizeSiteUrl };

package/dist/url.d.mts

@@ -0,0 +1,9 @@
+/**
+ * Equivalence key for matching two URLs that point at the same resource
+ * across protocol (http/https), www. prefix, trailing slash, and casing.
+ * Returns `null` for unparseable input.
+ *
+ * Example: `https://www.Example.com/Foo/` → `example.com/foo`
+ */
+declare function urlMatchKey(url: string): string | null;
+export { urlMatchKey };

package/dist/url.mjs

@@ -0,0 +1,6 @@
+function urlMatchKey(url) {
+	const u = URL.parse(url);
+	if (!u) return null;
+	return [u.hostname.replace(/^www\./, ""), u.pathname.replace(/\/$/, "") || "/"].join("").toLowerCase();
+}
+export { urlMatchKey };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gscdump",
   "type": "module",
-  "version": "0.3.1",
+  "version": "0.5.2",
   "description": "Google Search Console API wrapper with typed query builder, streaming pagination, and SEO analysis functions",
   "author": {
     "name": "Harlan Wilton",
@@ -40,9 +40,33 @@
       "types": "./dist/query/index.d.mts",
       "import": "./dist/query/index.mjs"
     },
-    "./analysis": {
-      "types": "./dist/analysis/index.d.mts",
-      "import": "./dist/analysis/index.mjs"
+    "./query/plan": {
+      "types": "./dist/query/plan.d.mts",
+      "import": "./dist/query/plan.mjs"
+    },
+    "./contracts": {
+      "types": "./dist/contracts.d.mts",
+      "import": "./dist/contracts.mjs"
+    },
+    "./driver": {
+      "types": "./dist/driver.d.mts",
+      "import": "./dist/driver.mjs"
+    },
+    "./tenant": {
+      "types": "./dist/tenant.d.mts",
+      "import": "./dist/tenant.mjs"
+    },
+    "./normalize": {
+      "types": "./dist/normalize.d.mts",
+      "import": "./dist/normalize.mjs"
+    },
+    "./url": {
+      "types": "./dist/url.d.mts",
+      "import": "./dist/url.mjs"
+    },
+    "./sitemap": {
+      "types": "./dist/sitemap.d.mts",
+      "import": "./dist/sitemap.mjs"
     }
   },
   "main": "./dist/index.mjs",
@@ -53,9 +77,21 @@
   "engines": {
     "node": ">=18"
   },
+  "peerDependencies": {
+    "@googleapis/indexing": "^6.0.1",
+    "@googleapis/searchconsole": "^6.0.1"
+  },
+  "peerDependenciesMeta": {
+    "@googleapis/indexing": {
+      "optional": true
+    },
+    "@googleapis/searchconsole": {
+      "optional": true
+    }
+  },
   "dependencies": {
-    "dayjs": "^1.11.19",
-    "defu": "^6.1.4",
+    "dayjs": "^1.11.20",
+    "defu": "^6.1.7",
     "ofetch": "^1.5.1",
     "ufo": "^1.6.3"
   },
@@ -68,8 +104,6 @@
     "build": "obuild",
     "typecheck": "tsc --noEmit",
     "test": "vitest",
-    "test:run": "vitest run",
-    "test:e2e": "vitest run e2e/",
-    "test:real": "RUN_REAL_CREDENTIALS=true vitest run e2e/real-credentials.test.ts"
+    "test:run": "vitest run"
   }
 }