gscdump 1.0.1 → 1.0.3

package/dist/_chunks/builder.d.mts

@@ -0,0 +1,24 @@
+import { GscAggregationType, GscDataState, GscSearchAnalyticsRequest, GscSearchType } from "./contracts.mjs";
+import { BuilderState, Column, Dimension, Filter, Metric, MetricColumn } from "./types.mjs";
+type SelectableColumn = Column<Dimension> | MetricColumn<Metric>;
+type OrderableColumn = MetricColumn<Metric> | Column<'date'>;
+type ExtractDimensions<T extends SelectableColumn[]> = { [K in keyof T]: T[K] extends Column<infer D> ? D : never; }[number] extends (infer U) ? Exclude<U, never>[] : never;
+interface GSCQueryBuilder<D extends Dimension[] = [], C = object> {
+  select: {
+    <T extends Dimension[]>(...dims: T): GSCQueryBuilder<T, C>;
+    <T extends SelectableColumn[]>(...cols: T): GSCQueryBuilder<ExtractDimensions<T> & Dimension[], C>;
+  };
+  where: <F extends Filter<any>>(filter: F) => GSCQueryBuilder<D, C & F['_constraints']>;
+  prefilter: <F extends Filter<any>>(filter: F) => GSCQueryBuilder<D, C>;
+  orderBy: (col: OrderableColumn, dir: 'asc' | 'desc') => GSCQueryBuilder<D, C>;
+  limit: (n: number) => GSCQueryBuilder<D, C>;
+  offset: (n: number) => GSCQueryBuilder<D, C>;
+  dataState: (state: GscDataState) => GSCQueryBuilder<D, C>;
+  aggregationType: (type: GscAggregationType) => GSCQueryBuilder<D, C>;
+  /** GSC search corpus (`type` on the wire). Equivalent to filtering by `searchType`. */
+  type: (t: GscSearchType) => GSCQueryBuilder<D, C>;
+  toBody: () => GscSearchAnalyticsRequest;
+  getState: () => BuilderState;
+}
+declare const gsc: GSCQueryBuilder<[], object>;
+export { GSCQueryBuilder, gsc };

package/dist/_chunks/contracts.d.mts

@@ -0,0 +1,47 @@
+import { ColumnDef, ColumnType, Row, TableName, TableSchema, TenantCtx } from "@gscdump/contracts";
+type GscSearchAnalyticsDimension = 'page' | 'query' | 'country' | 'device' | 'date' | 'hour' | 'searchAppearance';
+type GscDataState = 'final' | 'all' | 'hourly_all';
+interface GscSearchAnalyticsMetadata {
+  /** First date (YYYY-MM-DD, PT) still being collected. Populated when dataState=`all` and grouped by `date`. */
+  first_incomplete_date?: string;
+  /** First hour (YYYY-MM-DDThh:mm:ss±hh:mm, PT) still being collected. Populated when dataState=`hourly_all` and grouped by `hour`. */
+  first_incomplete_hour?: string;
+}
+type GscSearchAnalyticsFilterOperator = 'equals' | 'notEquals' | 'contains' | 'notContains' | 'includingRegex' | 'excludingRegex';
+interface GscSearchAnalyticsFilter {
+  dimension: GscSearchAnalyticsDimension;
+  expression: string;
+  operator?: GscSearchAnalyticsFilterOperator;
+}
+interface GscSearchAnalyticsFilterGroup {
+  groupType?: 'and' | 'or';
+  filters: GscSearchAnalyticsFilter[];
+}
+type GscSearchType = 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews';
+type GscAggregationType = 'auto' | 'byPage' | 'byProperty' | 'byNewsShowcasePanel';
+type GscResponseAggregationType = 'auto' | 'byPage' | 'byProperty' | 'byNewsShowcasePanel';
+interface GscSearchAnalyticsRequest {
+  startDate: string;
+  endDate: string;
+  dimensions?: GscSearchAnalyticsDimension[];
+  dimensionFilterGroups?: GscSearchAnalyticsFilterGroup[];
+  rowLimit?: number;
+  startRow?: number;
+  /** GSC search corpus. Maps to wire field `type` (the API still accepts the deprecated `searchType` alias). */
+  type?: GscSearchType;
+  dataState?: GscDataState;
+  aggregationType?: GscAggregationType;
+}
+interface GscSearchAnalyticsRow {
+  keys: string[];
+  clicks: number;
+  impressions: number;
+  ctr: number;
+  position: number;
+}
+interface GscSearchAnalyticsResponse {
+  rows?: GscSearchAnalyticsRow[];
+  responseAggregationType?: GscResponseAggregationType;
+  metadata?: GscSearchAnalyticsMetadata;
+}
+export { type ColumnDef, type ColumnType, GscAggregationType, GscDataState, GscResponseAggregationType, GscSearchAnalyticsDimension, GscSearchAnalyticsFilter, GscSearchAnalyticsFilterGroup, GscSearchAnalyticsFilterOperator, GscSearchAnalyticsMetadata, GscSearchAnalyticsRequest, GscSearchAnalyticsResponse, GscSearchAnalyticsRow, GscSearchType, type Row, type TableName, type TableSchema, type TenantCtx };

package/dist/_chunks/dayjs.d.mts

@@ -0,0 +1,3 @@
+declare function currentPstDate(): string;
+declare function daysAgoPst(n: number): string;
+export { currentPstDate, daysAgoPst };

package/dist/_chunks/dayjs.mjs

@@ -0,0 +1,30 @@
+import { format } from "date-fns/format";
+import { subDays } from "date-fns/subDays";
+let pstFormatter;
+function getPstFormatter() {
+	pstFormatter ??= new Intl.DateTimeFormat("en-CA", {
+		timeZone: "America/Los_Angeles",
+		year: "numeric",
+		month: "2-digit",
+		day: "2-digit"
+	});
+	return pstFormatter;
+}
+function pstDateParts(d = /* @__PURE__ */ new Date()) {
+	const parts = getPstFormatter().formatToParts(d);
+	const get = (t) => Number(parts.find((p) => p.type === t).value);
+	return {
+		year: get("year"),
+		month: get("month"),
+		day: get("day")
+	};
+}
+function currentPstDate() {
+	const { year, month, day } = pstDateParts();
+	return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
+}
+function daysAgoPst(n) {
+	const { year, month, day } = pstDateParts();
+	return format(subDays(new Date(year, month - 1, day, 12), n), "yyyy-MM-dd");
+}
+export { currentPstDate, daysAgoPst };

package/dist/_chunks/gsc-dates.d.mts

@@ -0,0 +1,64 @@
+/** Milliseconds in a UTC day. Use for date arithmetic without DST surprises. */
+declare const MS_PER_DAY = 86400000;
+/** Format a Date as YYYY-MM-DD (UTC). */
+declare function toIsoDate(d: Date): string;
+/** Most recent date GSC has fully finalized data for (no further updates). */
+declare const GSC_FINALIZED_LAG_DAYS = 3;
+/** Most recent date GSC will return data for, may still be updating. */
+declare const GSC_FRESHEST_LAG_DAYS = 1;
+/** Approximate historical retention window for Search Analytics. */
+declare const GSC_RETENTION_MONTHS = 16;
+/** Today's date (YYYY-MM-DD) in PST. */
+declare function getPstDate(): string;
+/** YYYY-MM-DD for `now() - n` days, UTC. */
+declare function daysAgo(n: number): string;
+/** Most recent finalized date (3 days ago PST). */
+declare function getLatestGscDate(): string;
+/** Freshest date GSC will return (yesterday PST, may still update). */
+declare function getFreshestGscDate(): string;
+/**
+ * Dates that are still pending (1-3 days ago PST). These need to be re-synced
+ * each day until they finalize.
+ */
+declare function getPendingDates(): string[];
+/** All dates between two YYYY-MM-DD strings (inclusive, oldest first). */
+declare function getDateRange(startDate: string, endDate: string): string[];
+/** Default span used when chunking long backfills into batches. */
+declare const DAYS_PER_RANGE = 30;
+/**
+ * Like {@link getDateRange} but skips dates outside GSC's queryable window
+ * (older than retention or newer than the freshest date).
+ */
+declare function generateGscDateRange(startDate: string, endDate: string): string[];
+/**
+ * Group a sorted list of dates into contiguous runs, splitting on either a
+ * gap or after `daysPerRange` dates accumulate. Useful for bucketing
+ * backfill work into bounded jobs.
+ */
+declare function groupIntoRanges(dates: string[], daysPerRange?: number): Array<{
+  startDate: string;
+  endDate: string;
+}>;
+/** Inclusive day count between two YYYY-MM-DD strings. */
+declare function countDays(startDate: string, endDate: string): number;
+/** Oldest date GSC retains (~16 months ago). */
+declare function getOldestGscDate(): string;
+/** Add `n` UTC days to a YYYY-MM-DD string (n may be negative). */
+declare function addDays(dateStr: string, n: number): string;
+declare function getPreviousDate(dateStr: string): string;
+declare function getNextDate(dateStr: string): string;
+/** True if `dateStr` falls within GSC's queryable window. */
+declare function isValidGscDate(dateStr: string): boolean;
+interface BackfillProgress {
+  progress: number;
+  daysAvailable: number;
+  daysSynced: number;
+  oldestGscDate: string;
+  isComplete: boolean;
+}
+/**
+ * Backfill progress (0-1) given the oldest + newest dates synced.
+ * Returns null when no sync data exists.
+ */
+declare function getBackfillProgress(oldestDateSynced: string | null, newestDateSynced: string | null): BackfillProgress | null;
+export { BackfillProgress, DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_RETENTION_MONTHS, MS_PER_DAY, addDays, countDays, daysAgo, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, groupIntoRanges, isValidGscDate, toIsoDate };

package/dist/_chunks/gsc-dates.mjs

@@ -0,0 +1,100 @@
+const MS_PER_DAY = 864e5;
+function toIsoDate(d) {
+	return d.toISOString().slice(0, 10);
+}
+const GSC_FINALIZED_LAG_DAYS = 3;
+const GSC_FRESHEST_LAG_DAYS = 1;
+const GSC_RETENTION_MONTHS = 16;
+function getPstDate() {
+	return (/* @__PURE__ */ new Date()).toLocaleDateString("en-CA", { timeZone: "America/Los_Angeles" });
+}
+function getPstDateDaysAgo(daysAgo) {
+	const pstNow = new Date((/* @__PURE__ */ new Date()).toLocaleString("en-US", { timeZone: "America/Los_Angeles" }));
+	pstNow.setDate(pstNow.getDate() - daysAgo);
+	return toIsoDate(pstNow);
+}
+function daysAgo(n) {
+	return toIsoDate(/* @__PURE__ */ new Date(Date.now() - n * MS_PER_DAY));
+}
+function getLatestGscDate() {
+	return getPstDateDaysAgo(3);
+}
+function getFreshestGscDate() {
+	return getPstDateDaysAgo(1);
+}
+function getPendingDates() {
+	const dates = [];
+	for (let daysAgo = 1; daysAgo <= 3; daysAgo++) dates.push(getPstDateDaysAgo(daysAgo));
+	return dates;
+}
+function getDateRange(startDate, endDate) {
+	const dates = [];
+	const endMs = Date.parse(`${endDate}T00:00:00Z`);
+	for (let cursor = Date.parse(`${startDate}T00:00:00Z`); cursor <= endMs; cursor += MS_PER_DAY) dates.push(toIsoDate(new Date(cursor)));
+	return dates;
+}
+const DAYS_PER_RANGE = 30;
+function generateGscDateRange(startDate, endDate) {
+	const start = startDate < getOldestGscDate() ? getOldestGscDate() : startDate;
+	const end = endDate > getFreshestGscDate() ? getFreshestGscDate() : endDate;
+	return start <= end ? getDateRange(start, end) : [];
+}
+function groupIntoRanges(dates, daysPerRange = 30) {
+	if (dates.length === 0) return [];
+	const sorted = [...dates].sort();
+	const ranges = [];
+	let rangeStart = sorted[0];
+	let rangePrev = sorted[0];
+	for (let i = 1; i < sorted.length; i++) {
+		const current = sorted[i];
+		const expected = getNextDate(rangePrev);
+		const daysInRange = countDays(rangeStart, rangePrev);
+		if (current !== expected || daysInRange >= daysPerRange) {
+			ranges.push({
+				startDate: rangeStart,
+				endDate: rangePrev
+			});
+			rangeStart = current;
+		}
+		rangePrev = current;
+	}
+	ranges.push({
+		startDate: rangeStart,
+		endDate: rangePrev
+	});
+	return ranges;
+}
+function countDays(startDate, endDate) {
+	return Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / MS_PER_DAY) + 1;
+}
+function getOldestGscDate() {
+	const date = /* @__PURE__ */ new Date();
+	date.setMonth(date.getMonth() - 16);
+	return toIsoDate(date);
+}
+function addDays(dateStr, n) {
+	return toIsoDate(new Date(Date.parse(`${dateStr}T00:00:00Z`) + n * MS_PER_DAY));
+}
+function getPreviousDate(dateStr) {
+	return addDays(dateStr, -1);
+}
+function getNextDate(dateStr) {
+	return addDays(dateStr, 1);
+}
+function isValidGscDate(dateStr) {
+	return dateStr >= getOldestGscDate() && dateStr <= getFreshestGscDate();
+}
+function getBackfillProgress(oldestDateSynced, newestDateSynced) {
+	if (!oldestDateSynced || !newestDateSynced) return null;
+	const oldestGsc = getOldestGscDate();
+	const totalDays = countDays(oldestGsc, getLatestGscDate());
+	const syncedDays = countDays(oldestDateSynced, newestDateSynced);
+	return {
+		progress: Math.round(Math.min(1, syncedDays / totalDays) * 100) / 100,
+		daysAvailable: totalDays,
+		daysSynced: syncedDays,
+		oldestGscDate: oldestGsc,
+		isComplete: oldestDateSynced <= oldestGsc
+	};
+}
+export { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_RETENTION_MONTHS, MS_PER_DAY, addDays, countDays, daysAgo, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, groupIntoRanges, isValidGscDate, toIsoDate };

package/dist/_chunks/plan.d.mts

@@ -0,0 +1,175 @@
+import { TableName } from "./contracts.mjs";
+import { Result } from "./result.mjs";
+import { BuilderState, Dimension, FilterOperator, Metric, MetricOperator, QueryParamName } from "./types.mjs";
+type QueryErrorKind = 'missing-date-range' | 'invalid-row-limit' | 'invalid-start-row' | 'invalid-data-state' | 'invalid-aggregation-type' | 'invalid-builder-state' | 'invalid-filter' | 'unsupported-capability' | 'unresolvable-dataset';
+type QueryError = {
+  kind: 'missing-date-range';
+  message: string;
+} | {
+  kind: 'invalid-row-limit';
+  value: unknown;
+  message: string;
+} | {
+  kind: 'invalid-start-row';
+  value: unknown;
+  message: string;
+} | {
+  kind: 'invalid-data-state';
+  message: string;
+} | {
+  kind: 'invalid-aggregation-type';
+  message: string;
+} | {
+  kind: 'invalid-builder-state';
+  message: string;
+  cause?: unknown;
+} | {
+  kind: 'invalid-filter';
+  message: string;
+} | {
+  kind: 'unsupported-capability';
+  capability: string;
+  context: string;
+  message: string;
+} | {
+  kind: 'unresolvable-dataset';
+  dimensions: readonly Dimension[];
+  filterDims: readonly Dimension[];
+  message: string;
+};
+declare const queryErrors: {
+  readonly missingDateRange: () => QueryError;
+  readonly invalidRowLimit: (value: unknown) => QueryError;
+  readonly invalidStartRow: (value: unknown) => QueryError;
+  readonly hourDimensionRequiresHourlyState: () => QueryError;
+  readonly hourlyStateRequiresHourDimension: () => QueryError;
+  readonly byPropertyUnsupportedSearchType: () => QueryError;
+  readonly byPropertyNotAllowedWithPage: () => QueryError;
+  readonly byNewsShowcaseRequiresSearchType: () => QueryError;
+  readonly byNewsShowcaseNotAllowedWithPage: () => QueryError;
+  readonly byNewsShowcaseRequiresShowcaseFilter: () => QueryError;
+  readonly invalidBuilderState: (cause?: unknown) => QueryError;
+  readonly malformedFilterLeaf: () => QueryError;
+  readonly unsupportedCapability: (capability: string, context: string) => QueryError;
+  readonly unresolvableDataset: (dimensions: readonly Dimension[], filterDims?: readonly Dimension[]) => QueryError;
+};
+declare function isQueryError(value: unknown): value is QueryError;
+/**
+ * Thrown when a query needs a planner capability (regex pushdown, comparison
+ * joins, multi-dataset reads) the target engine lacks. Engines catch it to fall
+ * back to the live GSC API. Carries the typed `queryError` value so a caller can
+ * read the modelled failure instead of parsing the message.
+ */
+declare class UnsupportedLogicalCapabilityError extends Error {
+  readonly queryError: Extract<QueryError, {
+    kind: 'unsupported-capability';
+  }>;
+  constructor(capability: string, context: string);
+}
+/**
+ * Thrown when a query's grouped + filtered dimensions span more than one stored
+ * dataset. Replaces the resolver's raw "unknown column" error so hosts can map
+ * it to a 4xx instead of leaking an opaque 500. Carries the typed `queryError`.
+ */
+declare class UnresolvableDatasetError extends Error {
+  readonly queryError: Extract<QueryError, {
+    kind: 'unresolvable-dataset';
+  }>;
+  constructor(dimensions: readonly Dimension[], filterDims?: readonly Dimension[]);
+}
+/**
+ * Re-raises a `QueryError` value as an `Error`, preserving the historical class
+ * identity for the two errors engines match on by name (`UnresolvableDatasetError`,
+ * `UnsupportedLogicalCapabilityError`). Pairs with `unwrapResult` so a
+ * `fooResult(): Result<A, QueryError>` core can back a throwing `foo()`.
+ */
+declare function queryErrorToException(error: QueryError): Error;
+type LogicalDataset = TableName;
+type ComparisonFilter = 'new' | 'lost' | 'improving' | 'declining';
+interface PlannerCapabilities {
+  regex?: boolean;
+  multiDataset?: boolean;
+  comparisonJoin?: boolean;
+  windowTotals?: boolean;
+}
+interface LogicalDimensionFilter {
+  dimension: Dimension;
+  operator: FilterOperator;
+  expression: string;
+  expression2?: string;
+}
+interface LogicalMetricFilter {
+  metric: Metric;
+  operator: MetricOperator;
+  expression: number;
+  expression2?: number;
+}
+interface LogicalFilterLeaf {
+  kind: 'leaf';
+  filter: LogicalDimensionFilter;
+}
+interface LogicalFilterGroup {
+  kind: 'group';
+  groupType: 'and' | 'or';
+  children: LogicalFilterNode[];
+}
+type LogicalFilterNode = LogicalFilterLeaf | LogicalFilterGroup;
+interface LogicalQueryPlan {
+  dataset: LogicalDataset;
+  dimensions: Dimension[];
+  groupByDimensions: Dimension[];
+  hasDate: boolean;
+  metrics: Metric[];
+  dateRange: {
+    startDate: string;
+    endDate: string;
+  };
+  dimensionFilters: LogicalDimensionFilter[];
+  dimensionFilterTree?: LogicalFilterNode;
+  metricFilters: LogicalMetricFilter[];
+  prefilters: LogicalMetricFilter[];
+  specialFilters: {
+    topLevel: boolean;
+  };
+  queryParams: Partial<Record<QueryParamName, string>>;
+  orderBy?: BuilderState['orderBy'];
+  rowLimit?: number;
+  startRow?: number;
+}
+interface LogicalComparisonPlan {
+  current: LogicalQueryPlan;
+  previous: LogicalQueryPlan;
+  comparisonFilter?: ComparisonFilter;
+}
+declare function inferDataset(dimensions: readonly Dimension[], filterDims?: readonly Dimension[]): LogicalDataset;
+/**
+ * True when every grouped + filtered dimension fits inside one stored dataset,
+ * i.e. the query is answerable from stored Parquet/D1 tables without a live
+ * GSC call. `inferDataset` always returns *some* dataset; this predicate is
+ * how callers tell a genuine match from one that will fail at column-resolve
+ * time.
+ */
+declare function isDatasetResolvable(dimensions: readonly Dimension[], filterDims?: readonly Dimension[]): boolean;
+/**
+ * `BuilderState`-level convenience for {@link isDatasetResolvable}: extracts
+ * the state's dimension filters (the same way `buildLogicalPlan` does) and
+ * checks them against the grouped dimensions. Lets routing code (e.g. the
+ * composite source) detect a cross-dimension query without rebuilding a plan.
+ */
+declare function isStateResolvable(state: BuilderState): boolean;
+/**
+ * Errors-as-values core: builds the logical plan or returns a typed `QueryError`
+ * for every modelled failure (missing date range, a regex filter on an engine
+ * without regex pushdown, a cross-dimension query with no stored home).
+ * `buildLogicalPlan` is the throwing wrapper for call sites that prefer exceptions.
+ */
+declare function buildLogicalPlanResult(state: BuilderState, capabilities?: PlannerCapabilities): Result<LogicalQueryPlan, QueryError>;
+declare function buildLogicalPlan(state: BuilderState, capabilities?: PlannerCapabilities): LogicalQueryPlan;
+/**
+ * Errors-as-values core for the comparison plan: returns a typed `QueryError`
+ * when the engine lacks the comparison-join or multi-dataset capability the
+ * paired queries need, or when either side fails to plan.
+ */
+declare function buildLogicalComparisonPlanResult(current: BuilderState, previous: BuilderState, capabilities?: PlannerCapabilities, comparisonFilter?: ComparisonFilter): Result<LogicalComparisonPlan, QueryError>;
+declare function buildLogicalComparisonPlan(current: BuilderState, previous: BuilderState, capabilities?: PlannerCapabilities, comparisonFilter?: ComparisonFilter): LogicalComparisonPlan;
+export { ComparisonFilter, LogicalComparisonPlan, LogicalDataset, LogicalDimensionFilter, LogicalFilterGroup, LogicalFilterLeaf, LogicalFilterNode, LogicalMetricFilter, LogicalQueryPlan, PlannerCapabilities, QueryError, QueryErrorKind, UnresolvableDatasetError, UnsupportedLogicalCapabilityError, buildLogicalComparisonPlan, buildLogicalComparisonPlanResult, buildLogicalPlan, buildLogicalPlanResult, inferDataset, isDatasetResolvable, isQueryError, isStateResolvable, queryErrorToException, queryErrors };