@gscdump/sdk 7.12.0 → 7.13.0

package/README.md

@@ -55,6 +55,19 @@ const report = await gscdump.queryAnalyticsReport({
 })
 ```
 
+`@gscdump/sdk/indexing` holds the pure indexing dashboard maths: Portfolio
+stacks and issue counts, the Indexing window, the Indexing overview, the Trust
+gate, Index cohorts, the Regression lead, and the Portfolio Markdown export.
+Every function takes data and returns data. A function that needs today takes `now`.
+
+```ts
+import { indexingWindow, portfolioIssueCounts, readSiteIndexing } from '@gscdump/sdk/indexing'
+
+const window = indexingWindow('28d', true, new Date())
+const sites = reads.map(read => ({ id: read.siteId, label: read.name, trend: readSiteIndexing(read.response).trend }))
+const issues = portfolioIssueCounts(sites, window)
+```
+
 ## Browser integration
 
 Keep `user_key` and `partner_key` credentials on your server.

package/dist/indexing/cohorts.d.mts

@@ -0,0 +1,201 @@
+import { GscdumpIndexingUrl } from "@gscdump/contracts";
+export interface IndexCohortPage {
+  /** Site-relative path, such as `/docs/og-image/v5/guides/emojis`. */
+  path: string;
+  /** Google reports this URL as indexed. */
+  indexed: boolean;
+  inSitemap: boolean;
+}
+export type IndexCohortDimension = 'section' | 'lifecycle';
+export type IndexCohortLifecycle = 'versioned' | 'evergreen';
+/** Exact Index cohort membership, for a URL list drill-through. */
+export type IndexCohortFilter = {
+  dimension: 'section';
+  key: string;
+} | {
+  dimension: 'lifecycle';
+  key: IndexCohortLifecycle;
+};
+export interface IndexCohortCell {
+  dimension: IndexCohortDimension;
+  /** A path prefix for `section`, a lifecycle tag for `lifecycle`. */
+  key: string;
+  label: string;
+  /** The section prefix for presentation, or null for a lifecycle cohort. */
+  pathPrefix: string | null;
+  total: number;
+  notIndexed: number;
+  /** Not-indexed share within the cohort. */
+  rate: number;
+  /** Not-indexed share across every analysed page outside this cohort. */
+  complementRate: number;
+  complementTotal: number;
+  ciLower: number;
+  ciUpper: number;
+  /**
+   * How many of this cohort's not-indexed pages a higher-ranked cohort already
+   * counts. Cohorts nest, so without this a reader adds 13 and 21 and believes
+   * 34 pages are affected when 21 are.
+   */
+  overlapNotIndexed: number;
+}
+export interface IndexCohortCoverage {
+  /** Pages in the analysed set: the denominator. */
+  analysed: number;
+  /** Inspected URLs dropped as `#fragment` anchors. They can never be indexed. */
+  excludedFragments: number;
+  /** Inspected URLs a Crawl never reached. Always 0 when built from inspected URLs. */
+  notCrawled: number;
+}
+export interface IndexCohortBaseline {
+  total: number;
+  notIndexed: number;
+  rate: number;
+}
+export type IndexCohortDiagnosis = {
+  _tag: 'no-evidence';
+  reason: 'no-inspection-join' | 'no-completed-crawl';
+} | {
+  _tag: 'no-evidence';
+  reason: 'sampled-index-state';
+  enumerated: number;
+  reported: number;
+} | {
+  _tag: 'uniform';
+  baseline: IndexCohortBaseline;
+  coverage: IndexCohortCoverage;
+  /** The corrected critical value applied. */
+  z: number;
+  tested: number;
+} | {
+  _tag: 'outliers';
+  baseline: IndexCohortBaseline;
+  coverage: IndexCohortCoverage;
+  z: number;
+  tested: number;
+  /** Worst first. Only cohorts Google treats worse than their complement. */
+  cells: IndexCohortCell[];
+};
+interface IndexCohortsEnvelope {
+  /** The Crawl settings behind the cohorts. Null when built from inspected URLs. */
+  crawlSettingsId: number | null;
+  /** The caller's snapshot timestamp. */
+  asOf: string | null;
+}
+/** The Index cohorts wire shape nuxtseo.com and request-indexing publish. */
+export type IndexCohortsResponse = (IndexCohortsEnvelope & {
+  _tag: 'no-evidence';
+  reason: 'no-inspection-join' | 'no-completed-crawl' | 'sampled-index-state';
+  /** Set only for `sampled-index-state`: the enumerated and reported not-indexed counts. */
+  sample: {
+    enumerated: number;
+    reported: number;
+  } | null;
+}) | (IndexCohortsEnvelope & Extract<IndexCohortDiagnosis, {
+  _tag: 'uniform';
+}>) | (IndexCohortsEnvelope & Extract<IndexCohortDiagnosis, {
+  _tag: 'outliers';
+}>);
+/**
+ * The enumerated not-indexed list must cover at least this share of the count
+ * gscdump reports. Below it the list is a sample, and every rate built on it
+ * would treat the unlisted URLs as absent (nuxtseo.com #1306).
+ */
+export declare const COHORT_MIN_ENUMERATED_SHARE = 0.9;
+/** From this share of pages not indexed, no family can be an outlier: the failure covers the whole Site. */
+export declare const SITE_WIDE_NOT_INDEXED_RATE = 0.5;
+/** Below this many pages, a cohort's interval is too wide for any claim. */
+export declare const COHORT_MIN_CELL_SIZE = 12;
+/** Message for the `sampled-index-state` reason, shared by every surface. */
+export declare function sampledIndexStateMessage(enumerated: number, reported: number): string;
+/**
+ * Wilson score interval for a binomial proportion. The normal approximation
+ * gives bounds outside 0 to 1 on small, skewed cohorts and zero width at p=0.
+ */
+export declare function wilsonInterval(successes: number, total: number, z: number): {
+  lower: number;
+  upper: number;
+};
+/** True when the path belongs to the Index cohort the filter names. */
+export declare function belongsToIndexCohort(path: string, filter: IndexCohortFilter): boolean;
+export type IndexCohortFilterParse = {
+  _tag: 'ok';
+  filter: IndexCohortFilter;
+} | {
+  _tag: 'invalid';
+  value: unknown;
+};
+/** Parse a `dimension:key` query value, such as `section:/docs` or `lifecycle:versioned`. */
+export declare function parseIndexCohortFilter(value: unknown): IndexCohortFilterParse;
+/** The `dimension:key` query value of a cell, the inverse of `parseIndexCohortFilter`. */
+export declare function indexCohortFilterValue(cell: Pick<IndexCohortCell, 'dimension' | 'key'>): string;
+export interface BuildIndexCohortInput {
+  pages: ReadonlyArray<IndexCohortPage>;
+  excludedFragments: number;
+  notCrawled: number;
+}
+/** Partition the analysed pages and keep only the cohorts Google treats worse than the rest of the Site. */
+export declare function buildIndexCohortDiagnosis(input: BuildIndexCohortInput): IndexCohortDiagnosis;
+/**
+ * One section's raw not-indexed tally, with no significance test. `established`
+ * rides on every row, so a consumer cannot show the ranking without also
+ * showing which rows passed the test.
+ */
+export interface IndexCohortSectionRank {
+  /** Path prefix at depth 1 or 2: the same keys the tested cohorts use. */
+  key: string;
+  total: number;
+  notIndexed: number;
+  /** Not-indexed share within the section. A point estimate, not a finding. */
+  rate: number;
+  /** This section also passed the corrected significance test in the same run. */
+  established: boolean;
+  /** Too few pages for any test (`COHORT_MIN_CELL_SIZE`). */
+  belowMinimumSample: boolean;
+}
+/** Every section ranked by not-indexed rate, worst first. Sections that span every page are dropped. */
+export declare function rankIndexCohortSections(pages: ReadonlyArray<IndexCohortPage>, established?: ReadonlyArray<IndexCohortCell>): IndexCohortSectionRank[];
+/** The lead derived from an Index cohort diagnosis. `none` says why there is no lead. */
+export type IndexCohortLead = {
+  _tag: 'none';
+  reason: 'no-evidence' | 'no-outlier' | 'site-wide';
+} | {
+  _tag: 'lead';
+  /** The noun and the number. */
+  title: string;
+  /** The comparison that makes the title a diagnosis. */
+  detail: string;
+  actionLabel: string;
+  cell: IndexCohortCell;
+};
+/**
+ * Promote the worst cohort to the primary diagnosis. A located claim is more
+ * actionable than a reason-led one. When no cohort separates, or most of the
+ * Site is not indexed, there is no lead.
+ */
+export declare function selectIndexCohortLead(diagnosis: {
+  _tag: 'no-evidence';
+} | Exclude<IndexCohortDiagnosis, {
+  _tag: 'no-evidence';
+}>): IndexCohortLead;
+/** One inspected URL row, as the hosted URLs read returns it. */
+export type IndexCohortUrlRow = Pick<GscdumpIndexingUrl, 'url' | 'verdict'> & Partial<Pick<GscdumpIndexingUrl, 'sitemaps'>>;
+export interface IndexCohortSource<Row extends IndexCohortUrlRow = IndexCohortUrlRow> {
+  urls: ReadonlyArray<Row>;
+  /** The Site URL the rows belong to, used to resolve relative rows. */
+  siteUrl?: string | null;
+  /**
+   * The not-indexed count gscdump reports for the Site: the diagnostics
+   * `not_indexed` issue. 0 when the issue is absent.
+   */
+  reportedNotIndexed: number;
+}
+/**
+ * Build the Index cohorts response from a complete set of inspected URLs. No
+ * Crawl joins it, so `notCrawled` is 0 and `crawlSettingsId` is null. Below
+ * `COHORT_MIN_ENUMERATED_SHARE` of the reported not-indexed count, the builder
+ * refuses with `sampled-index-state` rather than rank sections.
+ */
+export declare function buildIndexCohortsFromIndexingUrls(source: IndexCohortSource, asOf?: string | null): IndexCohortsResponse;
+/** The not-indexed rows that belong to one Index cohort. */
+export declare function selectIndexCohortUrls<Row extends IndexCohortUrlRow>(urls: ReadonlyArray<Row>, filter: IndexCohortFilter, siteUrl?: string | null): Row[];

package/dist/indexing/cohorts.mjs

@@ -0,0 +1,343 @@
+const COHORT_MIN_ENUMERATED_SHARE = .9;
+const SITE_WIDE_NOT_INDEXED_RATE = .5;
+const COHORT_MIN_CELL_SIZE = 12;
+const COHORT_FAMILY_ALPHA = .05;
+const VERSION_SEGMENT = /\/v\d+\/.+/;
+const SECTION_KEY = /^\/(?:[^/?#]+(?:\/[^/?#]+)?)?$/;
+function sampledIndexStateMessage(enumerated, reported) {
+	return `Google index state is a sample: ${enumerated.toLocaleString("en-US")} of ${reported.toLocaleString("en-US")} not-indexed URLs. Family rates would be invented. Read the indexing summary instead.`;
+}
+function wilsonInterval(successes, total, z) {
+	if (total <= 0) return {
+		lower: 0,
+		upper: 1
+	};
+	const p = successes / total;
+	const z2 = z * z;
+	const denominator = 1 + z2 / total;
+	const centre = p + z2 / (2 * total);
+	const spread = z * Math.sqrt(p * (1 - p) / total + z2 / (4 * total * total));
+	return {
+		lower: Math.max(0, (centre - spread) / denominator),
+		upper: Math.min(1, (centre + spread) / denominator)
+	};
+}
+function normalQuantile(p) {
+	if (p <= 0) return Number.NEGATIVE_INFINITY;
+	if (p >= 1) return Number.POSITIVE_INFINITY;
+	const a = [
+		-39.69683028665376,
+		220.9460984245205,
+		-275.9285104469687,
+		138.357751867269,
+		-30.66479806614716,
+		2.506628277459239
+	];
+	const b = [
+		-54.47609879822406,
+		161.5858368580409,
+		-155.6989798598866,
+		66.80131188771972,
+		-13.28068155288572
+	];
+	const c = [
+		-.007784894002430293,
+		-.3223964580411365,
+		-2.400758277161838,
+		-2.549732539343734,
+		4.374664141464968,
+		2.938163982698783
+	];
+	const d = [
+		.007784695709041462,
+		.3224671290700398,
+		2.445134137142996,
+		3.754408661907416
+	];
+	const pLow = .02425;
+	const pHigh = .97575;
+	if (p < pLow) {
+		const q = Math.sqrt(-2 * Math.log(p));
+		return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+	}
+	if (p > pHigh) {
+		const q = Math.sqrt(-2 * Math.log(1 - p));
+		return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+	}
+	const q = p - .5;
+	const r = q * q;
+	return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
+}
+function criticalValue(tested) {
+	const alpha = COHORT_FAMILY_ALPHA / Math.max(1, tested);
+	return Math.abs(normalQuantile(alpha / 2));
+}
+function sectionKeys(path) {
+	const segments = path.split("/").filter(Boolean);
+	if (segments.length === 0) return ["/"];
+	const depth1 = `/${segments[0]}`;
+	if (segments.length === 1) return [depth1];
+	return [depth1, `/${segments.slice(0, 2).join("/")}`];
+}
+function lifecycleKey(path) {
+	return VERSION_SEGMENT.test(path) ? "versioned" : "evergreen";
+}
+function belongsToIndexCohort(path, filter) {
+	return filter.dimension === "section" ? sectionKeys(path).includes(filter.key) : lifecycleKey(path) === filter.key;
+}
+function parseIndexCohortFilter(value) {
+	if (typeof value !== "string") return {
+		_tag: "invalid",
+		value
+	};
+	const colon = value.indexOf(":");
+	const dimension = value.slice(0, colon);
+	const key = value.slice(colon + 1);
+	if (colon > 0 && dimension === "section" && SECTION_KEY.test(key)) return {
+		_tag: "ok",
+		filter: {
+			dimension,
+			key
+		}
+	};
+	if (colon > 0 && dimension === "lifecycle" && (key === "versioned" || key === "evergreen")) return {
+		_tag: "ok",
+		filter: {
+			dimension,
+			key
+		}
+	};
+	return {
+		_tag: "invalid",
+		value
+	};
+}
+function indexCohortFilterValue(cell) {
+	return `${cell.dimension}:${cell.key}`;
+}
+const LIFECYCLE_LABEL = {
+	versioned: "Version-pinned paths",
+	evergreen: "Unversioned paths"
+};
+function tally(pages, keysOf) {
+	const out = /* @__PURE__ */ new Map();
+	for (const page of pages) for (const key of keysOf(page)) {
+		const current = out.get(key) ?? {
+			total: 0,
+			notIndexed: 0,
+			notIndexedPaths: []
+		};
+		current.total += 1;
+		if (!page.indexed) {
+			current.notIndexed += 1;
+			current.notIndexedPaths.push(page.path);
+		}
+		out.set(key, current);
+	}
+	return out;
+}
+function buildIndexCohortDiagnosis(input) {
+	const { pages, excludedFragments, notCrawled } = input;
+	if (pages.length === 0) return {
+		_tag: "no-evidence",
+		reason: "no-inspection-join"
+	};
+	const notIndexed = pages.reduce((sum, page) => sum + (page.indexed ? 0 : 1), 0);
+	const baseline = {
+		total: pages.length,
+		notIndexed,
+		rate: notIndexed / pages.length
+	};
+	const coverage = {
+		analysed: pages.length,
+		excludedFragments,
+		notCrawled
+	};
+	const candidates = [{
+		dimension: "section",
+		tallies: tally(pages, (page) => sectionKeys(page.path)),
+		label: (key) => key
+	}, {
+		dimension: "lifecycle",
+		tallies: tally(pages, (page) => [lifecycleKey(page.path)]),
+		label: (key) => LIFECYCLE_LABEL[key] ?? key
+	}].flatMap(({ dimension, tallies, label }) => [...tallies.entries()].filter(([, cell]) => cell.total >= 12 && cell.total < pages.length).map(([key, cell]) => ({
+		dimension,
+		key,
+		label: label(key),
+		cell
+	})));
+	const tested = candidates.length;
+	const z = criticalValue(tested);
+	if (tested === 0) return {
+		_tag: "uniform",
+		baseline,
+		coverage,
+		z,
+		tested
+	};
+	const survivors = [];
+	for (const { dimension, key, label, cell } of candidates) {
+		const complementTotal = baseline.total - cell.total;
+		const complementNotIndexed = baseline.notIndexed - cell.notIndexed;
+		const complementRate = complementTotal > 0 ? complementNotIndexed / complementTotal : 0;
+		const { lower, upper } = wilsonInterval(cell.notIndexed, cell.total, z);
+		if (complementTotal < 12) continue;
+		if (lower <= complementRate) continue;
+		survivors.push({
+			notIndexedPaths: cell.notIndexedPaths,
+			cell: {
+				dimension,
+				key,
+				label,
+				pathPrefix: dimension === "section" ? key : null,
+				total: cell.total,
+				notIndexed: cell.notIndexed,
+				rate: cell.notIndexed / cell.total,
+				complementRate,
+				complementTotal,
+				ciLower: lower,
+				ciUpper: upper,
+				overlapNotIndexed: 0
+			}
+		});
+	}
+	if (survivors.length === 0) return {
+		_tag: "uniform",
+		baseline,
+		coverage,
+		z,
+		tested
+	};
+	survivors.sort((left, right) => right.cell.rate - right.cell.complementRate - (left.cell.rate - left.cell.complementRate) || right.cell.notIndexed - left.cell.notIndexed);
+	const claimed = /* @__PURE__ */ new Set();
+	return {
+		_tag: "outliers",
+		baseline,
+		coverage,
+		z,
+		tested,
+		cells: survivors.map(({ cell, notIndexedPaths }) => {
+			const overlapNotIndexed = notIndexedPaths.reduce((sum, path) => sum + (claimed.has(path) ? 1 : 0), 0);
+			for (const path of notIndexedPaths) claimed.add(path);
+			return {
+				...cell,
+				overlapNotIndexed
+			};
+		})
+	};
+}
+function rankIndexCohortSections(pages, established = []) {
+	const establishedKeys = new Set(established.filter((cell) => cell.dimension === "section").map((cell) => cell.key));
+	return [...tally(pages, (page) => sectionKeys(page.path)).entries()].filter(([, cell]) => cell.total < pages.length).map(([key, cell]) => ({
+		key,
+		total: cell.total,
+		notIndexed: cell.notIndexed,
+		rate: cell.notIndexed / cell.total,
+		established: establishedKeys.has(key),
+		belowMinimumSample: cell.total < 12
+	})).sort((left, right) => right.rate - left.rate || right.notIndexed - left.notIndexed || left.key.localeCompare(right.key));
+}
+function cohortSubject(cell) {
+	return cell.dimension === "section" ? `pages under ${cell.key}` : "version-pinned pages";
+}
+function asPercent(value) {
+	return `${Math.round(value * 100)}%`;
+}
+function selectIndexCohortLead(diagnosis) {
+	if (diagnosis._tag === "no-evidence") return {
+		_tag: "none",
+		reason: "no-evidence"
+	};
+	if (diagnosis._tag === "uniform") return {
+		_tag: "none",
+		reason: "no-outlier"
+	};
+	if (diagnosis.baseline.rate >= .5) return {
+		_tag: "none",
+		reason: "site-wide"
+	};
+	const cell = diagnosis.cells[0];
+	if (!cell) return {
+		_tag: "none",
+		reason: "no-outlier"
+	};
+	return {
+		_tag: "lead",
+		title: `${cell.notIndexed.toLocaleString("en-US")} of ${cell.total.toLocaleString("en-US")} ${cohortSubject(cell)} are missing from Google's index.`,
+		detail: `That is ${asPercent(cell.rate)} not indexed, against ${asPercent(cell.complementRate)} across the rest of the site.`,
+		actionLabel: `Review ${cell.notIndexed.toLocaleString("en-US")} ${cell.notIndexed === 1 ? "URL" : "URLs"}`,
+		cell
+	};
+}
+function siteOrigin(siteUrl) {
+	const site = siteUrl?.trim();
+	if (!site) return void 0;
+	if (site.startsWith("sc-domain:")) return `https://${site.slice(10)}`;
+	return site.includes("://") ? site : `https://${site}`;
+}
+function parseRow(row, siteUrl) {
+	const candidate = row.url?.trim();
+	if (!candidate) return { _tag: "unparsable" };
+	const base = siteOrigin(siteUrl);
+	const parsed = URL.parse(candidate) ?? (base ? URL.parse(candidate, base) : null);
+	if (!parsed || parsed.protocol !== "https:" && parsed.protocol !== "http:") return { _tag: "unparsable" };
+	if (parsed.hash) return { _tag: "fragment" };
+	return {
+		_tag: "page",
+		href: parsed.href,
+		page: {
+			path: parsed.pathname,
+			indexed: row.verdict === "PASS",
+			inSitemap: Array.isArray(row.sitemaps) ? row.sitemaps.length > 0 : false
+		}
+	};
+}
+function buildIndexCohortsFromIndexingUrls(source, asOf = null) {
+	const pages = [];
+	const notIndexedUrls = /* @__PURE__ */ new Set();
+	let excludedFragments = 0;
+	for (const row of source.urls) {
+		const parsed = parseRow(row, source.siteUrl);
+		if (parsed._tag === "page") {
+			pages.push(parsed.page);
+			if (!parsed.page.indexed) notIndexedUrls.add(parsed.href);
+		} else if (parsed._tag === "fragment") excludedFragments += 1;
+	}
+	const enumerated = notIndexedUrls.size;
+	const reported = source.reportedNotIndexed;
+	if (reported > 0 && enumerated / reported < .9) return {
+		_tag: "no-evidence",
+		crawlSettingsId: null,
+		asOf,
+		reason: "sampled-index-state",
+		sample: {
+			enumerated,
+			reported
+		}
+	};
+	const diagnosis = buildIndexCohortDiagnosis({
+		pages,
+		excludedFragments,
+		notCrawled: 0
+	});
+	if (diagnosis._tag === "no-evidence") return {
+		_tag: "no-evidence",
+		crawlSettingsId: null,
+		asOf,
+		reason: diagnosis.reason,
+		sample: null
+	};
+	return {
+		...diagnosis,
+		crawlSettingsId: null,
+		asOf
+	};
+}
+function selectIndexCohortUrls(urls, filter, siteUrl) {
+	return urls.filter((row) => {
+		const parsed = parseRow(row, siteUrl);
+		return parsed._tag === "page" && !parsed.page.indexed && belongsToIndexCohort(parsed.page.path, filter);
+	});
+}
+export { COHORT_MIN_CELL_SIZE, COHORT_MIN_ENUMERATED_SHARE, SITE_WIDE_NOT_INDEXED_RATE, belongsToIndexCohort, buildIndexCohortDiagnosis, buildIndexCohortsFromIndexingUrls, indexCohortFilterValue, parseIndexCohortFilter, rankIndexCohortSections, sampledIndexStateMessage, selectIndexCohortLead, selectIndexCohortUrls, wilsonInterval };