pagesight 0.16.0 → 0.18.0

package/src/api/evidence.ts

@@ -0,0 +1,99 @@
+import { RequestError } from "../shared/http.js";
+
+export interface Evidence {
+  schemaVersion: 1;
+  provider: string;
+  operation: string;
+  name?: string;
+  target: string;
+  startedAt: string;
+  finishedAt: string;
+  status: "ok" | "partial" | "error";
+  pages: Array<{ request: unknown; response: unknown }>;
+  pagination?: { exhausted: boolean; nextOffset: number | null; rowsReturned: number };
+  warnings: string[];
+  credential?: { source: string; type: string | null; clientEmail: string | null };
+  failedRequest?: unknown;
+  error?: { code: string; message: string; httpStatus: number | null; name?: string };
+}
+
+export function createEvidence(provider: string, operation: string, target: string): Evidence {
+  return {
+    schemaVersion: 1,
+    provider,
+    operation,
+    target,
+    startedAt: new Date().toISOString(),
+    finishedAt: "",
+    status: "ok",
+    pages: [],
+    warnings: [],
+  };
+}
+
+export function fail(result: Evidence, error: unknown): void {
+  result.status = result.pages.length ? "partial" : "error";
+  result.error =
+    error instanceof RequestError
+      ? { code: error.code, message: error.message, httpStatus: error.status }
+      : {
+          code: "operation_failed",
+          message: "Operation failed; check credentials, request and provider availability",
+          httpStatus: null,
+          name: error instanceof Error ? error.name : "UnknownError",
+        };
+}
+
+export async function capture(
+  provider: string,
+  operation: string,
+  target: string,
+  request: unknown,
+  run: () => Promise<unknown>,
+  warnings: string[] = [],
+): Promise<Evidence> {
+  const result = createEvidence(provider, operation, target);
+  result.warnings.push(...warnings);
+  try {
+    result.pages.push({ request, response: await run() });
+  } catch (error) {
+    result.failedRequest = request;
+    fail(result, error);
+  }
+  result.finishedAt = new Date().toISOString();
+  return result;
+}
+
+export function aggregate(operation: string, target: string, request: unknown, observations: Evidence[]): Evidence {
+  if (!observations.length) throw new RequestError("Select at least one observation", null, "invalid_input");
+  return {
+    schemaVersion: 1,
+    provider: "pagesight",
+    operation,
+    target,
+    startedAt: observations.map((o) => o.startedAt).sort()[0] ?? new Date().toISOString(),
+    finishedAt: new Date().toISOString(),
+    status: observations.every((o) => o.status === "ok")
+      ? "ok"
+      : observations.every((o) => o.status === "error")
+        ? "error"
+        : "partial",
+    pages: [
+      {
+        request,
+        response: {
+          observations,
+          summary: observations.map((o) => ({
+            provider: o.provider,
+            operation: o.operation,
+            ...(o.name ? { name: o.name } : {}),
+            target: o.target,
+            status: o.status,
+            errorCode: o.error?.code ?? null,
+          })),
+        },
+      },
+    ],
+    warnings: [],
+  };
+}

package/src/api/execute.ts

@@ -0,0 +1,147 @@
+import { importUiFindings } from "./ui-findings.js";
+import { doctor } from "./doctor.js";
+import { ZodError } from "zod";
+import { queryCrux, queryCruxHistory } from "../providers/crux.js";
+import { gaCredentialInfo, gaFetch, normalizeGaProperty } from "../providers/ga.js";
+import { inspectUrlResponse, listSitemapsResponse, listSitesResponse } from "../providers/gsc.js";
+import { RequestError } from "../shared/http.js";
+import { runPagespeed } from "../providers/pagespeed.js";
+import { bingDiagnostics, bingObservation } from "./bing.js";
+import { compareSnapshots } from "./compare-snapshots.js";
+import { capture, type Evidence } from "./evidence.js";
+import { gaReport, gscReport } from "./reports.js";
+import { operationSchema } from "./schema.js";
+import { discover } from "./discover.js";
+import { snapshot } from "./snapshot.js";
+import { observePage } from "../web/page-observation.js";
+
+export type Executor = (input: unknown) => Promise<Evidence>;
+type ParsedOperation = ReturnType<typeof operationSchema.parse>;
+
+export async function execute(input: unknown): Promise<Evidence> {
+  let op: ParsedOperation;
+  try {
+    op = operationSchema.parse(input);
+  } catch (error) {
+    throw new RequestError(
+      error instanceof ZodError
+        ? error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")
+        : "Invalid operation",
+      null,
+      "invalid_input",
+    );
+  }
+  const result = await dispatch(op);
+  if (op.operation.startsWith("ga.")) result.credential = await gaCredentialInfo();
+  if (op.operation.startsWith("bing."))
+    result.credential = { source: "BING_WEBMASTER_API_KEY", type: "api_key", clientEmail: null };
+  const response = result.pages[0]?.response as { nextPageToken?: string } | undefined;
+  if (response?.nextPageToken) {
+    result.status = "partial";
+    result.warnings.push("More metadata pages are available; nextPageToken is preserved in the response.");
+  }
+  return result;
+}
+
+async function dispatch(op: ParsedOperation): Promise<Evidence> {
+  switch (op.operation) {
+    case "evidence.import":
+      return importUiFindings(op.document);
+    case "compare":
+      return compareSnapshots(op.baseline, op.current, op.maxRows);
+    case "discover":
+      return discover(op.url, op.providers, execute);
+    case "bing.crawl-stats":
+    case "bing.crawl-issues":
+      return bingDiagnostics(op.operation === "bing.crawl-stats" ? "crawl-stats" : "crawl-issues", op.site);
+    case "bing.url-info":
+      return bingDiagnostics("url-info", op.site, op.url);
+    case "bing.link-counts":
+      return bingDiagnostics("link-counts", op.site, undefined, op.maxPages);
+    case "bing.url-links":
+      return bingDiagnostics("url-links", op.site, op.url, op.maxPages);
+    case "bing.sites":
+      return bingObservation("sites");
+    case "bing.queries":
+      return bingObservation("queries", op.site);
+    case "bing.pages":
+      return bingObservation("pages", op.site);
+    case "bing.traffic":
+      return bingObservation("traffic", op.site);
+    case "gsc.sites":
+      return capture("gsc", "sites", "accessible-properties", {}, listSitesResponse);
+    case "gsc.sitemaps":
+      return capture("gsc", "sitemaps", op.site, { site: op.site }, () => listSitemapsResponse(op.site), [
+        "contents[].indexed is deprecated and must not be interpreted.",
+      ]);
+    case "gsc.inspect":
+      return capture(
+        "gsc",
+        "inspect",
+        op.site,
+        { inspectionUrl: op.url, siteUrl: op.site },
+        () => inspectUrlResponse(op.url, op.site),
+        ["Inspection describes Google's indexed state, not a live fetch. This is one URL, not site coverage."],
+      );
+    case "gsc.report":
+      return gscReport(op.site, op.request, op.maxPages);
+    case "ga.accounts":
+      return capture(
+        "ga",
+        "accounts",
+        "accessible-properties",
+        { pageSize: 200 },
+        () => gaFetch("accountSummaries?pageSize=200", undefined, true),
+        ["If nextPageToken is present, this discovery response is incomplete."],
+      );
+    case "ga.property":
+      return capture("ga", "property", normalizeGaProperty(op.property), {}, () =>
+        gaFetch(normalizeGaProperty(op.property), undefined, true),
+      );
+    case "ga.key-events":
+      return capture(
+        "ga",
+        "key-events",
+        normalizeGaProperty(op.property),
+        { pageSize: 200 },
+        () => gaFetch(`${normalizeGaProperty(op.property)}/keyEvents?pageSize=200`, undefined, true),
+        [
+          "Configured key events are not necessarily validated product outcomes; inspect event names. Check nextPageToken.",
+        ],
+      );
+    case "ga.report":
+      return gaReport(op.property, op.request, op.maxPages);
+    case "page":
+      return capture("web", "page", op.url, { url: op.url }, () => observePage(op.url));
+    case "speed.psi":
+      return capture(
+        "psi",
+        "run",
+        op.url,
+        { url: op.url, strategy: op.strategy, categories: ["performance", "seo"] },
+        () => runPagespeed(op.url, { strategy: op.strategy, categories: ["performance", "seo"] }),
+        ["Single Lighthouse lab run; not field performance or ranking evidence."],
+      );
+    case "speed.crux":
+    case "speed.history": {
+      const request = {
+        ...(op.origin ? { origin: new URL(op.url).origin } : { url: op.url }),
+        formFactor: op.formFactor,
+      };
+      return capture(
+        "crux",
+        op.operation,
+        op.url,
+        request,
+        () => (op.operation === "speed.crux" ? queryCrux(request) : queryCruxHistory(request)),
+        [
+          "NOT_FOUND means no record for the requested scope, not zero performance. CrUX aggregates 28-day windows; history periods overlap.",
+        ],
+      );
+    }
+    case "doctor":
+      return doctor(op.config, execute);
+    case "snapshot":
+      return snapshot(op, execute);
+  }
+}

package/src/api/index.ts

@@ -0,0 +1,5 @@
+export { execute } from "./execute.js";
+export type { Evidence } from "./evidence.js";
+export { configSchema, type Operation, operationSchema } from "./schema.js";
+export { evidenceSchema, snapshotEvidenceSchema } from "./evidence-schema.js";
+export { RequestError } from "../shared/http.js";

package/src/api/reports.ts

@@ -0,0 +1,90 @@
+import { type GaReport, gaFetch, normalizeGaProperty } from "../providers/ga.js";
+import { querySearchAnalytics, type SearchAnalyticsResponse } from "../providers/gsc.js";
+import { RequestError } from "../shared/http.js";
+import { type Evidence, createEvidence, fail } from "./evidence.js";
+import type { GaRequest, GscRequest } from "./schema.js";
+
+type Query<T> = (request: T) => Promise<SearchAnalyticsResponse | GaReport>;
+
+async function pages<T extends GscRequest | GaRequest>(
+  provider: "gsc" | "ga",
+  target: string,
+  request: T,
+  maxPages: number,
+  query: Query<T>,
+): Promise<Evidence> {
+  const result = createEvidence(provider, "report", target);
+  const isGsc = provider === "gsc";
+  const offsetKey = isGsc ? "startRow" : "offset";
+  let offset = Number(isGsc ? (request as GscRequest).startRow : (request as GaRequest).offset);
+  const limit = Number(isGsc ? (request as GscRequest).rowLimit : (request as GaRequest).limit);
+  result.pagination = { exhausted: false, nextOffset: offset, rowsReturned: 0 };
+  if (isGsc) {
+    result.warnings.push(
+      "GSC dates are America/Los_Angeles; API top-row limits apply even after pagination is exhausted.",
+    );
+    if ((request as GscRequest).dimensions.includes("query"))
+      result.warnings.push("Anonymized queries are omitted. Query-row sums are not property totals.");
+    if ((request as GscRequest).dimensions.includes("page"))
+      result.warnings.push("Page aggregation differs from property aggregation; do not reconcile their row sums.");
+    if ((request as GscRequest).dataState !== "final")
+      result.warnings.push("Fresh data can be incomplete even when metadata is absent for this grouping.");
+  }
+  for (let page = 0; page < maxPages; page++) {
+    const effective = { ...request, [offsetKey]: offset };
+    try {
+      const response = await query(effective);
+      if (response.rows !== undefined && !Array.isArray(response.rows))
+        throw new RequestError("Invalid report rows", null, "invalid_response");
+      const count = response.rows?.length ?? 0;
+      const rowCount =
+        (response as GaReport).rowCount ??
+        (count === 0 && Array.isArray((response as GaReport).metricHeaders) ? 0 : undefined);
+      if (!isGsc && (!Number.isSafeInteger(rowCount) || Number(rowCount) < 0))
+        throw new RequestError("GA response missing a valid rowCount", null, "invalid_response");
+      result.pages.push({ request: effective, response });
+      result.pagination.rowsReturned += count;
+      offset += count;
+      const exhausted = isGsc ? count < limit : offset >= Number(rowCount);
+      result.pagination.exhausted = exhausted;
+      result.pagination.nextOffset = exhausted ? null : offset;
+      if (!isGsc) {
+        const metadata = (response as GaReport).metadata;
+        if (metadata?.subjectToThresholding) result.warnings.push("GA report is subject to thresholding.");
+        if (metadata?.dataLossFromOtherRow)
+          result.warnings.push("GA high-cardinality rows were combined into (other).");
+        if (metadata?.samplingMetadatas?.length) result.warnings.push("GA report is sampled; see response metadata.");
+      }
+      if (exhausted) break;
+      if (!count) throw new RequestError("Provider pagination made no progress", null, "invalid_response");
+    } catch (error) {
+      result.failedRequest = effective;
+      fail(result, error);
+      break;
+    }
+  }
+  if (!result.pagination.exhausted && !result.error) result.status = "partial";
+  if (!result.pagination.exhausted)
+    result.warnings.push("Pagination not exhausted. Missing rows are unknown, not zero.");
+  result.warnings = [...new Set(result.warnings)];
+  result.finishedAt = new Date().toISOString();
+  return result;
+}
+
+export function gscReport(
+  site: string,
+  request: GscRequest,
+  maxPages: number,
+  query = (r: GscRequest) => querySearchAnalytics(site, r),
+): Promise<Evidence> {
+  return pages("gsc", site, request, maxPages, query);
+}
+
+export function gaReport(
+  property: string,
+  request: GaRequest,
+  maxPages: number,
+  query = (r: GaRequest) => gaFetch<GaReport>(`${normalizeGaProperty(property)}:runReport`, r),
+): Promise<Evidence> {
+  return pages("ga", normalizeGaProperty(property), request, maxPages, query);
+}

package/src/api/schema.ts

@@ -0,0 +1,207 @@
+import { uiFindingsSchema } from "./ui-findings.js";
+import { z } from "zod";
+import { snapshotEvidenceSchema } from "./evidence-schema.js";
+import { dateSchema, pacificDate } from "../shared/dates.js";
+
+export { dateSchema } from "../shared/dates.js";
+const offset = z.coerce.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
+export const maxPagesSchema = z.number().int().min(1).max(20).default(1);
+export const gscRequestSchema = z
+  .object({
+    startDate: dateSchema,
+    endDate: dateSchema,
+    dimensions: z
+      .array(z.enum(["date", "hour", "query", "page", "country", "device", "searchAppearance"]))
+      .max(7)
+      .default([]),
+    type: z.enum(["web", "image", "video", "news", "discover", "googleNews"]).default("web"),
+    dataState: z.enum(["final", "all", "hourly_all"]).default("final"),
+    rowLimit: z.number().int().min(1).max(25000).default(25000),
+    startRow: offset.default(0),
+    aggregationType: z.enum(["auto", "byPage", "byProperty", "byNewsShowcasePanel"]).optional(),
+    dimensionFilterGroups: z
+      .array(
+        z
+          .object({
+            groupType: z.literal("and").optional(),
+            filters: z.array(
+              z.object({ dimension: z.string(), operator: z.string(), expression: z.string() }).strict(),
+            ),
+          })
+          .strict(),
+      )
+      .optional(),
+  })
+  .strict()
+  .refine((v) => v.startDate <= v.endDate, "startDate must be on or before endDate")
+  .refine((v) => v.endDate <= pacificDate(), "GSC endDate cannot be in the future");
+export type GscRequest = z.infer<typeof gscRequestSchema>;
+
+export const gaRequestSchema = z
+  .object({
+    dateRanges: z
+      .array(
+        z
+          .object({ startDate: dateSchema, endDate: dateSchema, name: z.string().optional() })
+          .strict()
+          .refine((v) => v.startDate <= v.endDate, "Invalid date range"),
+      )
+      .min(1)
+      .max(4),
+    dimensions: z.array(z.object({ name: z.string() }).passthrough()).default([]),
+    metrics: z.array(z.object({ name: z.string() }).passthrough()).min(1),
+    dimensionFilter: z.record(z.unknown()).optional(),
+    metricFilter: z.record(z.unknown()).optional(),
+    orderBys: z.array(z.record(z.unknown())).optional(),
+    metricAggregations: z.array(z.string()).optional(),
+    currencyCode: z.string().optional(),
+    keepEmptyRows: z.boolean().optional(),
+    limit: z.coerce.number().int().min(1).max(250000).default(10000),
+    offset: offset.default(0),
+    returnPropertyQuota: z.boolean().default(true),
+  })
+  .strict();
+export type GaRequest = z.infer<typeof gaRequestSchema>;
+
+const httpUrl = z
+  .string()
+  .url()
+  .refine(
+    (v) => ["http:", "https:"].includes(new URL(v).protocol) && !new URL(v).username && !new URL(v).password,
+    "Use an HTTP(S) URL without credentials",
+  );
+export const configSchema = z
+  .object({
+    site: httpUrl,
+    gscSite: z.string().min(1).optional(),
+    bingSite: httpUrl.optional(),
+    gaProperty: z
+      .string()
+      .regex(/^(properties\/)?\d+$/)
+      .optional(),
+    productionHostname: z.string().min(1).optional(),
+    sitemap: httpUrl.optional(),
+    pages: z.array(httpUrl).max(10).optional(),
+    context: z
+      .object({
+        objective: z.string().default("Define a product objective before evaluating outcomes."),
+        successEvents: z.array(z.string()).default([]),
+        excludedKeyEvents: z.array(z.string()).default([]),
+        locale: z.string().default("unspecified"),
+        country: z.string().default("unspecified"),
+        routes: z
+          .array(
+            z
+              .object({ pattern: z.string(), purpose: z.string(), indexing: z.enum(["index", "noindex", "verify"]) })
+              .strict(),
+          )
+          .default([]),
+        measurementCaveats: z.array(z.string()).default([]),
+      })
+      .strict()
+      .default({}),
+  })
+  .strict()
+  .transform((c) => ({
+    ...c,
+    productionHostname: c.productionHostname ?? new URL(c.site).hostname,
+    pages: [...new Set(c.pages ?? [c.site])],
+  }))
+  .refine((c) => new URL(c.site).hostname === c.productionHostname, "productionHostname must match site")
+  .refine(
+    (c) => Boolean(c.pages.length || c.sitemap || c.gscSite || c.gaProperty || c.bingSite),
+    "Select at least one page or provider",
+  );
+export type SiteConfig = z.infer<typeof configSchema>;
+const operationVariants = z.discriminatedUnion("operation", [
+  z.object({ operation: z.literal("evidence.import"), document: uiFindingsSchema }).strict(),
+  z
+    .object({
+      operation: z.literal("compare"),
+      baseline: snapshotEvidenceSchema,
+      current: snapshotEvidenceSchema,
+      maxRows: z.number().int().min(1).max(1000).default(100),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("discover"),
+      url: httpUrl,
+      providers: z
+        .array(z.enum(["gsc", "ga", "bing"]))
+        .min(1)
+        .max(3)
+        .default(["gsc", "ga"]),
+    })
+    .strict(),
+  z.object({ operation: z.literal("bing.crawl-stats"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.crawl-issues"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.url-info"), site: httpUrl, url: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.link-counts"), site: httpUrl, maxPages: maxPagesSchema }).strict(),
+  z.object({ operation: z.literal("bing.url-links"), site: httpUrl, url: httpUrl, maxPages: maxPagesSchema }).strict(),
+  z.object({ operation: z.literal("bing.sites") }).strict(),
+  z.object({ operation: z.literal("bing.queries"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.pages"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.traffic"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("gsc.sites") }).strict(),
+  z.object({ operation: z.literal("gsc.sitemaps"), site: z.string().min(1) }).strict(),
+  z.object({ operation: z.literal("gsc.inspect"), site: z.string().min(1), url: httpUrl }).strict(),
+  z
+    .object({
+      operation: z.literal("gsc.report"),
+      site: z.string().min(1),
+      request: gscRequestSchema,
+      maxPages: maxPagesSchema,
+    })
+    .strict(),
+  z.object({ operation: z.literal("ga.accounts") }).strict(),
+  z.object({ operation: z.literal("ga.property"), property: z.string() }).strict(),
+  z.object({ operation: z.literal("ga.key-events"), property: z.string() }).strict(),
+  z
+    .object({
+      operation: z.literal("ga.report"),
+      property: z.string(),
+      request: gaRequestSchema,
+      maxPages: maxPagesSchema,
+    })
+    .strict(),
+  z.object({ operation: z.literal("page"), url: httpUrl }).strict(),
+  z
+    .object({
+      operation: z.literal("speed.psi"),
+      url: httpUrl,
+      strategy: z.enum(["mobile", "desktop"]).default("mobile"),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("speed.crux"),
+      url: httpUrl,
+      formFactor: z.enum(["PHONE", "DESKTOP", "TABLET"]).default("PHONE"),
+      origin: z.boolean().default(false),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("speed.history"),
+      url: httpUrl,
+      formFactor: z.enum(["PHONE", "DESKTOP", "TABLET"]).default("PHONE"),
+      origin: z.boolean().default(false),
+    })
+    .strict(),
+  z.object({ operation: z.literal("doctor"), config: configSchema }).strict(),
+  z
+    .object({
+      operation: z.literal("snapshot"),
+      config: configSchema,
+      startDate: dateSchema,
+      endDate: dateSchema,
+      maxPages: maxPagesSchema.default(4),
+    })
+    .strict(),
+]);
+export const operationSchema = operationVariants.refine(
+  (op) => op.operation !== "snapshot" || (op.startDate <= op.endDate && op.endDate <= pacificDate()),
+  "Invalid or future snapshot interval",
+);
+export type Operation = z.input<typeof operationSchema>;