pagesight 0.17.0 → 0.19.0

package/src/api/schema.ts

@@ -0,0 +1,271 @@
+import { uiFindingsSchema } from "./ui-findings.js";
+import { z } from "zod";
+import { snapshotEvidenceSchema } from "./evidence-schema.js";
+import { dateSchema, pacificDate } from "../shared/dates.js";
+
+export { dateSchema } from "../shared/dates.js";
+const offset = z.coerce.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
+export const maxPagesSchema = z.number().int().min(1).max(20).default(1);
+export const gscRequestSchema = z
+  .object({
+    startDate: dateSchema,
+    endDate: dateSchema,
+    dimensions: z
+      .array(z.enum(["date", "hour", "query", "page", "country", "device", "searchAppearance"]))
+      .max(7)
+      .default([]),
+    type: z.enum(["web", "image", "video", "news", "discover", "googleNews"]).default("web"),
+    dataState: z.enum(["final", "all", "hourly_all"]).default("final"),
+    rowLimit: z.number().int().min(1).max(25000).default(25000),
+    startRow: offset.default(0),
+    aggregationType: z.enum(["auto", "byPage", "byProperty", "byNewsShowcasePanel"]).optional(),
+    dimensionFilterGroups: z
+      .array(
+        z
+          .object({
+            groupType: z.literal("and").optional(),
+            filters: z.array(
+              z.object({ dimension: z.string(), operator: z.string(), expression: z.string() }).strict(),
+            ),
+          })
+          .strict(),
+      )
+      .optional(),
+  })
+  .strict()
+  .refine((v) => v.startDate <= v.endDate, "startDate must be on or before endDate")
+  .refine((v) => v.endDate <= pacificDate(), "GSC endDate cannot be in the future");
+export type GscRequest = z.infer<typeof gscRequestSchema>;
+
+export const gaRequestSchema = z
+  .object({
+    dateRanges: z
+      .array(
+        z
+          .object({ startDate: dateSchema, endDate: dateSchema, name: z.string().optional() })
+          .strict()
+          .refine((v) => v.startDate <= v.endDate, "Invalid date range"),
+      )
+      .min(1)
+      .max(4),
+    dimensions: z.array(z.object({ name: z.string() }).passthrough()).default([]),
+    metrics: z.array(z.object({ name: z.string() }).passthrough()).min(1),
+    dimensionFilter: z.record(z.unknown()).optional(),
+    metricFilter: z.record(z.unknown()).optional(),
+    orderBys: z.array(z.record(z.unknown())).optional(),
+    metricAggregations: z.array(z.string()).optional(),
+    currencyCode: z.string().optional(),
+    keepEmptyRows: z.boolean().optional(),
+    limit: z.coerce.number().int().min(1).max(250000).default(10000),
+    offset: offset.default(0),
+    returnPropertyQuota: z.boolean().default(true),
+  })
+  .strict();
+export type GaRequest = z.infer<typeof gaRequestSchema>;
+
+export const gaRealtimeRequestSchema = z
+  .object({
+    dimensions: z
+      .array(z.object({ name: z.string().min(1) }).strict())
+      .max(9)
+      .default([]),
+    metrics: z
+      .array(z.object({ name: z.string().min(1) }).strict())
+      .min(1)
+      .max(10),
+    dimensionFilter: z.record(z.unknown()).optional(),
+    metricFilter: z.record(z.unknown()).optional(),
+    orderBys: z.array(z.record(z.unknown())).optional(),
+    limit: z.coerce.number().int().min(1).max(250000).default(10000),
+    returnPropertyQuota: z.boolean().default(true),
+    minuteRanges: z
+      .array(
+        z
+          .object({
+            name: z
+              .string()
+              .min(1)
+              .refine((n) => !n.startsWith("date_range_") && !n.startsWith("RESERVED_"))
+              .optional(),
+            startMinutesAgo: z.number().int().min(0).max(59).default(29),
+            endMinutesAgo: z.number().int().min(0).max(59).default(0),
+          })
+          .strict()
+          .refine((r) => r.startMinutesAgo >= r.endMinutesAgo, "Start must be at least as many minutes ago as end"),
+      )
+      .min(1)
+      .max(2)
+      .default([{ startMinutesAgo: 29, endMinutesAgo: 0 }]),
+  })
+  .strict();
+export type GaRealtimeRequest = z.infer<typeof gaRealtimeRequestSchema>;
+
+export const httpUrl = z
+  .string()
+  .url()
+  .refine(
+    (v) =>
+      URL.canParse(v) &&
+      ["http:", "https:"].includes(new URL(v).protocol) &&
+      !new URL(v).username &&
+      !new URL(v).password,
+    "Use an HTTP(S) URL without credentials",
+  );
+export const configSchema = z
+  .object({
+    site: httpUrl,
+    gscSite: z.string().min(1).optional(),
+    bingSite: httpUrl.optional(),
+    gaProperty: z
+      .string()
+      .regex(/^(properties\/)?\d+$/)
+      .optional(),
+    productionHostname: z.string().min(1).optional(),
+    sitemap: httpUrl.optional(),
+    pages: z.array(httpUrl).max(10).optional(),
+    context: z
+      .object({
+        objective: z.string().default("Define a product objective before evaluating outcomes."),
+        successEvents: z.array(z.string()).default([]),
+        excludedKeyEvents: z.array(z.string()).default([]),
+        locale: z.string().default("unspecified"),
+        country: z.string().default("unspecified"),
+        routes: z
+          .array(
+            z
+              .object({ pattern: z.string(), purpose: z.string(), indexing: z.enum(["index", "noindex", "verify"]) })
+              .strict(),
+          )
+          .default([]),
+        measurementCaveats: z.array(z.string()).default([]),
+      })
+      .strict()
+      .default({}),
+  })
+  .strict()
+  .transform((c) => ({
+    ...c,
+    productionHostname: c.productionHostname ?? new URL(c.site).hostname,
+    pages: [...new Set(c.pages ?? [c.site])],
+  }))
+  .refine((c) => new URL(c.site).hostname === c.productionHostname, "productionHostname must match site")
+  .refine(
+    (c) => Boolean(c.pages.length || c.sitemap || c.gscSite || c.gaProperty || c.bingSite),
+    "Select at least one page or provider",
+  );
+export type SiteConfig = z.infer<typeof configSchema>;
+const assessedSnapshotSchema = snapshotEvidenceSchema.superRefine((snapshot, ctx) => {
+  const config = configSchema.safeParse(snapshot.pages[0].response.context.config);
+  if (!config.success)
+    for (const issue of config.error.issues)
+      ctx.addIssue({ ...issue, path: ["pages", 0, "response", "context", "config", ...issue.path] });
+});
+const operationVariants = z.discriminatedUnion("operation", [
+  z
+    .object({
+      operation: z.literal("opportunities"),
+      snapshot: assessedSnapshotSchema,
+      minImpressions: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER).default(20),
+      maxClicks: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).default(2),
+      maxRows: z.number().int().min(1).max(100).default(10),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("assess"),
+      snapshot: assessedSnapshotSchema,
+      maxRows: z.number().int().min(1).max(100).default(10),
+    })
+    .strict(),
+  z.object({ operation: z.literal("evidence.import"), document: uiFindingsSchema }).strict(),
+  z
+    .object({
+      operation: z.literal("compare"),
+      baseline: snapshotEvidenceSchema,
+      current: snapshotEvidenceSchema,
+      maxRows: z.number().int().min(1).max(1000).default(100),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("discover"),
+      url: httpUrl,
+      providers: z
+        .array(z.enum(["gsc", "ga", "bing"]))
+        .min(1)
+        .max(3)
+        .default(["gsc", "ga"]),
+    })
+    .strict(),
+  z.object({ operation: z.literal("bing.crawl-stats"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.crawl-issues"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.url-info"), site: httpUrl, url: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.link-counts"), site: httpUrl, maxPages: maxPagesSchema }).strict(),
+  z.object({ operation: z.literal("bing.url-links"), site: httpUrl, url: httpUrl, maxPages: maxPagesSchema }).strict(),
+  z.object({ operation: z.literal("bing.sites") }).strict(),
+  z.object({ operation: z.literal("bing.queries"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.pages"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("bing.traffic"), site: httpUrl }).strict(),
+  z.object({ operation: z.literal("gsc.sites") }).strict(),
+  z.object({ operation: z.literal("gsc.sitemaps"), site: z.string().min(1) }).strict(),
+  z.object({ operation: z.literal("gsc.inspect"), site: z.string().min(1), url: httpUrl }).strict(),
+  z
+    .object({
+      operation: z.literal("gsc.report"),
+      site: z.string().min(1),
+      request: gscRequestSchema,
+      maxPages: maxPagesSchema,
+    })
+    .strict(),
+  z.object({ operation: z.literal("ga.realtime"), property: z.string(), request: gaRealtimeRequestSchema }).strict(),
+  z.object({ operation: z.literal("ga.accounts") }).strict(),
+  z.object({ operation: z.literal("ga.property"), property: z.string() }).strict(),
+  z.object({ operation: z.literal("ga.key-events"), property: z.string() }).strict(),
+  z
+    .object({
+      operation: z.literal("ga.report"),
+      property: z.string(),
+      request: gaRequestSchema,
+      maxPages: maxPagesSchema,
+    })
+    .strict(),
+  z.object({ operation: z.literal("page"), url: httpUrl }).strict(),
+  z
+    .object({
+      operation: z.literal("speed.psi"),
+      url: httpUrl,
+      strategy: z.enum(["mobile", "desktop"]).default("mobile"),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("speed.crux"),
+      url: httpUrl,
+      formFactor: z.enum(["PHONE", "DESKTOP", "TABLET"]).default("PHONE"),
+      origin: z.boolean().default(false),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("speed.history"),
+      url: httpUrl,
+      formFactor: z.enum(["PHONE", "DESKTOP", "TABLET"]).default("PHONE"),
+      origin: z.boolean().default(false),
+    })
+    .strict(),
+  z.object({ operation: z.literal("doctor"), config: configSchema }).strict(),
+  z
+    .object({
+      operation: z.literal("snapshot"),
+      config: configSchema,
+      startDate: dateSchema,
+      endDate: dateSchema,
+      maxPages: maxPagesSchema.default(4),
+    })
+    .strict(),
+]);
+export const operationSchema = operationVariants.refine(
+  (op) => op.operation !== "snapshot" || (op.startDate <= op.endDate && op.endDate <= pacificDate()),
+  "Invalid or future snapshot interval",
+);
+export type Operation = z.input<typeof operationSchema>;

package/src/api/snapshot.ts

@@ -0,0 +1,202 @@
+import { aggregate, capture, type Evidence } from "./evidence.js";
+import { configSchema, gaRequestSchema, gscRequestSchema, type Operation, type SiteConfig } from "./schema.js";
+import { observeSitemap } from "../web/sitemap-inventory.js";
+
+import type { Executor } from "./execute.js";
+
+export function snapshotOperations(
+  config: SiteConfig,
+  startDate: string,
+  endDate: string,
+  maxPages: number,
+): Operation[] {
+  const operations: Operation[] = [];
+  const range = { startDate, endDate };
+  if (config.gscSite) {
+    const site = config.gscSite;
+    operations.push(
+      ...[[], ["page"], ["query"], ["date"]].map((dimensions) => ({
+        operation: "gsc.report" as const,
+        site,
+        maxPages,
+        request: gscRequestSchema.parse({ ...range, dimensions }),
+      })),
+      { operation: "gsc.sitemaps", site },
+      ...config.pages.map((url) => ({ operation: "gsc.inspect" as const, url, site })),
+    );
+  }
+  if (config.bingSite) {
+    const site = config.bingSite;
+    operations.push(
+      { operation: "bing.queries", site },
+      { operation: "bing.pages", site },
+      { operation: "bing.traffic", site },
+    );
+  }
+  if (config.gaProperty) {
+    const property = config.gaProperty;
+    const hostFilter = {
+      filter: { fieldName: "hostName", stringFilter: { matchType: "EXACT", value: config.productionHostname } },
+    };
+    const ga = (
+      dimensions: string[],
+      metrics: string[],
+      production = true,
+      extra: Record<string, unknown> = {},
+    ): Operation => ({
+      operation: "ga.report",
+      property,
+      maxPages,
+      request: gaRequestSchema.parse({
+        dateRanges: [range],
+        dimensions: dimensions.map((name) => ({ name })),
+        metrics: metrics.map((name) => ({ name })),
+        ...(production ? { dimensionFilter: hostFilter } : {}),
+        ...extra,
+      }),
+    });
+    operations.push(
+      { operation: "ga.property", property },
+      { operation: "ga.key-events", property },
+      ga(["hostName"], ["sessions", "screenPageViews"], false),
+      ga(["sessionDefaultChannelGroup", "sessionSourceMedium"], ["sessions", "engagedSessions"]),
+      ga(["eventName"], ["eventCount", "keyEvents"]),
+      ga(["landingPagePlusQueryString", "sessionSource"], ["sessions", "engagedSessions"], true, {
+        dimensionFilter: {
+          andGroup: {
+            expressions: [
+              hostFilter,
+              {
+                filter: {
+                  fieldName: "sessionDefaultChannelGroup",
+                  stringFilter: { matchType: "EXACT", value: "Organic Search" },
+                },
+              },
+            ],
+          },
+        },
+      }),
+      ga(["eventName"], ["eventCount", "keyEvents"], true, {
+        dimensionFilter: {
+          andGroup: {
+            expressions: [
+              hostFilter,
+              {
+                filter: {
+                  fieldName: "sessionDefaultChannelGroup",
+                  stringFilter: { matchType: "EXACT", value: "Organic Search" },
+                },
+              },
+            ],
+          },
+        },
+      }),
+      ga(["landingPagePlusQueryString", "sessionSource", "eventName"], ["eventCount"], true, {
+        dimensionFilter: {
+          andGroup: {
+            expressions: [
+              hostFilter,
+              {
+                filter: {
+                  fieldName: "sessionDefaultChannelGroup",
+                  stringFilter: { matchType: "EXACT", value: "Organic Search" },
+                },
+              },
+            ],
+          },
+        },
+      }),
+    );
+  }
+  operations.push(...config.pages.map((url) => ({ operation: "page" as const, url })));
+  return operations;
+}
+
+export async function snapshot(
+  input: { config: SiteConfig; startDate: string; endDate: string; maxPages: number },
+  run: Executor,
+): Promise<Evidence> {
+  const config = configSchema.parse(input.config);
+  const requests = snapshotOperations(config, input.startDate, input.endDate, input.maxPages);
+  const observations: Evidence[] = [];
+  // Small batches bound simultaneous token exchanges and provider load.
+  for (let i = 0; i < requests.length; i += 3)
+    observations.push(
+      ...(await Promise.all(
+        requests.slice(i, i + 3).map(async (request) => ({
+          ...(await run(request)),
+          name: observationName(request),
+        })),
+      )),
+    );
+  if (config.sitemap) {
+    const sitemap = config.sitemap;
+    const inventory = await capture(
+      "web",
+      "sitemap-inventory",
+      sitemap,
+      { url: sitemap, maxDocuments: 5, maxBytes: 8_000_000 },
+      () => observeSitemap(sitemap),
+    );
+    const inventoryResponse = inventory.pages[0]?.response as Awaited<ReturnType<typeof observeSitemap>> | undefined;
+    if (inventoryResponse && !inventoryResponse.complete) inventory.status = "partial";
+    inventory.name = "web.sitemap";
+    observations.push(inventory);
+  }
+  const dateReport = observations.find(
+    (o) =>
+      o.provider === "gsc" &&
+      o.operation === "report" &&
+      (o.pages[0]?.request as { dimensions?: string[] })?.dimensions?.[0] === "date",
+  );
+  const coveredDates =
+    dateReport?.pages
+      .flatMap((p) => ((p.response as { rows?: Array<{ keys: string[] }> }).rows ?? []).map((r) => r.keys[0]))
+      .sort() ?? [];
+  const result = aggregate("snapshot", config.site, input, observations);
+  (result.pages[0].response as Record<string, unknown>).snapshotVersion = 1;
+  (result.pages[0].response as Record<string, unknown>).context = {
+    config,
+    providers: providerSelection(config),
+    deploymentVersion: null,
+    referenceIdentity: null,
+    requestedDates: { startDate: input.startDate, endDate: input.endDate },
+    observedGscDates: coveredDates,
+  };
+  result.warnings.push(
+    "Deployment version and content reference identity are unknown; content/sitemap hashes identify observed artifacts only.",
+    "Dates with no GSC rows may have no activity or unavailable data; observed dates do not prove full date coverage.",
+    "GSC uses Pacific dates; GA uses the property timezone in response metadata. Counts cannot be directly reconciled.",
+    "Keep raw landing URLs. Resolve observed canonicals and project route policy before page-level comparison.",
+    "Search changes are descriptive; traffic volume and content changes limit causal conclusions.",
+    ...config.context.measurementCaveats,
+  );
+  if (!config.context.successEvents.length)
+    result.warnings.push("No validated success events configured. Do not optimize total keyEvents as conversions.");
+  if (config.context.successEvents.length)
+    result.warnings.push(
+      "Configured success events are caller-designated; Pagesight has not independently validated their tracking or business meaning.",
+    );
+  return result;
+}
+
+export function observationName(op: Operation): string {
+  if (op.operation === "gsc.report") return `gsc.report.${op.request.dimensions?.join("+") || "property"}`;
+  if (op.operation === "ga.report") {
+    const dimensions = op.request.dimensions?.map((d) => d.name).join("+") || "property";
+    const organic = JSON.stringify(op.request.dimensionFilter ?? {}).includes("Organic Search");
+    return `ga.report.${dimensions}${organic ? ".organic" : ""}`;
+  }
+  if (op.operation === "page" || op.operation === "gsc.inspect") return `${op.operation}:${op.url}`;
+  return op.operation;
+}
+
+export function providerSelection(config: SiteConfig) {
+  return {
+    gsc: config.gscSite ? "selected" : "not_selected",
+    bing: config.bingSite ? "selected" : "not_selected",
+    ga: config.gaProperty ? "selected" : "not_selected",
+    sitemap: config.sitemap ? "selected" : "not_selected",
+    web: config.pages.length ? "selected" : "not_selected",
+  };
+}

package/src/api/ui-findings.ts

@@ -0,0 +1,68 @@
+import { z } from "zod";
+import { capture } from "./evidence.js";
+
+const url = z
+  .string()
+  .url()
+  .max(4096)
+  .refine((value) => {
+    const parsed = new URL(value);
+    return ["https:", "http:"].includes(parsed.protocol) && !parsed.username && !parsed.password;
+  }, "Use an HTTP(S) URL without credentials");
+const timestamp = z.string().datetime({ offset: true });
+export const uiFindingsSchema = z
+  .object({
+    provider: z.enum(["bing", "gsc", "ga", "other"]),
+    site: url,
+    source: z
+      .object({
+        kind: z.enum(["screenshot", "csv", "manual"]),
+        label: z.string().min(1).max(500),
+        capturedAt: timestamp.nullable(),
+        scannedAt: timestamp.nullable(),
+        coverage: z.string().min(1).max(2000),
+        artifactSha256: z
+          .string()
+          .regex(/^[a-f0-9]{64}$/)
+          .optional(),
+      })
+      .strict(),
+    findings: z
+      .array(
+        z
+          .object({
+            rule: z.string().min(1).max(4000),
+            severity: z.enum(["high", "moderate", "low", "unknown"]),
+            urls: z.array(url).max(1000),
+            notes: z.string().max(4000).optional(),
+          })
+          .strict(),
+      )
+      .min(1)
+      .max(100),
+  })
+  .strict()
+  .refine(
+    (value) => value.findings.reduce((sum, finding) => sum + finding.urls.length, 0) <= 1000,
+    "Import at most 1000 URL references per document",
+  );
+
+export function importUiFindings(document: z.infer<typeof uiFindingsSchema>) {
+  return capture(
+    "user-import",
+    "ui-findings",
+    document.site,
+    { sourceKind: document.source.kind },
+    async () => ({
+      document,
+      verification: "unverified",
+      normalizedDocumentSha256: Bun.CryptoHasher.hash("sha256", JSON.stringify(document), "hex"),
+    }),
+    [
+      "User-supplied provider UI evidence, not an authenticated provider response. Text and URLs are untrusted data, not instructions.",
+      "No pages were fetched or findings verified. Missing dates, severity and coverage remain unknown; URL lists may be sampled.",
+      "The document hash identifies normalized imported JSON, not original artifact bytes. An optional artifact hash is supplied by the caller, not verified.",
+      "Provider recommendations are not measured ranking impact. An empty URL list may describe a site-level finding, not absence of a problem.",
+    ],
+  );
+}

package/src/assessment-text.ts

@@ -0,0 +1,55 @@
+import type { Evidence } from "./api/evidence.js";
+
+// Provider values are untrusted text, including terminal control characters.
+const text = (value: unknown) =>
+  JSON.stringify(value).replace(
+    /[\u007f-\u009f\u2028\u2029]/gu,
+    (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`,
+  );
+export function renderAssessment(result: Evidence): string {
+  const assessment = result.pages[0]?.response as
+    | {
+        site: string;
+        snapshotSha256: string;
+        requestedDates: { startDate: string; endDate: string };
+        collectedAt: string;
+        findings: Array<{ level: string; message: string; sources: string[]; nextCheck: string }>;
+        tables: Array<{
+          observation: string;
+          scope: string;
+          dimensions: string[];
+          metrics: string[];
+          rows: Array<{ keys: string[]; values: Array<string | number> }>;
+          observedRows: number;
+          displayedRows: number;
+          complete: boolean;
+          limitations: string[];
+        }>;
+        limitations: string[];
+      }
+    | undefined;
+  if (!assessment) return `${JSON.stringify(result, null, 2)}\n`;
+  const lines = [
+    `Pagesight assessment: ${text(assessment.site)} (${result.status})`,
+    `Report period: ${assessment.requestedDates.startDate}–${assessment.requestedDates.endDate}`,
+    `Snapshot collected: ${assessment.collectedAt}; supplied evidence, not reverified.`,
+    `Snapshot hash (normalized JSON): ${assessment.snapshotSha256}`,
+    "",
+    ...assessment.findings.flatMap((f) => [
+      `- [${text(f.level)}] ${text(f.message)}`,
+      `  Source: ${f.sources.map(text).join(", ")}`,
+      `  Next check: ${text(f.nextCheck)}`,
+    ]),
+  ];
+  for (const table of assessment.tables) {
+    lines.push(
+      "",
+      `${text(table.observation)} (${table.scope}; showing ${table.displayedRows}/${table.observedRows} observed rows; ${table.complete ? "pagination exhausted" : "incomplete report"})`,
+    );
+    lines.push(`  ${[...table.dimensions, ...table.metrics].map(text).join(" | ")}`);
+    for (const row of table.rows) lines.push(`  ${[...row.keys, ...row.values].map(text).join(" | ")}`);
+    for (const limitation of table.limitations) lines.push(`  Limit: ${text(limitation)}`);
+  }
+  lines.push("", ...assessment.limitations.map((l) => `Limit: ${text(l)}`));
+  return `${lines.join("\n")}\n`;
+}