pagesight 0.18.0 → 0.20.0

package/src/api/schema.ts

@@ -1,3 +1,6 @@
+import { httpUrl } from "./http-url.js";
+export { httpUrl } from "./http-url.js";
+import { changeRecordSchema } from "./change-record.js";
 import { uiFindingsSchema } from "./ui-findings.js";
 import { z } from "zod";
 import { snapshotEvidenceSchema } from "./evidence-schema.js";
@@ -63,13 +66,43 @@ export const gaRequestSchema = z
   .strict();
 export type GaRequest = z.infer<typeof gaRequestSchema>;
 
-const httpUrl = z
-  .string()
-  .url()
-  .refine(
-    (v) => ["http:", "https:"].includes(new URL(v).protocol) && !new URL(v).username && !new URL(v).password,
-    "Use an HTTP(S) URL without credentials",
-  );
+export const gaRealtimeRequestSchema = z
+  .object({
+    dimensions: z
+      .array(z.object({ name: z.string().min(1) }).strict())
+      .max(9)
+      .default([]),
+    metrics: z
+      .array(z.object({ name: z.string().min(1) }).strict())
+      .min(1)
+      .max(10),
+    dimensionFilter: z.record(z.unknown()).optional(),
+    metricFilter: z.record(z.unknown()).optional(),
+    orderBys: z.array(z.record(z.unknown())).optional(),
+    limit: z.coerce.number().int().min(1).max(250000).default(10000),
+    returnPropertyQuota: z.boolean().default(true),
+    minuteRanges: z
+      .array(
+        z
+          .object({
+            name: z
+              .string()
+              .min(1)
+              .refine((n) => !n.startsWith("date_range_") && !n.startsWith("RESERVED_"))
+              .optional(),
+            startMinutesAgo: z.number().int().min(0).max(59).default(29),
+            endMinutesAgo: z.number().int().min(0).max(59).default(0),
+          })
+          .strict()
+          .refine((r) => r.startMinutesAgo >= r.endMinutesAgo, "Start must be at least as many minutes ago as end"),
+      )
+      .min(1)
+      .max(2)
+      .default([{ startMinutesAgo: 29, endMinutesAgo: 0 }]),
+  })
+  .strict();
+export type GaRealtimeRequest = z.infer<typeof gaRealtimeRequestSchema>;
+
 export const configSchema = z
   .object({
     site: httpUrl,
@@ -113,7 +146,113 @@ export const configSchema = z
     "Select at least one page or provider",
   );
 export type SiteConfig = z.infer<typeof configSchema>;
+export const assessedSnapshotSchema = snapshotEvidenceSchema.superRefine((snapshot, ctx) => {
+  const config = configSchema.safeParse(snapshot.pages[0].response.context.config);
+  if (!config.success)
+    for (const issue of config.error.issues)
+      ctx.addIssue({ ...issue, path: ["pages", 0, "response", "context", "config", ...issue.path] });
+});
+const renderRequestSchema = z
+  .object({
+    operation: z.literal("page.verify"),
+    url: httpUrl,
+    navigation: z
+      .object({ fromUrl: httpUrl, linkSelector: z.string().min(1).max(500) })
+      .strict()
+      .optional(),
+    settleMs: z.number().int().min(0).max(5000).default(1000),
+    timeoutMs: z.number().int().min(1000).max(60000).default(20000),
+    viewport: z
+      .object({ width: z.number().int().min(320).max(2560), height: z.number().int().min(240).max(2160) })
+      .strict()
+      .default({ width: 1280, height: 800 }),
+  })
+  .strict();
 const operationVariants = z.discriminatedUnion("operation", [
+  renderRequestSchema,
+  z
+    .object({
+      operation: z.literal("change.followup"),
+      experiments: z
+        .array(z.unknown())
+        .min(1)
+        .max(50)
+        .describe(
+          "Entries: {label, record: change record, baseline: configured snapshot, current?: configured snapshot}. Each entry is validated independently; use embedded objects, not filesystem paths.",
+        ),
+      asOf: z.string().datetime().optional(),
+      lagDays: z.number().int().min(1).max(30).default(3),
+    })
+    .strict(),
+
+  z
+    .object({
+      operation: z.literal("cloudflare.audit"),
+      zone: z.string().regex(/^[a-f0-9]{32}$/),
+      hostname: z
+        .string()
+        .min(1)
+        .max(253)
+        .regex(/^[a-zA-Z0-9.-]+$/),
+      startTime: z.string().datetime(),
+      endTime: z.string().datetime(),
+      limit: z.number().int().min(1).max(100).default(50),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("change.evaluate"),
+      record: changeRecordSchema,
+      baseline: assessedSnapshotSchema,
+      current: assessedSnapshotSchema.optional(),
+      maxRows: z.number().int().min(1).max(100).default(20),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("crawl"),
+      config: configSchema,
+      maxPages: z.number().int().min(1).max(100).default(20),
+      maxDepth: z.number().int().min(0).max(10).default(3),
+      maxLinks: z.number().int().min(1).max(1000).default(500),
+      includeQuery: z.boolean().default(false),
+      inspectLimit: z.number().int().min(0).max(10).default(3),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("technical.compare"),
+      baseline: assessedSnapshotSchema.optional(),
+      current: assessedSnapshotSchema,
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("investigate"),
+      config: configSchema,
+      url: httpUrl,
+      startDate: dateSchema,
+      endDate: dateSchema,
+      maxPages: maxPagesSchema.default(1),
+      maxRows: z.number().int().min(1).max(100).default(28),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("opportunities"),
+      snapshot: assessedSnapshotSchema,
+      minImpressions: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER).default(20),
+      maxClicks: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).default(2),
+      maxRows: z.number().int().min(1).max(100).default(10),
+    })
+    .strict(),
+  z
+    .object({
+      operation: z.literal("assess"),
+      snapshot: assessedSnapshotSchema,
+      maxRows: z.number().int().min(1).max(100).default(10),
+    })
+    .strict(),
   z.object({ operation: z.literal("evidence.import"), document: uiFindingsSchema }).strict(),
   z
     .object({
@@ -154,6 +293,7 @@ const operationVariants = z.discriminatedUnion("operation", [
       maxPages: maxPagesSchema,
     })
     .strict(),
+  z.object({ operation: z.literal("ga.realtime"), property: z.string(), request: gaRealtimeRequestSchema }).strict(),
   z.object({ operation: z.literal("ga.accounts") }).strict(),
   z.object({ operation: z.literal("ga.property"), property: z.string() }).strict(),
   z.object({ operation: z.literal("ga.key-events"), property: z.string() }).strict(),
@@ -200,8 +340,19 @@ const operationVariants = z.discriminatedUnion("operation", [
     })
     .strict(),
 ]);
-export const operationSchema = operationVariants.refine(
-  (op) => op.operation !== "snapshot" || (op.startDate <= op.endDate && op.endDate <= pacificDate()),
-  "Invalid or future snapshot interval",
-);
+export const operationSchema = operationVariants
+  .refine(
+    (op) =>
+      op.operation !== "cloudflare.audit" ||
+      (Date.parse(op.startTime) < Date.parse(op.endTime) &&
+        Date.parse(op.endTime) - Date.parse(op.startTime) <= 86400000 &&
+        Date.parse(op.endTime) <= Date.now()),
+    "Cloudflare requires a past UTC interval of at most 24 hours",
+  )
+  .refine(
+    (op) =>
+      (op.operation !== "snapshot" && op.operation !== "investigate") ||
+      (op.startDate <= op.endDate && op.endDate <= pacificDate()),
+    "Invalid or future reporting interval",
+  );
 export type Operation = z.input<typeof operationSchema>;

package/src/api/snapshot.ts

@@ -91,6 +91,21 @@ export function snapshotOperations(
           },
         },
       }),
+      ga(["landingPagePlusQueryString", "sessionSource", "eventName"], ["eventCount"], true, {
+        dimensionFilter: {
+          andGroup: {
+            expressions: [
+              hostFilter,
+              {
+                filter: {
+                  fieldName: "sessionDefaultChannelGroup",
+                  stringFilter: { matchType: "EXACT", value: "Organic Search" },
+                },
+              },
+            ],
+          },
+        },
+      }),
     );
   }
   operations.push(...config.pages.map((url) => ({ operation: "page" as const, url })));
@@ -158,10 +173,14 @@ export async function snapshot(
   );
   if (!config.context.successEvents.length)
     result.warnings.push("No validated success events configured. Do not optimize total keyEvents as conversions.");
+  if (config.context.successEvents.length)
+    result.warnings.push(
+      "Configured success events are caller-designated; Pagesight has not independently validated their tracking or business meaning.",
+    );
   return result;
 }
 
-function observationName(op: Operation): string {
+export function observationName(op: Operation): string {
   if (op.operation === "gsc.report") return `gsc.report.${op.request.dimensions?.join("+") || "property"}`;
   if (op.operation === "ga.report") {
     const dimensions = op.request.dimensions?.map((d) => d.name).join("+") || "property";

package/src/api/technical-changes.ts

@@ -0,0 +1,124 @@
+import { z } from "zod";
+import { capture } from "./evidence.js";
+import type { ImportedSnapshot } from "./evidence-schema.js";
+import { configSchema } from "./schema.js";
+import { canonical } from "./report-table.js";
+const pageSchema = z.object({
+  url: z.string().url(),
+  finalUrl: z.string().url(),
+  status: z.number().int(),
+  canonical: z.string().nullable(),
+  robots: z.array(z.string()),
+  xRobotsTag: z.string().nullable(),
+  title: z.string().nullable(),
+  description: z.string().nullable(),
+  sha256: z.string(),
+  redirects: z.array(z.unknown()),
+});
+
+export function technicalChanges(baseline: ImportedSnapshot | undefined, current: ImportedSnapshot) {
+  const hash = (v: unknown) => Bun.CryptoHasher.hash("sha256", canonical(v), "hex");
+  const before = baseline?.pages[0].response;
+  const after = current.pages[0].response;
+  const sameScope = Boolean(
+    baseline &&
+    baseline.target === current.target &&
+    canonical(before?.context.config) === canonical(after.context.config),
+  );
+  return capture(
+    "pagesight",
+    "technical.compare",
+    current.target,
+    { baselineSha256: baseline ? hash(baseline) : null, currentSha256: hash(current) },
+    async () => {
+      const alerts: Array<{ kind: string; source: string; message: string; before?: unknown; after?: unknown }> = [];
+      for (const url of configSchema.parse(after.context.config).pages) {
+        if (!after.observations.some((o) => o.provider === "web" && o.operation === "page" && o.target === url))
+          alerts.push({
+            kind: "unknown",
+            source: `page:${url}`,
+            message: "Configured page observation is missing; no regression inferred",
+          });
+      }
+      for (const observation of after.observations) {
+        if (observation.status === "error" || observation.error) {
+          alerts.push({
+            kind: "availability",
+            source: observation.name,
+            message: `Observation unavailable/incomplete: ${observation.error?.code ?? observation.status}`,
+          });
+          continue;
+        }
+        if (observation.provider !== "web" || observation.operation !== "page") continue;
+        const parsed = pageSchema.safeParse(observation.pages[0]?.response);
+        const request = observation.pages[0]?.request as { url?: string } | undefined;
+        if (!parsed.success || request?.url !== parsed.data.url || observation.target !== parsed.data.url) {
+          alerts.push({
+            kind: "unknown",
+            source: observation.name,
+            message: "Page response is unavailable or not bound to the requested URL",
+          });
+          continue;
+        }
+        if (parsed.data.status >= 400)
+          alerts.push({
+            kind: "http-error",
+            source: observation.name,
+            message: `HTTP ${parsed.data.status} observed`,
+            after: parsed.data.status,
+          });
+        if (!sameScope) continue;
+        const previous = before?.observations.find(
+          (o) => o.provider === "web" && o.operation === "page" && o.target === observation.target,
+        );
+        const prior = pageSchema.safeParse(previous?.pages[0]?.response);
+        const priorRequest = previous?.pages[0]?.request as { url?: string } | undefined;
+        if (
+          !prior.success ||
+          priorRequest?.url !== parsed.data.url ||
+          prior.data.url !== parsed.data.url ||
+          previous?.status === "error" ||
+          previous?.error
+        ) {
+          alerts.push({
+            kind: "unknown",
+            source: observation.name,
+            message: "No usable matching prior page; no regression inferred",
+          });
+          continue;
+        }
+        for (const field of [
+          "status",
+          "finalUrl",
+          "canonical",
+          "robots",
+          "xRobotsTag",
+          "redirects",
+          "title",
+          "description",
+          "sha256",
+        ] as const) {
+          if (canonical(prior.data[field]) !== canonical(parsed.data[field]))
+            alerts.push({
+              kind: ["title", "description", "sha256"].includes(field) ? "content-change" : "technical-change",
+              source: observation.name,
+              message: `Observed ${field} changed; verify against intended route policy`,
+              before: prior.data[field],
+              after: parsed.data[field],
+            });
+        }
+      }
+      return {
+        comparisonVersion: 1,
+        baselineState: !baseline ? "first-run" : sameScope ? "compatible" : "scope-reset",
+        alerts,
+        limitations: [
+          "Exact common requested URLs and equal configuration only. Added or removed URLs reset the baseline.",
+          "Changes are observations, not automatically defects; noindex and canonical policy may be intentional.",
+          "Daily reporting windows overlap. No traffic deltas, disappearance claims, ranking impact or cross-provider comparisons are calculated.",
+          "HTML only; rendered content and Google's stored indexing state require separate checks.",
+        ],
+      };
+    },
+  );
+}

package/src/api/ui-findings.ts

@@ -12,7 +12,7 @@ const url = z
 const timestamp = z.string().datetime({ offset: true });
 export const uiFindingsSchema = z
   .object({
-    provider: z.enum(["bing", "gsc", "other"]),
+    provider: z.enum(["bing", "gsc", "ga", "other"]),
     site: url,
     source: z
       .object({

package/src/api/verify-render.ts

@@ -0,0 +1,134 @@
+import { startRenderNetwork } from "../web/render-network.js";
+import { capture } from "./evidence.js";
+import { fetchText } from "../web/fetch.js";
+import { launchRenderer, parseServer, captureRendered, type RenderOptions } from "../web/render-browser.js";
+import type { SeoDom } from "../web/render-dom.js";
+import { RequestError } from "../shared/http.js";
+
+function compare(a: SeoDom | undefined, b: SeoDom | undefined) {
+  if (!a || !b || a.url !== b.url || a.truncated || b.truncated)
+    return { status: "unavailable", reason: "Both complete captures must identify the same exact final URL." };
+  const fields = ["baseHref", "titles", "descriptions", "canonicals", "robots", "h1", "jsonLd", "links"] as const;
+  const differences = fields
+    .filter((field) => JSON.stringify(a[field]) !== JSON.stringify(b[field]))
+    .map((field) => ({ field, before: a[field], after: b[field] }));
+  return {
+    status: differences.length ? "different" : "equal",
+    scope: "Extracted DOM fields only; HTTP response evidence is compared separately.",
+    differences,
+  };
+}
+export async function verifyRender(options: RenderOptions) {
+  return capture("pagesight", "page.verify", options.url, options, async () => {
+    const network = await startRenderNetwork(options.url);
+    try {
+      const browser = await launchRenderer(network.proxy);
+      try {
+        const server = await capture("web", "server_html", options.url, { url: options.url }, async () => {
+          const { body, finalUrl, response, redirects } = await fetchText(
+            options.url,
+            2_000_000,
+            new URL(options.url).origin,
+            network.proxy,
+          );
+          if (!response.headers.get("content-type")?.includes("text/html"))
+            throw new RequestError("Server response is not HTML.", response.status, "not_html");
+          return {
+            finalUrl,
+            status: response.status,
+            redirects,
+            xRobotsTag: response.headers.get("x-robots-tag"),
+            contentType: response.headers.get("content-type"),
+            sha256: Bun.CryptoHasher.hash("sha256", body, "hex"),
+            bytes: Buffer.byteLength(body),
+            dom: await parseServer(browser, body, finalUrl),
+          };
+        });
+        const direct = await capture("chromium", "direct", options.url, options, () =>
+          captureRendered(browser, options, false),
+        );
+        const navigation = options.navigation
+          ? await capture("chromium", "internal_navigation", options.url, options, () =>
+              captureRendered(browser, options, true),
+            )
+          : null;
+        const getDom = (e: typeof server | null) => (e?.pages[0]?.response as { dom?: SeoDom } | undefined)?.dom;
+        return {
+          verificationVersion: 1,
+          browser: { engine: "chromium", version: browser.version(), viewport: options.viewport },
+          capturePolicy: {
+            waitUntil: "domcontentloaded",
+            settleMs: options.settleMs,
+            timeoutMsPerBrowserPath: options.timeoutMs,
+            anonymousFreshContextPerPath: true,
+            maxRequestsPerPath: 500,
+            allowedMethods: ["GET", "HEAD"],
+            serviceWorkers: "blocked",
+            webSockets: "blocked",
+            topLevelOrigin: new URL(options.url).origin,
+          },
+          network: {
+            blockedRequests: network.blockedRequests,
+            policy:
+              "Public addresses only; an explicitly requested literal private IP permits that exact host and port.",
+          },
+          observations: { server, direct, navigation },
+          documentComparison: (() => {
+            const a = server.pages[0]?.response as { status: number; xRobotsTag: string | null } | undefined;
+            const b = (
+              direct.pages[0]?.response as
+                | { targetDocumentResponse?: { status: number; xRobotsTag: string | null } }
+                | undefined
+            )?.targetDocumentResponse;
+            if (!a || !b)
+              return {
+                status: "unavailable",
+                reason: "Both server and target browser document responses are required.",
+              };
+            const fields = ["status", "xRobotsTag"] as const;
+            return {
+              status: fields.every((f) => a[f] === b[f]) ? "equal" : "different",
+              fields: fields.map((field) => ({ field, server: a[field], browser: b[field] })),
+            };
+          })(),
+          comparisons: {
+            serverToDirect: compare(getDom(server), getDom(direct)),
+            directToNavigation: navigation ? compare(getDom(direct), getDom(navigation)) : { status: "not_requested" },
+          },
+          limitations: [
+            "Bounded anonymous Chromium observation, not Google rendering or indexing evidence.",
+            "A fixed settle interval does not prove hydration or network completion. Captures occur at different times and can see different deployments or personalized content.",
+            "Differences are observations, not automatic SEO defects. Equal extracted fields do not prove visible layout, complete content or functional correctness.",
+            "Only GET/HEAD requests run; service workers and cross-origin top-level navigation are blocked. This may alter application behavior; ordinary page requests can still reach analytics.",
+            "Server HTML is fetched independently with Pagesight's user agent, then parsed inertly in Chromium. A client route may have no target document response; source-document HTTP status is not target status.",
+            "No authenticated profile, storage state, browser actions other than one validated anchor click, or arbitrary evaluation scripts are accepted.",
+          ],
+        };
+      } finally {
+        await browser.close();
+      }
+    } finally {
+      await network.close();
+    }
+  }).then((result) => {
+    const data = result.pages[0]?.response as
+      | {
+          observations: Record<
+            string,
+            { status: string; pages: Array<{ response: { dom?: SeoDom; documentsTruncated?: boolean } }> } | null
+          >;
+          comparisons: Record<string, { status: string }>;
+        }
+      | undefined;
+    if (
+      data &&
+      (Object.values(data.observations).some(
+        (e) =>
+          e && (e.status !== "ok" || e.pages[0]?.response.dom?.truncated || e.pages[0]?.response.documentsTruncated),
+      ) ||
+        Object.values(data.comparisons).some((c) => c.status === "unavailable"))
+    )
+      result.status = "partial";
+    return result;
+  });
+}

package/src/assessment-text.ts

@@ -0,0 +1,55 @@
+import type { Evidence } from "./api/evidence.js";
+
+// Provider values are untrusted text, including terminal control characters.
+const text = (value: unknown) =>
+  JSON.stringify(value).replace(
+    /[\u007f-\u009f\u2028\u2029]/gu,
+    (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`,
+  );
+export function renderAssessment(result: Evidence): string {
+  const assessment = result.pages[0]?.response as
+    | {
+        site: string;
+        snapshotSha256: string;
+        requestedDates: { startDate: string; endDate: string };
+        collectedAt: string;
+        findings: Array<{ level: string; message: string; sources: string[]; nextCheck: string }>;
+        tables: Array<{
+          observation: string;
+          scope: string;
+          dimensions: string[];
+          metrics: string[];
+          rows: Array<{ keys: string[]; values: Array<string | number> }>;
+          observedRows: number;
+          displayedRows: number;
+          complete: boolean;
+          limitations: string[];
+        }>;
+        limitations: string[];
+      }
+    | undefined;
+  if (!assessment) return `${JSON.stringify(result, null, 2)}\n`;
+  const lines = [
+    `Pagesight assessment: ${text(assessment.site)} (${result.status})`,
+    `Report period: ${assessment.requestedDates.startDate}–${assessment.requestedDates.endDate}`,
+    `Snapshot collected: ${assessment.collectedAt}; supplied evidence, not reverified.`,
+    `Snapshot hash (normalized JSON): ${assessment.snapshotSha256}`,
+    "",
+    ...assessment.findings.flatMap((f) => [
+      `- [${text(f.level)}] ${text(f.message)}`,
+      `  Source: ${f.sources.map(text).join(", ")}`,
+      `  Next check: ${text(f.nextCheck)}`,
+    ]),
+  ];
+  for (const table of assessment.tables) {
+    lines.push(
+      "",
+      `${text(table.observation)} (${table.scope}; showing ${table.displayedRows}/${table.observedRows} observed rows; ${table.complete ? "pagination exhausted" : "incomplete report"})`,
+    );
+    lines.push(`  ${[...table.dimensions, ...table.metrics].map(text).join(" | ")}`);
+    for (const row of table.rows) lines.push(`  ${[...row.keys, ...row.values].map(text).join(" | ")}`);
+    for (const limitation of table.limitations) lines.push(`  Limit: ${text(limitation)}`);
+  }
+  lines.push("", ...assessment.limitations.map((l) => `Limit: ${text(l)}`));
+  return `${lines.join("\n")}\n`;
+}