@pagebridge/core 0.0.1

package/dist/url-matcher.js

@@ -0,0 +1,227 @@
+export class URLMatcher {
+    sanityClient;
+    config;
+    constructor(sanityClient, config) {
+        this.sanityClient = sanityClient;
+        this.config = config;
+    }
+    async matchUrls(gscUrls) {
+        const query = `*[_type in $types]{
+      _id,
+      _type,
+      "${this.config.slugField}": ${this.config.slugField}.current,
+      _createdAt
+    }`;
+        const documents = await this.sanityClient.fetch(query, {
+            types: this.config.contentTypes,
+        });
+        const slugToDoc = new Map();
+        const allSlugs = [];
+        for (const doc of documents) {
+            const slug = doc[this.config.slugField];
+            if (slug) {
+                const normalized = this.normalizeSlug(slug);
+                slugToDoc.set(normalized, doc);
+                allSlugs.push(normalized);
+            }
+        }
+        return gscUrls.map((url) => this.matchSingleUrl(url, slugToDoc, allSlugs));
+    }
+    /**
+     * Get all available slugs from Sanity for diagnostic purposes
+     */
+    async getAvailableSlugs() {
+        const query = `*[_type in $types]{
+      "${this.config.slugField}": ${this.config.slugField}.current
+    }`;
+        const documents = await this.sanityClient.fetch(query, {
+            types: this.config.contentTypes,
+        });
+        return documents
+            .map((doc) => doc[this.config.slugField])
+            .filter((slug) => !!slug)
+            .map((slug) => this.normalizeSlug(slug));
+    }
+    matchSingleUrl(gscUrl, slugToDoc, allSlugs) {
+        const normalized = this.normalizeUrl(gscUrl);
+        const extractionResult = this.extractSlugWithDiagnostics(normalized);
+        // Check if URL is outside path prefix
+        if (extractionResult.outsidePrefix) {
+            return {
+                gscUrl,
+                sanityId: undefined,
+                confidence: "none",
+                unmatchReason: "outside_path_prefix",
+                diagnostics: {
+                    normalizedUrl: normalized,
+                    pathAfterPrefix: null,
+                    configuredPrefix: this.config.pathPrefix ?? null,
+                    availableSlugsCount: slugToDoc.size,
+                    similarSlugs: [],
+                },
+            };
+        }
+        const slug = extractionResult.slug;
+        if (!slug) {
+            return {
+                gscUrl,
+                sanityId: undefined,
+                confidence: "none",
+                unmatchReason: "no_slug_extracted",
+                diagnostics: {
+                    normalizedUrl: normalized,
+                    pathAfterPrefix: extractionResult.pathAfterPrefix,
+                    configuredPrefix: this.config.pathPrefix ?? null,
+                    availableSlugsCount: slugToDoc.size,
+                    similarSlugs: [],
+                },
+            };
+        }
+        const exactMatch = slugToDoc.get(slug);
+        if (exactMatch) {
+            return {
+                gscUrl,
+                sanityId: exactMatch._id,
+                confidence: "exact",
+                matchedSlug: slug,
+                unmatchReason: "matched",
+                extractedSlug: slug,
+            };
+        }
+        const withoutTrailing = slug.replace(/\/$/, "");
+        const trailingMatch = slugToDoc.get(withoutTrailing);
+        if (trailingMatch) {
+            return {
+                gscUrl,
+                sanityId: trailingMatch._id,
+                confidence: "normalized",
+                matchedSlug: withoutTrailing,
+                unmatchReason: "matched",
+                extractedSlug: slug,
+            };
+        }
+        const withTrailing = slug + "/";
+        const addedTrailingMatch = slugToDoc.get(withTrailing);
+        if (addedTrailingMatch) {
+            return {
+                gscUrl,
+                sanityId: addedTrailingMatch._id,
+                confidence: "normalized",
+                matchedSlug: withTrailing,
+                unmatchReason: "matched",
+                extractedSlug: slug,
+            };
+        }
+        // No match found - find similar slugs for suggestions
+        const similarSlugs = this.findSimilarSlugs(slug, allSlugs, 3);
+        return {
+            gscUrl,
+            sanityId: undefined,
+            confidence: "none",
+            unmatchReason: "no_matching_document",
+            extractedSlug: slug,
+            diagnostics: {
+                normalizedUrl: normalized,
+                pathAfterPrefix: extractionResult.pathAfterPrefix,
+                configuredPrefix: this.config.pathPrefix ?? null,
+                availableSlugsCount: slugToDoc.size,
+                similarSlugs,
+            },
+        };
+    }
+    normalizeUrl(url) {
+        try {
+            const parsed = new URL(url);
+            parsed.hostname = parsed.hostname.replace(/^www\./, "");
+            parsed.search = "";
+            parsed.hash = "";
+            return parsed.toString().toLowerCase();
+        }
+        catch {
+            return url.toLowerCase();
+        }
+    }
+    extractSlug(normalizedUrl) {
+        return this.extractSlugWithDiagnostics(normalizedUrl).slug;
+    }
+    extractSlugWithDiagnostics(normalizedUrl) {
+        try {
+            const parsed = new URL(normalizedUrl);
+            let path = parsed.pathname;
+            // Check if the URL is outside the configured path prefix
+            if (this.config.pathPrefix) {
+                const prefixRegex = new RegExp(`^${this.escapeRegex(this.config.pathPrefix)}(/|$)`);
+                if (!prefixRegex.test(path)) {
+                    return {
+                        slug: undefined,
+                        pathAfterPrefix: null,
+                        outsidePrefix: true,
+                    };
+                }
+                path = path.replace(new RegExp(`^${this.escapeRegex(this.config.pathPrefix)}`), "");
+            }
+            const slug = path.replace(/^\/+|\/+$/g, "");
+            return {
+                slug: slug || undefined,
+                pathAfterPrefix: path,
+                outsidePrefix: false,
+            };
+        }
+        catch {
+            return {
+                slug: undefined,
+                pathAfterPrefix: null,
+                outsidePrefix: false,
+            };
+        }
+    }
+    escapeRegex(str) {
+        return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    }
+    normalizeSlug(slug) {
+        return slug.replace(/^\/+|\/+$/g, "").toLowerCase();
+    }
+    /**
+     * Find similar slugs using Levenshtein distance
+     */
+    findSimilarSlugs(target, candidates, limit) {
+        const scored = candidates
+            .map((candidate) => ({
+            slug: candidate,
+            distance: this.levenshteinDistance(target, candidate),
+        }))
+            .filter((item) => item.distance <= Math.max(target.length * 0.5, 10)) // Only include reasonably similar
+            .sort((a, b) => a.distance - b.distance)
+            .slice(0, limit);
+        return scored.map((item) => item.slug);
+    }
+    /**
+     * Calculate Levenshtein distance between two strings
+     */
+    levenshteinDistance(a, b) {
+        if (a.length === 0)
+            return b.length;
+        if (b.length === 0)
+            return a.length;
+        const matrix = [];
+        for (let i = 0; i <= b.length; i++) {
+            matrix[i] = [i];
+        }
+        for (let j = 0; j <= a.length; j++) {
+            matrix[0][j] = j;
+        }
+        for (let i = 1; i <= b.length; i++) {
+            for (let j = 1; j <= a.length; j++) {
+                if (b.charAt(i - 1) === a.charAt(j - 1)) {
+                    matrix[i][j] = matrix[i - 1][j - 1];
+                }
+                else {
+                    matrix[i][j] = Math.min(matrix[i - 1][j - 1] + 1, // substitution
+                    matrix[i][j - 1] + 1, // insertion
+                    matrix[i - 1][j] + 1);
+                }
+            }
+        }
+        return matrix[b.length][a.length];
+    }
+}

package/eslint.config.js

@@ -0,0 +1,3 @@
+import { config } from "@pagebridge/eslint-config/base";
+
+export default [...config];

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@pagebridge/core",
+  "version": "0.0.1",
+  "license": "MIT",
+  "private": false,
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@sanity/client": "^7.14.1",
+    "drizzle-orm": "^0.45.1",
+    "google-auth-library": "^10.5.0",
+    "googleapis": "^171.1.0",
+    "@pagebridge/db": "^0.0.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.15.3",
+    "eslint": "^9.39.1",
+    "typescript": "^5.9.3",
+    "@pagebridge/eslint-config": "^0.0.0",
+    "@pagebridge/typescript-config": "^0.0.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "lint": "eslint .",
+    "check-types": "tsc --noEmit"
+  }
+}

package/src/decay-detector.ts

@@ -0,0 +1,254 @@
+import type { DrizzleClient } from "@pagebridge/db";
+import { searchAnalytics } from "@pagebridge/db";
+import { and, avg, gte, lte, sql, eq } from "drizzle-orm";
+
+export interface DecayRule {
+  type: "position_decay" | "low_ctr" | "impressions_drop";
+  threshold: number;
+  minImpressions: number;
+  comparisonWindowDays: number;
+  sustainedDays: number;
+}
+
+export interface QuietPeriodConfig {
+  enabled: boolean;
+  days: number;
+}
+
+export interface DecaySignal {
+  page: string;
+  reason: "position_decay" | "low_ctr" | "impressions_drop";
+  severity: "low" | "medium" | "high";
+  metrics: {
+    positionBefore: number;
+    positionNow: number;
+    positionDelta: number;
+    ctrBefore: number;
+    ctrNow: number;
+    impressions: number;
+  };
+}
+
+interface PageMetrics {
+  page: string;
+  position: number;
+  ctr: number;
+  impressions: number;
+}
+
+export const defaultRules: DecayRule[] = [
+  {
+    type: "position_decay",
+    threshold: 3,
+    minImpressions: 100,
+    comparisonWindowDays: 28,
+    sustainedDays: 14,
+  },
+  {
+    type: "low_ctr",
+    threshold: 0.01,
+    minImpressions: 1000,
+    comparisonWindowDays: 28,
+    sustainedDays: 7,
+  },
+  {
+    type: "impressions_drop",
+    threshold: 0.5,
+    minImpressions: 500,
+    comparisonWindowDays: 28,
+    sustainedDays: 14,
+  },
+];
+
+export class DecayDetector {
+  constructor(
+    private db: DrizzleClient,
+    private rules: DecayRule[] = defaultRules,
+  ) {}
+
+  async detectDecay(
+    siteId: string,
+    publishedDates: Map<string, Date>,
+    quietPeriod: QuietPeriodConfig = { enabled: true, days: 45 },
+  ): Promise<DecaySignal[]> {
+    const signals: DecaySignal[] = [];
+    const now = new Date();
+
+    for (const rule of this.rules) {
+      const currentPeriodEnd = now;
+      const currentPeriodStart = daysAgo(rule.sustainedDays);
+      const previousPeriodEnd = daysAgo(rule.comparisonWindowDays);
+      const previousPeriodStart = daysAgo(
+        rule.comparisonWindowDays + rule.sustainedDays,
+      );
+
+      const [currentMetrics, previousMetrics] = await Promise.all([
+        this.getAverageMetrics(siteId, currentPeriodStart, currentPeriodEnd),
+        this.getAverageMetrics(siteId, previousPeriodStart, previousPeriodEnd),
+      ]);
+
+      for (const current of currentMetrics) {
+        if (quietPeriod.enabled) {
+          const publishDate = publishedDates.get(current.page);
+          if (publishDate && daysSince(publishDate) < quietPeriod.days) {
+            continue;
+          }
+        }
+
+        if (current.impressions < rule.minImpressions) continue;
+
+        const previous = previousMetrics.find((p) => p.page === current.page);
+        if (!previous) continue;
+
+        const signal = this.evaluateRule(rule, current, previous);
+        if (signal) signals.push(signal);
+      }
+    }
+
+    return this.deduplicateSignals(signals);
+  }
+
+  private async getAverageMetrics(
+    siteId: string,
+    startDate: Date,
+    endDate: Date,
+  ): Promise<PageMetrics[]> {
+    const results = await this.db
+      .select({
+        page: searchAnalytics.page,
+        avgPosition: avg(searchAnalytics.position),
+        avgCtr: avg(searchAnalytics.ctr),
+        totalImpressions: sql<number>`sum(${searchAnalytics.impressions})`,
+      })
+      .from(searchAnalytics)
+      .where(
+        and(
+          eq(searchAnalytics.siteId, siteId),
+          gte(searchAnalytics.date, formatDate(startDate)),
+          lte(searchAnalytics.date, formatDate(endDate)),
+        ),
+      )
+      .groupBy(searchAnalytics.page);
+
+    return results.map((r) => ({
+      page: r.page,
+      position: Number(r.avgPosition) || 0,
+      ctr: Number(r.avgCtr) || 0,
+      impressions: Number(r.totalImpressions) || 0,
+    }));
+  }
+
+  private evaluateRule(
+    rule: DecayRule,
+    current: PageMetrics,
+    previous: PageMetrics,
+  ): DecaySignal | undefined {
+    switch (rule.type) {
+      case "position_decay": {
+        const delta = current.position - previous.position;
+        if (delta >= rule.threshold) {
+          return {
+            page: current.page,
+            reason: "position_decay",
+            severity: this.calculateSeverity(delta, [3, 5, 8]),
+            metrics: {
+              positionBefore: previous.position,
+              positionNow: current.position,
+              positionDelta: delta,
+              ctrBefore: previous.ctr,
+              ctrNow: current.ctr,
+              impressions: current.impressions,
+            },
+          };
+        }
+        break;
+      }
+
+      case "low_ctr": {
+        if (current.ctr < rule.threshold && current.position <= 10) {
+          return {
+            page: current.page,
+            reason: "low_ctr",
+            severity: this.calculateSeverity(
+              rule.threshold - current.ctr,
+              [0.005, 0.01, 0.02],
+            ),
+            metrics: {
+              positionBefore: previous.position,
+              positionNow: current.position,
+              positionDelta: current.position - previous.position,
+              ctrBefore: previous.ctr,
+              ctrNow: current.ctr,
+              impressions: current.impressions,
+            },
+          };
+        }
+        break;
+      }
+
+      case "impressions_drop": {
+        const dropRatio = 1 - current.impressions / previous.impressions;
+        if (dropRatio >= rule.threshold) {
+          return {
+            page: current.page,
+            reason: "impressions_drop",
+            severity: this.calculateSeverity(dropRatio, [0.3, 0.5, 0.7]),
+            metrics: {
+              positionBefore: previous.position,
+              positionNow: current.position,
+              positionDelta: current.position - previous.position,
+              ctrBefore: previous.ctr,
+              ctrNow: current.ctr,
+              impressions: current.impressions,
+            },
+          };
+        }
+        break;
+      }
+    }
+
+    return undefined;
+  }
+
+  private calculateSeverity(
+    value: number,
+    thresholds: [number, number, number],
+  ): "low" | "medium" | "high" {
+    if (value >= thresholds[2]) return "high";
+    if (value >= thresholds[1]) return "medium";
+    return "low";
+  }
+
+  private deduplicateSignals(signals: DecaySignal[]): DecaySignal[] {
+    const byPage = new Map<string, DecaySignal>();
+    const severityOrder = { high: 3, medium: 2, low: 1 };
+
+    for (const signal of signals) {
+      const existing = byPage.get(signal.page);
+      if (
+        !existing ||
+        severityOrder[signal.severity] > severityOrder[existing.severity]
+      ) {
+        byPage.set(signal.page, signal);
+      }
+    }
+
+    return Array.from(byPage.values());
+  }
+}
+
+function daysAgo(days: number): Date {
+  const date = new Date();
+  date.setDate(date.getDate() - days);
+  return date;
+}
+
+function daysSince(date: Date): number {
+  const now = new Date();
+  const diff = now.getTime() - date.getTime();
+  return Math.floor(diff / (1000 * 60 * 60 * 24));
+}
+
+function formatDate(date: Date): string {
+  return date.toISOString().split("T")[0]!;
+}

package/src/gsc-client.ts

@@ -0,0 +1,126 @@
+import { google, type searchconsole_v1 } from "googleapis";
+import type { JWT } from "google-auth-library";
+
+export interface GSCClientOptions {
+  credentials: {
+    client_email: string;
+    private_key: string;
+  };
+}
+
+export interface SearchAnalyticsRow {
+  page: string;
+  query?: string;
+  date?: string;
+  clicks: number;
+  impressions: number;
+  ctr: number;
+  position: number;
+}
+
+export interface FetchOptions {
+  siteUrl: string;
+  startDate: Date;
+  endDate: Date;
+  dimensions?: ("page" | "query" | "date")[];
+  rowLimit?: number;
+}
+
+export type IndexVerdict = "PASS" | "FAIL" | "NEUTRAL" | "VERDICT_UNSPECIFIED";
+
+export interface IndexStatusResult {
+  verdict: IndexVerdict;
+  coverageState: string | null;
+  indexingState: string | null;
+  pageFetchState: string | null;
+  lastCrawlTime: Date | null;
+  robotsTxtState: string | null;
+}
+
+export class GSCClient {
+  private auth: JWT;
+  private searchConsole: searchconsole_v1.Searchconsole;
+
+  constructor(options: GSCClientOptions) {
+    this.auth = new google.auth.JWT({
+      email: options.credentials.client_email,
+      key: options.credentials.private_key,
+      scopes: ["https://www.googleapis.com/auth/webmasters.readonly"],
+    });
+
+    this.searchConsole = google.searchconsole({
+      version: "v1",
+      auth: this.auth,
+    });
+  }
+
+  async fetchSearchAnalytics(options: FetchOptions): Promise<SearchAnalyticsRow[]> {
+    const { siteUrl, startDate, endDate, dimensions = ["page", "date"], rowLimit = 25000 } = options;
+
+    const rows: SearchAnalyticsRow[] = [];
+    let startRow = 0;
+
+    while (true) {
+      const response = await this.searchConsole.searchanalytics.query({
+        siteUrl,
+        requestBody: {
+          startDate: formatDate(startDate),
+          endDate: formatDate(endDate),
+          dimensions,
+          rowLimit,
+          startRow,
+        },
+      });
+
+      const responseRows = response.data.rows ?? [];
+      if (responseRows.length === 0) break;
+
+      for (const row of responseRows) {
+        const keys = row.keys ?? [];
+        rows.push({
+          page: dimensions.includes("page") ? keys[dimensions.indexOf("page")]! : "",
+          query: dimensions.includes("query") ? keys[dimensions.indexOf("query")] : undefined,
+          date: dimensions.includes("date") ? keys[dimensions.indexOf("date")] : undefined,
+          clicks: row.clicks ?? 0,
+          impressions: row.impressions ?? 0,
+          ctr: row.ctr ?? 0,
+          position: row.position ?? 0,
+        });
+      }
+
+      if (responseRows.length < rowLimit) break;
+      startRow += rowLimit;
+    }
+
+    return rows;
+  }
+
+  async listSites(): Promise<string[]> {
+    const response = await this.searchConsole.sites.list();
+    return (response.data.siteEntry ?? []).map((site) => site.siteUrl!).filter(Boolean);
+  }
+
+  async inspectUrl(siteUrl: string, inspectionUrl: string): Promise<IndexStatusResult> {
+    const response = await this.searchConsole.urlInspection.index.inspect({
+      requestBody: {
+        inspectionUrl,
+        siteUrl,
+      },
+    });
+
+    const result = response.data.inspectionResult?.indexStatusResult;
+
+    return {
+      verdict: (result?.verdict as IndexVerdict) ?? "VERDICT_UNSPECIFIED",
+      coverageState: result?.coverageState ?? null,
+      indexingState: result?.indexingState ?? null,
+      pageFetchState: result?.pageFetchState ?? null,
+      lastCrawlTime: result?.lastCrawlTime ? new Date(result.lastCrawlTime) : null,
+      robotsTxtState: result?.robotsTxtState ?? null,
+    };
+  }
+}
+
+function formatDate(date: Date): string {
+  return date.toISOString().split("T")[0]!;
+}

package/src/index.ts

@@ -0,0 +1,31 @@
+export {
+  GSCClient,
+  type GSCClientOptions,
+  type IndexStatusResult,
+  type IndexVerdict,
+} from "./gsc-client.js";
+export {
+  SyncEngine,
+  type SyncOptions,
+  type SyncResult,
+  type IndexStatusSyncResult,
+} from "./sync-engine.js";
+export {
+  DecayDetector,
+  defaultRules,
+  type DecayRule,
+  type DecaySignal,
+  type QuietPeriodConfig,
+} from "./decay-detector.js";
+export {
+  URLMatcher,
+  type MatchResult,
+  type URLMatcherConfig,
+  type UnmatchReason,
+  type MatchDiagnostics,
+} from "./url-matcher.js";
+export {
+  TaskGenerator,
+  type TaskGeneratorOptions,
+  type QueryContext,
+} from "./task-generator.js";