@pagebridge/core 0.0.1

package/dist/sync-engine.js

@@ -0,0 +1,333 @@
+import { searchAnalytics, queryAnalytics, syncLog, pageIndexStatus, } from "@pagebridge/db";
+import { and, eq, gte, lte } from "drizzle-orm";
+export class SyncEngine {
+    gsc;
+    db;
+    sanity;
+    constructor(options) {
+        this.gsc = options.gsc;
+        this.db = options.db;
+        this.sanity = options.sanity;
+    }
+    async sync(options) {
+        const { siteUrl, startDate = daysAgo(90), endDate = daysAgo(3), dimensions = ["page", "date"], } = options;
+        const syncLogId = `${siteUrl}:${Date.now()}`;
+        await this.db.insert(syncLog).values({
+            id: syncLogId,
+            siteId: siteUrl,
+            startedAt: new Date(),
+            status: "running",
+        });
+        try {
+            const rows = await this.gsc.fetchSearchAnalytics({
+                siteUrl,
+                startDate,
+                endDate,
+                dimensions,
+            });
+            const pages = new Set();
+            for (const row of rows) {
+                pages.add(row.page);
+                if (row.date) {
+                    const id = `${siteUrl}:${row.page}:${row.date}`;
+                    await this.db
+                        .insert(searchAnalytics)
+                        .values({
+                        id,
+                        siteId: siteUrl,
+                        page: row.page,
+                        date: row.date,
+                        clicks: row.clicks,
+                        impressions: row.impressions,
+                        ctr: row.ctr,
+                        position: row.position,
+                    })
+                        .onConflictDoUpdate({
+                        target: searchAnalytics.id,
+                        set: {
+                            clicks: row.clicks,
+                            impressions: row.impressions,
+                            ctr: row.ctr,
+                            position: row.position,
+                            fetchedAt: new Date(),
+                        },
+                    });
+                }
+                if (row.query && row.date) {
+                    const id = `${siteUrl}:${row.page}:${row.query}:${row.date}`;
+                    await this.db
+                        .insert(queryAnalytics)
+                        .values({
+                        id,
+                        siteId: siteUrl,
+                        page: row.page,
+                        query: row.query,
+                        date: row.date,
+                        clicks: row.clicks,
+                        impressions: row.impressions,
+                        ctr: row.ctr,
+                        position: row.position,
+                    })
+                        .onConflictDoUpdate({
+                        target: queryAnalytics.id,
+                        set: {
+                            clicks: row.clicks,
+                            impressions: row.impressions,
+                            ctr: row.ctr,
+                            position: row.position,
+                        },
+                    });
+                }
+            }
+            await this.db
+                .update(syncLog)
+                .set({
+                status: "completed",
+                completedAt: new Date(),
+                rowsProcessed: rows.length,
+            })
+                .where(eq(syncLog.id, syncLogId));
+            return {
+                pages: Array.from(pages),
+                rowsProcessed: rows.length,
+                syncLogId,
+            };
+        }
+        catch (error) {
+            await this.db
+                .update(syncLog)
+                .set({
+                status: "failed",
+                completedAt: new Date(),
+                error: error instanceof Error ? error.message : String(error),
+            })
+                .where(eq(syncLog.id, syncLogId));
+            throw error;
+        }
+    }
+    async writeSnapshots(siteId, matches, siteUrl) {
+        // Get the siteUrl from Sanity if not provided
+        let resolvedSiteUrl = siteUrl;
+        if (!resolvedSiteUrl) {
+            const siteDoc = await this.sanity.fetch(`*[_type == "gscSite" && _id == $siteId][0]{ siteUrl }`, { siteId });
+            resolvedSiteUrl = siteDoc?.siteUrl;
+        }
+        if (!resolvedSiteUrl) {
+            throw new Error(`Could not find siteUrl for site ID: ${siteId}`);
+        }
+        const periods = ["last7", "last28", "last90"];
+        const periodDays = { last7: 7, last28: 28, last90: 90 };
+        for (const period of periods) {
+            const startDate = daysAgo(periodDays[period]);
+            const endDate = daysAgo(3);
+            for (const match of matches) {
+                if (!match.sanityId)
+                    continue;
+                const metrics = await this.getAggregatedMetrics(resolvedSiteUrl, match.gscUrl, startDate, endDate);
+                if (!metrics)
+                    continue;
+                const topQueries = await this.getTopQueries(resolvedSiteUrl, match.gscUrl, startDate, endDate);
+                // Get index status from database
+                const indexStatusData = await this.getIndexStatus(resolvedSiteUrl, match.gscUrl);
+                const existingSnapshot = await this.sanity.fetch(`*[_type == "gscSnapshot" && site._ref == $siteId && page == $page && period == $period][0]._id`, { siteId, page: match.gscUrl, period });
+                const snapshotData = {
+                    _type: "gscSnapshot",
+                    site: { _type: "reference", _ref: siteId },
+                    page: match.gscUrl,
+                    linkedDocument: { _type: "reference", _ref: match.sanityId },
+                    period,
+                    clicks: metrics.clicks,
+                    impressions: metrics.impressions,
+                    ctr: metrics.ctr,
+                    position: metrics.position,
+                    topQueries,
+                    fetchedAt: new Date().toISOString(),
+                    indexStatus: indexStatusData
+                        ? {
+                            verdict: mapVerdictToSanity(indexStatusData.verdict),
+                            coverageState: indexStatusData.coverageState,
+                            lastCrawlTime: indexStatusData.lastCrawlTime?.toISOString() ?? null,
+                            robotsTxtState: indexStatusData.robotsTxtState,
+                            pageFetchState: indexStatusData.pageFetchState,
+                        }
+                        : undefined,
+                };
+                if (existingSnapshot) {
+                    await this.sanity.patch(existingSnapshot).set(snapshotData).commit();
+                }
+                else {
+                    await this.sanity.create(snapshotData);
+                }
+            }
+        }
+    }
+    async syncIndexStatus(siteUrl, pages) {
+        const result = {
+            checked: 0,
+            indexed: 0,
+            notIndexed: 0,
+            skipped: 0,
+        };
+        const CACHE_DURATION_MS = 24 * 60 * 60 * 1000; // 24 hours
+        for (const page of pages) {
+            const id = `${siteUrl}:${page}`;
+            // Check if we already have a recent status
+            const existing = await this.db
+                .select({ fetchedAt: pageIndexStatus.fetchedAt })
+                .from(pageIndexStatus)
+                .where(eq(pageIndexStatus.id, id))
+                .limit(1);
+            if (existing.length > 0 &&
+                existing[0]?.fetchedAt &&
+                Date.now() - existing[0].fetchedAt.getTime() < CACHE_DURATION_MS) {
+                result.skipped++;
+                continue;
+            }
+            try {
+                const status = await this.gsc.inspectUrl(siteUrl, page);
+                await this.db
+                    .insert(pageIndexStatus)
+                    .values({
+                    id,
+                    siteId: siteUrl,
+                    page,
+                    verdict: status.verdict,
+                    coverageState: status.coverageState,
+                    indexingState: status.indexingState,
+                    pageFetchState: status.pageFetchState,
+                    lastCrawlTime: status.lastCrawlTime,
+                    robotsTxtState: status.robotsTxtState,
+                    fetchedAt: new Date(),
+                })
+                    .onConflictDoUpdate({
+                    target: pageIndexStatus.id,
+                    set: {
+                        verdict: status.verdict,
+                        coverageState: status.coverageState,
+                        indexingState: status.indexingState,
+                        pageFetchState: status.pageFetchState,
+                        lastCrawlTime: status.lastCrawlTime,
+                        robotsTxtState: status.robotsTxtState,
+                        fetchedAt: new Date(),
+                    },
+                });
+                result.checked++;
+                if (status.verdict === "PASS") {
+                    result.indexed++;
+                }
+                else {
+                    result.notIndexed++;
+                }
+                // Small delay to respect rate limits (600/min = 100ms between requests)
+                await delay(100);
+            }
+            catch (error) {
+                console.error(`Failed to check index status for ${page}:`, error);
+                result.skipped++;
+            }
+        }
+        return result;
+    }
+    async getIndexStatus(siteUrl, page) {
+        const id = `${siteUrl}:${page}`;
+        const rows = await this.db
+            .select()
+            .from(pageIndexStatus)
+            .where(eq(pageIndexStatus.id, id))
+            .limit(1);
+        if (rows.length === 0)
+            return null;
+        const row = rows[0];
+        return {
+            verdict: row.verdict,
+            coverageState: row.coverageState,
+            indexingState: row.indexingState,
+            pageFetchState: row.pageFetchState,
+            lastCrawlTime: row.lastCrawlTime,
+            robotsTxtState: row.robotsTxtState,
+        };
+    }
+    async getAggregatedMetrics(siteId, page, startDate, endDate) {
+        const results = await this.db
+            .select({
+            totalClicks: searchAnalytics.clicks,
+            totalImpressions: searchAnalytics.impressions,
+            avgCtr: searchAnalytics.ctr,
+            avgPosition: searchAnalytics.position,
+        })
+            .from(searchAnalytics)
+            .where(and(eq(searchAnalytics.siteId, siteId), eq(searchAnalytics.page, page), gte(searchAnalytics.date, formatDate(startDate)), lte(searchAnalytics.date, formatDate(endDate))));
+        if (results.length === 0)
+            return undefined;
+        const totalClicks = results.reduce((sum, r) => sum + (r.totalClicks ?? 0), 0);
+        const totalImpressions = results.reduce((sum, r) => sum + (r.totalImpressions ?? 0), 0);
+        const avgCtr = totalImpressions > 0 ? totalClicks / totalImpressions : 0;
+        const avgPosition = results.reduce((sum, r) => sum + (r.avgPosition ?? 0), 0) /
+            results.length;
+        return {
+            clicks: totalClicks,
+            impressions: totalImpressions,
+            ctr: avgCtr,
+            position: avgPosition,
+        };
+    }
+    async getTopQueries(siteId, page, startDate, endDate) {
+        const results = await this.db
+            .select({
+            query: queryAnalytics.query,
+            clicks: queryAnalytics.clicks,
+            impressions: queryAnalytics.impressions,
+            position: queryAnalytics.position,
+        })
+            .from(queryAnalytics)
+            .where(and(eq(queryAnalytics.siteId, siteId), eq(queryAnalytics.page, page), gte(queryAnalytics.date, formatDate(startDate)), lte(queryAnalytics.date, formatDate(endDate))))
+            .limit(10);
+        const queryMap = new Map();
+        for (const row of results) {
+            const existing = queryMap.get(row.query);
+            if (existing) {
+                existing.clicks += row.clicks ?? 0;
+                existing.impressions += row.impressions ?? 0;
+                existing.positions.push(row.position ?? 0);
+            }
+            else {
+                queryMap.set(row.query, {
+                    clicks: row.clicks ?? 0,
+                    impressions: row.impressions ?? 0,
+                    positions: [row.position ?? 0],
+                });
+            }
+        }
+        return Array.from(queryMap.entries())
+            .map(([query, data]) => ({
+            query,
+            clicks: data.clicks,
+            impressions: data.impressions,
+            position: data.positions.reduce((a, b) => a + b, 0) / data.positions.length,
+        }))
+            .sort((a, b) => b.clicks - a.clicks)
+            .slice(0, 10);
+    }
+}
+function daysAgo(days) {
+    const date = new Date();
+    date.setDate(date.getDate() - days);
+    return date;
+}
+function formatDate(date) {
+    return date.toISOString().split("T")[0];
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function mapVerdictToSanity(verdict) {
+    switch (verdict) {
+        case "PASS":
+            return "indexed";
+        case "NEUTRAL":
+            return "excluded";
+        case "FAIL":
+        default:
+            return "not_indexed";
+    }
+}

package/dist/task-generator.d.ts

@@ -0,0 +1,26 @@
+import type { SanityClient } from "@sanity/client";
+import type { DrizzleClient } from "@pagebridge/db";
+import type { DecaySignal } from "./decay-detector.js";
+import type { MatchResult } from "./url-matcher.js";
+export interface QueryContext {
+    query: string;
+    impressions: number;
+    clicks: number;
+    position: number;
+}
+export interface TaskGeneratorOptions {
+    sanity: SanityClient;
+    db?: DrizzleClient;
+}
+export declare class TaskGenerator {
+    private sanity;
+    private db?;
+    constructor(options: TaskGeneratorOptions | SanityClient);
+    createTasks(siteId: string, signals: DecaySignal[], matches: MatchResult[], siteUrl?: string): Promise<number>;
+    private getTopQueries;
+    updateTaskStatus(taskId: string, status: "open" | "snoozed" | "in_progress" | "done" | "dismissed", options?: {
+        snoozeDays?: number;
+        notes?: string;
+    }): Promise<void>;
+}
+//# sourceMappingURL=task-generator.d.ts.map

package/dist/task-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-generator.d.ts","sourceRoot":"","sources":["../src/task-generator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,YAAY,CAAC;IACrB,EAAE,CAAC,EAAE,aAAa,CAAC;CACpB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,EAAE,CAAC,CAAgB;gBAEf,OAAO,EAAE,oBAAoB,GAAG,YAAY;IAUlD,WAAW,CACf,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,WAAW,EAAE,EACtB,OAAO,EAAE,WAAW,EAAE,EACtB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;YAuDJ,aAAa;IAuCrB,gBAAgB,CACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,GAAG,MAAM,GAAG,WAAW,EACjE,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAChD,OAAO,CAAC,IAAI,CAAC;CAmBjB"}

package/dist/task-generator.js

@@ -0,0 +1,101 @@
+import { queryAnalytics } from "@pagebridge/db";
+import { and, eq, gte, lte, sql, desc } from "drizzle-orm";
+export class TaskGenerator {
+    sanity;
+    db;
+    constructor(options) {
+        // Support both old (SanityClient) and new (options object) signatures
+        if ("fetch" in options) {
+            this.sanity = options;
+        }
+        else {
+            this.sanity = options.sanity;
+            this.db = options.db;
+        }
+    }
+    async createTasks(siteId, signals, matches, siteUrl) {
+        let created = 0;
+        // Get siteUrl from Sanity if not provided (needed for query lookup)
+        let resolvedSiteUrl = siteUrl;
+        if (!resolvedSiteUrl && this.db) {
+            const siteDoc = await this.sanity.fetch(`*[_type == "gscSite" && _id == $siteId][0]{ siteUrl }`, { siteId });
+            resolvedSiteUrl = siteDoc?.siteUrl;
+        }
+        for (const signal of signals) {
+            const match = matches.find((m) => m.gscUrl === signal.page);
+            if (!match?.sanityId)
+                continue;
+            const existingTask = await this.sanity.fetch(`*[_type == "gscRefreshTask" && linkedDocument._ref == $docId && status in ["open", "in_progress"]][0]._id`, { docId: match.sanityId });
+            if (existingTask)
+                continue;
+            // Fetch top queries for this page if database is available
+            let queryContext;
+            if (this.db && resolvedSiteUrl) {
+                queryContext = await this.getTopQueries(resolvedSiteUrl, signal.page);
+            }
+            await this.sanity.create({
+                _type: "gscRefreshTask",
+                site: { _type: "reference", _ref: siteId },
+                linkedDocument: { _type: "reference", _ref: match.sanityId },
+                reason: signal.reason,
+                severity: signal.severity,
+                status: "open",
+                metrics: {
+                    positionBefore: signal.metrics.positionBefore,
+                    positionNow: signal.metrics.positionNow,
+                    positionDelta: signal.metrics.positionDelta,
+                    ctrBefore: signal.metrics.ctrBefore,
+                    ctrNow: signal.metrics.ctrNow,
+                    impressions: signal.metrics.impressions,
+                },
+                ...(queryContext && queryContext.length > 0 && { queryContext }),
+                createdAt: new Date().toISOString(),
+            });
+            created++;
+        }
+        return created;
+    }
+    async getTopQueries(siteId, page, limit = 5) {
+        if (!this.db)
+            return [];
+        const endDate = new Date();
+        const startDate = new Date();
+        startDate.setDate(startDate.getDate() - 28);
+        const results = await this.db
+            .select({
+            query: queryAnalytics.query,
+            totalClicks: sql `sum(${queryAnalytics.clicks})`,
+            totalImpressions: sql `sum(${queryAnalytics.impressions})`,
+            avgPosition: sql `avg(${queryAnalytics.position})`,
+        })
+            .from(queryAnalytics)
+            .where(and(eq(queryAnalytics.siteId, siteId), eq(queryAnalytics.page, page), gte(queryAnalytics.date, formatDate(startDate)), lte(queryAnalytics.date, formatDate(endDate))))
+            .groupBy(queryAnalytics.query)
+            .orderBy(desc(sql `sum(${queryAnalytics.impressions})`))
+            .limit(limit);
+        return results.map((r) => ({
+            query: r.query,
+            clicks: Number(r.totalClicks) || 0,
+            impressions: Number(r.totalImpressions) || 0,
+            position: Number(r.avgPosition) || 0,
+        }));
+    }
+    async updateTaskStatus(taskId, status, options) {
+        const patch = { status };
+        if (status === "snoozed" && options?.snoozeDays) {
+            const until = new Date();
+            until.setDate(until.getDate() + options.snoozeDays);
+            patch.snoozedUntil = until.toISOString();
+        }
+        if (status === "done" || status === "dismissed") {
+            patch.resolvedAt = new Date().toISOString();
+        }
+        if (options?.notes) {
+            patch.notes = options.notes;
+        }
+        await this.sanity.patch(taskId).set(patch).commit();
+    }
+}
+function formatDate(date) {
+    return date.toISOString().split("T")[0];
+}

package/dist/url-matcher.d.ts

@@ -0,0 +1,49 @@
+import type { SanityClient } from "@sanity/client";
+export type UnmatchReason = "matched" | "no_slug_extracted" | "no_matching_document" | "outside_path_prefix";
+export interface MatchDiagnostics {
+    normalizedUrl: string;
+    pathAfterPrefix: string | null;
+    configuredPrefix: string | null;
+    availableSlugsCount: number;
+    similarSlugs: string[];
+}
+export interface MatchResult {
+    gscUrl: string;
+    sanityId: string | undefined;
+    confidence: "exact" | "normalized" | "fuzzy" | "none";
+    matchedSlug?: string;
+    unmatchReason: UnmatchReason;
+    extractedSlug?: string;
+    diagnostics?: MatchDiagnostics;
+}
+export interface URLMatcherConfig {
+    contentTypes: string[];
+    slugField: string;
+    pathPrefix?: string;
+    baseUrl: string;
+}
+export declare class URLMatcher {
+    private sanityClient;
+    private config;
+    constructor(sanityClient: SanityClient, config: URLMatcherConfig);
+    matchUrls(gscUrls: string[]): Promise<MatchResult[]>;
+    /**
+     * Get all available slugs from Sanity for diagnostic purposes
+     */
+    getAvailableSlugs(): Promise<string[]>;
+    private matchSingleUrl;
+    private normalizeUrl;
+    private extractSlug;
+    private extractSlugWithDiagnostics;
+    private escapeRegex;
+    private normalizeSlug;
+    /**
+     * Find similar slugs using Levenshtein distance
+     */
+    private findSimilarSlugs;
+    /**
+     * Calculate Levenshtein distance between two strings
+     */
+    private levenshteinDistance;
+}
+//# sourceMappingURL=url-matcher.d.ts.map

package/dist/url-matcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-matcher.d.ts","sourceRoot":"","sources":["../src/url-matcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,mBAAmB,GACnB,sBAAsB,GACtB,qBAAqB,CAAC;AAE1B,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,UAAU,EAAE,OAAO,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAQD,qBAAa,UAAU;IAEnB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,MAAM;gBADN,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,gBAAgB;IAG5B,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAyB1D;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAc5C,OAAO,CAAC,cAAc;IAoGtB,OAAO,CAAC,YAAY;IAYpB,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,0BAA0B;IA0ClC,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,aAAa;IAIrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;CA8B5B"}