@ainyc/canonry 4.60.1 → 4.60.2

package/dist/{chunk-X4ZZFZQZ.js → chunk-AUR7VMQF.js}

@@ -3542,6 +3542,71 @@ function buildPhases(input) {
 // src/gsc-sync.ts
 import crypto4 from "crypto";
 import { eq as eq2, and as and2, sql as sql2 } from "drizzle-orm";
+
+// src/gsc-inspect-paced.ts
+var INSPECT_BASE_DELAY_MS = 1e3;
+var INSPECT_PACING_JITTER_MS = 250;
+var INSPECT_MAX_RETRIES = 3;
+var INSPECT_MAX_BACKOFF_MS = 3e4;
+var INSPECT_FAILFAST_THRESHOLD = 5;
+function isRetryableGscInspectError(err) {
+  if (err != null && typeof err === "object" && "status" in err) {
+    if (err.status === 403) return true;
+  }
+  return isRetryableHttpError(err);
+}
+function defaultSleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function inspectUrlsPaced(urls, cb, deps = {}) {
+  const sleep3 = deps.sleep ?? defaultSleep;
+  const jitter = deps.jitter ?? Math.random;
+  let inspected = 0;
+  let errors = 0;
+  let consecutiveRetryableFailures = 0;
+  for (let index = 0; index < urls.length; index++) {
+    const url = urls[index];
+    try {
+      const result = await withRetry(() => cb.inspectOne(url), {
+        maxRetries: INSPECT_MAX_RETRIES,
+        baseDelayMs: INSPECT_BASE_DELAY_MS,
+        maxDelayMs: INSPECT_MAX_BACKOFF_MS,
+        isRetryable: isRetryableGscInspectError,
+        sleep: sleep3,
+        onRetry: ({ attempt, delayMs, err }) => deps.log?.info("inspect.retry", {
+          url,
+          attempt,
+          delayMs: Math.round(delayMs),
+          error: err instanceof Error ? err.message : String(err)
+        })
+      });
+      cb.onResult(url, result, index);
+      inspected++;
+      consecutiveRetryableFailures = 0;
+    } catch (err) {
+      errors++;
+      cb.onError(url, err, index);
+      if (isRetryableGscInspectError(err)) {
+        consecutiveRetryableFailures++;
+        if (consecutiveRetryableFailures >= INSPECT_FAILFAST_THRESHOLD) {
+          deps.log?.error("inspect.circuit-break", {
+            consecutiveFailures: consecutiveRetryableFailures,
+            inspected,
+            errors,
+            remaining: urls.length - index - 1
+          });
+          return { inspected, errors, aborted: true, abortError: err };
+        }
+      }
+    }
+    if (index < urls.length - 1) {
+      await sleep3(INSPECT_BASE_DELAY_MS + jitter() * INSPECT_PACING_JITTER_MS);
+    }
+  }
+  return { inspected, errors, aborted: false };
+}
+
+// src/gsc-sync.ts
 var log2 = createLogger("GscSync");
 function formatDate(d) {
   return d.toISOString().split("T")[0];
@@ -3570,6 +3635,7 @@ async function executeGscSync(db, runId, projectId, opts) {
     if (!conn.propertyId) {
       throw new Error('No GSC property selected. Use "canonry google properties" to list available sites, then set one with the API.');
     }
+    const propertyId = conn.propertyId;
     let accessToken = conn.accessToken;
     const expiresAt = conn.tokenExpiresAt ? new Date(conn.tokenExpiresAt).getTime() : 0;
     if (Date.now() > expiresAt - 5 * 60 * 1e3) {
@@ -3631,35 +3697,53 @@ async function executeGscSync(db, runId, projectId, opts) {
     }
     const topPages = [...pageClicks.entries()].sort((a, b) => b[1] - a[1]).slice(0, 50).map(([page]) => page);
     log2.info("inspect.start", { runId, projectId, urlCount: topPages.length });
-    for (const pageUrl of topPages) {
-      try {
-        const result = await inspectUrl(accessToken, pageUrl, conn.propertyId);
-        const ir = result.inspectionResult;
-        const idx = ir.indexStatusResult;
-        const mob = ir.mobileUsabilityResult;
-        const rich = ir.richResultsResult;
-        const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
-        db.insert(gscUrlInspections).values({
-          id: crypto4.randomUUID(),
-          projectId,
-          syncRunId: runId,
-          url: pageUrl,
-          indexingState: idx?.indexingState ?? null,
-          verdict: idx?.verdict ?? null,
-          coverageState: idx?.coverageState ?? null,
-          pageFetchState: idx?.pageFetchState ?? null,
-          robotsTxtState: idx?.robotsTxtState ?? null,
-          crawlTime: idx?.lastCrawlTime ?? null,
-          lastCrawlResult: idx?.crawlResult ?? null,
-          isMobileFriendly: mob?.verdict === "PASS" ? true : mob?.verdict === "FAIL" ? false : null,
-          richResults: rich?.detectedItems?.map((d) => d.richResultType) ?? [],
-          referringUrls: idx?.referringUrls ?? [],
-          inspectedAt,
-          createdAt: inspectedAt
-        }).run();
-      } catch (err) {
-        log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
+    const inspectOutcome = await inspectUrlsPaced(
+      topPages,
+      {
+        inspectOne: (pageUrl) => inspectUrl(accessToken, pageUrl, propertyId),
+        onResult: (pageUrl, result) => {
+          const ir = result.inspectionResult;
+          const idx = ir.indexStatusResult;
+          const mob = ir.mobileUsabilityResult;
+          const rich = ir.richResultsResult;
+          const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
+          db.insert(gscUrlInspections).values({
+            id: crypto4.randomUUID(),
+            projectId,
+            syncRunId: runId,
+            url: pageUrl,
+            indexingState: idx?.indexingState ?? null,
+            verdict: idx?.verdict ?? null,
+            coverageState: idx?.coverageState ?? null,
+            pageFetchState: idx?.pageFetchState ?? null,
+            robotsTxtState: idx?.robotsTxtState ?? null,
+            crawlTime: idx?.lastCrawlTime ?? null,
+            lastCrawlResult: idx?.crawlResult ?? null,
+            isMobileFriendly: mob?.verdict === "PASS" ? true : mob?.verdict === "FAIL" ? false : null,
+            richResults: rich?.detectedItems?.map((d) => d.richResultType) ?? [],
+            referringUrls: idx?.referringUrls ?? [],
+            inspectedAt,
+            createdAt: inspectedAt
+          }).run();
+        },
+        onError: (pageUrl, err) => {
+          log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
+        }
+      },
+      {
+        log: {
+          info: (action, ctx) => log2.info(action, { runId, projectId, ...ctx }),
+          error: (action, ctx) => log2.error(action, { runId, projectId, ...ctx })
+        }
       }
+    );
+    if (inspectOutcome.aborted) {
+      log2.error("inspect.stopped-early", {
+        runId,
+        projectId,
+        inspected: inspectOutcome.inspected,
+        note: "URL inspection stopped early after sustained rate/access failures; search-analytics data was still saved"
+      });
     }
     const allInspections = db.select().from(gscUrlInspections).where(eq2(gscUrlInspections.projectId, projectId)).all();
     const latestByUrl = /* @__PURE__ */ new Map();
@@ -4057,6 +4141,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
     if (!conn.propertyId) {
       throw new Error('No GSC property selected. Use "canonry google properties" to list available sites, then set one.');
     }
+    const propertyId = conn.propertyId;
     let accessToken = conn.accessToken;
     const expiresAt = conn.tokenExpiresAt ? new Date(conn.tokenExpiresAt).getTime() : 0;
     if (Date.now() > expiresAt - 5 * 60 * 1e3) {
@@ -4076,43 +4161,52 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
     if (urls.length === 0) {
       throw new Error("No URLs found in sitemap");
     }
-    let inspected = 0;
-    let errors = 0;
-    for (const pageUrl of urls) {
-      try {
-        const result = await inspectUrl(accessToken, pageUrl, conn.propertyId);
-        const ir = result.inspectionResult;
-        const idx = ir.indexStatusResult;
-        const mob = ir.mobileUsabilityResult;
-        const rich = ir.richResultsResult;
-        const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
-        db.insert(gscUrlInspections).values({
-          id: crypto6.randomUUID(),
-          projectId,
-          syncRunId: runId,
-          url: pageUrl,
-          indexingState: idx?.indexingState ?? null,
-          verdict: idx?.verdict ?? null,
-          coverageState: idx?.coverageState ?? null,
-          pageFetchState: idx?.pageFetchState ?? null,
-          robotsTxtState: idx?.robotsTxtState ?? null,
-          crawlTime: idx?.lastCrawlTime ?? null,
-          lastCrawlResult: idx?.crawlResult ?? null,
-          isMobileFriendly: mob?.verdict === "PASS" ? true : mob?.verdict === "FAIL" ? false : null,
-          richResults: rich?.detectedItems?.map((d) => d.richResultType) ?? [],
-          referringUrls: idx?.referringUrls ?? [],
-          inspectedAt,
-          createdAt: inspectedAt
-        }).run();
-        inspected++;
-        log5.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${inspected}/${urls.length}` });
-      } catch (err) {
-        errors++;
-        log5.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
-      }
-      if (inspected + errors < urls.length) {
-        await new Promise((r) => setTimeout(r, 1e3));
+    const { inspected, errors, aborted, abortError } = await inspectUrlsPaced(
+      urls,
+      {
+        inspectOne: (pageUrl) => inspectUrl(accessToken, pageUrl, propertyId),
+        onResult: (pageUrl, result, index) => {
+          const ir = result.inspectionResult;
+          const idx = ir.indexStatusResult;
+          const mob = ir.mobileUsabilityResult;
+          const rich = ir.richResultsResult;
+          const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
+          db.insert(gscUrlInspections).values({
+            id: crypto6.randomUUID(),
+            projectId,
+            syncRunId: runId,
+            url: pageUrl,
+            indexingState: idx?.indexingState ?? null,
+            verdict: idx?.verdict ?? null,
+            coverageState: idx?.coverageState ?? null,
+            pageFetchState: idx?.pageFetchState ?? null,
+            robotsTxtState: idx?.robotsTxtState ?? null,
+            crawlTime: idx?.lastCrawlTime ?? null,
+            lastCrawlResult: idx?.crawlResult ?? null,
+            isMobileFriendly: mob?.verdict === "PASS" ? true : mob?.verdict === "FAIL" ? false : null,
+            richResults: rich?.detectedItems?.map((d) => d.richResultType) ?? [],
+            referringUrls: idx?.referringUrls ?? [],
+            inspectedAt,
+            createdAt: inspectedAt
+          }).run();
+          log5.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${index + 1}/${urls.length}` });
+        },
+        onError: (pageUrl, err) => {
+          log5.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
+        }
+      },
+      {
+        log: {
+          info: (action, ctx) => log5.info(action, { runId, projectId, ...ctx }),
+          error: (action, ctx) => log5.error(action, { runId, projectId, ...ctx })
+        }
       }
+    );
+    if (aborted) {
+      const detail = abortError instanceof Error ? abortError.message : String(abortError);
+      throw new Error(
+        `URL inspection aborted after ${INSPECT_FAILFAST_THRESHOLD} consecutive rate/access failures (likely GSC URL Inspection quota exhaustion or property access loss). Last error: ${detail}`
+      );
     }
     const allInspections = db.select().from(gscUrlInspections).where(eq4(gscUrlInspections.projectId, projectId)).all();
     const latestByUrl = /* @__PURE__ */ new Map();

package/dist/cli.js

@@ -27,7 +27,7 @@ import {
   setTelemetrySource,
   showFirstRunNotice,
   trackEvent
-} from "./chunk-X4ZZFZQZ.js";
+} from "./chunk-AUR7VMQF.js";
 import {
   CliError,
   EXIT_SYSTEM_ERROR,

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   createServer
-} from "./chunk-X4ZZFZQZ.js";
+} from "./chunk-AUR7VMQF.js";
 import {
   loadConfig
 } from "./chunk-CKWHFAVB.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ainyc/canonry",
-  "version": "4.60.1",
+  "version": "4.60.2",
   "type": "module",
   "description": "Agent-first open-source AEO operating platform - track how answer engines cite your domain",
   "license": "FSL-1.1-ALv2",
@@ -61,25 +61,25 @@
     "@types/node-cron": "^3.0.11",
     "tsup": "^8.5.1",
     "tsx": "^4.19.0",
+    "@ainyc/canonry-api-routes": "0.0.0",
     "@ainyc/canonry-api-client": "0.0.0",
-    "@ainyc/canonry-contracts": "0.0.0",
     "@ainyc/canonry-config": "0.0.0",
+    "@ainyc/canonry-contracts": "0.0.0",
     "@ainyc/canonry-db": "0.0.0",
-    "@ainyc/canonry-api-routes": "0.0.0",
     "@ainyc/canonry-integration-bing": "0.0.0",
-    "@ainyc/canonry-integration-cloud-run": "0.0.0",
     "@ainyc/canonry-integration-commoncrawl": "0.0.0",
     "@ainyc/canonry-integration-google": "0.0.0",
-    "@ainyc/canonry-integration-google-business-profile": "0.0.0",
+    "@ainyc/canonry-integration-cloud-run": "0.0.0",
     "@ainyc/canonry-integration-traffic": "0.0.0",
     "@ainyc/canonry-integration-wordpress": "0.0.0",
     "@ainyc/canonry-intelligence": "0.0.0",
+    "@ainyc/canonry-integration-google-business-profile": "0.0.0",
     "@ainyc/canonry-provider-cdp": "0.0.0",
     "@ainyc/canonry-provider-claude": "0.0.0",
-    "@ainyc/canonry-provider-gemini": "0.0.0",
     "@ainyc/canonry-provider-local": "0.0.0",
-    "@ainyc/canonry-provider-openai": "0.0.0",
-    "@ainyc/canonry-provider-perplexity": "0.0.0"
+    "@ainyc/canonry-provider-gemini": "0.0.0",
+    "@ainyc/canonry-provider-perplexity": "0.0.0",
+    "@ainyc/canonry-provider-openai": "0.0.0"
   },
   "scripts": {
     "build": "tsx scripts/copy-agent-assets.ts && tsup && tsx build-web.ts",