@ainyc/canonry 2.4.3 → 2.4.6

package/README.md

@@ -166,7 +166,17 @@ Integration setup guides: [Google Search Console](docs/google-search-console-set
 
 ## Skills
 
-Canonry ships a bundled `canonry-setup` skill that documents the CLI commands, provider setup, analysis workflows, and troubleshooting patterns an agent needs to operate the platform. **Claude Code** picks up the skill automatically from `.claude/skills/canonry-setup/` when you open this repo.
+Canonry ships a bundled `canonry-setup` skill that turns Aero (or any Claude-powered agent) into an AEO/SEO operator. **Claude Code** picks it up automatically from `.claude/skills/canonry-setup/` when you open this repo; the same content lives under [`skills/canonry-setup/`](skills/canonry-setup/) for portable use with other harnesses.
+
+The skill covers the end-to-end answer-engine optimization loop:
+
+- **AEO monitoring.** Running citation sweeps across Gemini, ChatGPT, Claude, and Perplexity via `canonry run` / `canonry evidence` / `canonry status`, including how to interpret per-phrase citation state and regressions.
+- **Technical SEO audits.** Driving the companion [`@ainyc/aeo-audit`](https://www.npmjs.com/package/@ainyc/aeo-audit) CLI for 14-factor scoring — structured data (JSON-LD), content depth, AI-readable files (`llms.txt`, `llms-full.txt`), E-E-A-T signals, FAQ blocks, definition blocks, H1/alt/meta hygiene.
+- **Indexing diagnosis.** Google Search Console and Bing Webmaster Tools coverage, URL inspection, and one-shot submissions via `canonry google request-indexing` / `canonry bing request-indexing`.
+- **Schema & content execution.** Patterns for injecting LocalBusiness/FAQPage JSON-LD, writing `llms.txt` with service-area detail, trimming keyphrase lists to high-intent queries, and handling WordPress/Elementor specifics (REST API, Application Passwords, Elementor Custom Code).
+- **Diagnose → prioritize → execute → monitor → report workflow.** Opinionated defaults for new sites (0 citations), regressions on established sites, and county-level targeting — with guardrails like "never fabricate citation data" and "back up `~/.canonry/config.yaml` before editing".
+
+See [`skills/canonry-setup/SKILL.md`](skills/canonry-setup/SKILL.md) plus the reference files under [`skills/canonry-setup/references/`](skills/canonry-setup/references/) (`canonry-cli.md`, `aeo-analysis.md`, `indexing.md`, `wordpress-integration.md`) for the full playbook. Aero loads the same material natively, so anything an external agent can do through the skill, Aero can do from the CLI or dashboard command bar.
 
 ## Deployment
 

package/dist/{chunk-GZF3YIHY.js → chunk-32YTAZBL.js}

@@ -1036,7 +1036,22 @@ var MIGRATIONS = [
   // v42: Per-project auto-extract toggle — when a release sync transitions
   // to ready, projects with this flag get a backlink-extract run enqueued.
   // Stored as INTEGER (0/1) to match SQLite boolean convention.
-  `ALTER TABLE projects ADD COLUMN auto_extract_backlinks INTEGER NOT NULL DEFAULT 0`
+  `ALTER TABLE projects ADD COLUMN auto_extract_backlinks INTEGER NOT NULL DEFAULT 0`,
+  // v43: Backfill bing_url_inspections.in_index using the new crawl-signal
+  // decision tree. Legacy rows were classified with the retired Bing `InIndex`
+  // flag plus a DocumentSize>0 check, which mis-classifies URLs that modern
+  // Bing returns with DocumentSize=0 but a valid LastCrawledDate. Use a
+  // created_at cutoff so rows written by the new code (which applies a live
+  // GetCrawlIssues demotion that can't be replayed offline) are preserved.
+  `UPDATE bing_url_inspections
+   SET in_index = CASE
+     WHEN document_size IS NOT NULL AND document_size > 0 THEN 1
+     WHEN last_crawled_date IS NOT NULL AND http_code IS NOT NULL AND http_code >= 400 THEN 0
+     WHEN last_crawled_date IS NOT NULL THEN 1
+     WHEN discovery_date IS NOT NULL THEN 0
+     ELSE NULL
+   END
+   WHERE created_at < '2026-04-22T00:00:00Z'`
 ];
 function isDuplicateColumnError(err) {
   if (!(err instanceof Error)) return false;

package/dist/{chunk-KGOT5OFT.js → chunk-6UY2PETG.js}

@@ -30,7 +30,7 @@ import {
   runs,
   schedules,
   usageCounters
-} from "./chunk-GZF3YIHY.js";
+} from "./chunk-32YTAZBL.js";
 
 // src/config.ts
 import fs from "fs";
@@ -8148,6 +8148,13 @@ async function getKeywordStats(apiKey, siteUrl) {
   const data = await bingFetch(apiKey, `GetQueryStats?siteUrl=${encodedSite}`);
   return data ?? [];
 }
+async function getCrawlIssues(apiKey, siteUrl) {
+  validateApiKey(apiKey);
+  validateSiteUrl2(siteUrl);
+  const encodedSite = encodeURIComponent(siteUrl);
+  const data = await bingFetch(apiKey, `GetCrawlIssues?siteUrl=${encodedSite}`);
+  return data ?? [];
+}
 
 // ../api-routes/src/bing.ts
 function parseBingDate(value) {
@@ -8163,6 +8170,30 @@ function bingLog(level, action, ctx) {
   const stream = level === "error" ? process.stderr : process.stdout;
   stream.write(JSON.stringify(entry) + "\n");
 }
+var CRAWL_ISSUES_CACHE_TTL_MS = 6e4;
+var crawlIssuesCache = /* @__PURE__ */ new Map();
+function isBlockingIssueType(issueType) {
+  if (!issueType) return true;
+  const trimmed = issueType.trim();
+  if (!trimmed) return true;
+  return trimmed.split(/\s+/).some((flag) => !/^(None|Seo(Issues|Concerns))$/i.test(flag));
+}
+async function loadBlockingCrawlIssues(apiKey, siteUrl, domain) {
+  const now = Date.now();
+  const cached = crawlIssuesCache.get(domain);
+  if (cached && now - cached.fetchedAt < CRAWL_ISSUES_CACHE_TTL_MS) {
+    return cached.blockedUrls;
+  }
+  const issues = await getCrawlIssues(apiKey, siteUrl);
+  const blockedUrls = /* @__PURE__ */ new Set();
+  for (const issue of issues) {
+    if (issue.Url && isBlockingIssueType(issue.IssueType ?? null)) {
+      blockedUrls.add(issue.Url);
+    }
+  }
+  crawlIssuesCache.set(domain, { blockedUrls, fetchedAt: now });
+  return blockedUrls;
+}
 async function bingRoutes(app, opts) {
   function requireConnectionStore() {
     if (opts.bingConnectionStore) return opts.bingConnectionStore;
@@ -8411,22 +8442,38 @@ async function bingRoutes(app, opts) {
         domain: project.canonicalDomain,
         url,
         httpStatus: result.HttpStatus ?? result.HttpCode ?? null,
-        inIndex: result.InIndex ?? null,
         documentSize: result.DocumentSize ?? null,
-        lastCrawledDate: result.LastCrawledDate ?? null
+        lastCrawledDate: result.LastCrawledDate ?? null,
+        discoveryDate: result.DiscoveryDate ?? null
       });
       const now = (/* @__PURE__ */ new Date()).toISOString();
       const id = crypto15.randomUUID();
       const httpCode = result.HttpStatus ?? result.HttpCode ?? null;
-      let derivedInIndex = null;
-      if (result.InIndex != null) {
-        derivedInIndex = result.InIndex;
-      } else if (result.DocumentSize != null && result.DocumentSize > 0) {
-        derivedInIndex = true;
-      }
       const lastCrawledDate = parseBingDate(result.LastCrawledDate);
       const inIndexDate = parseBingDate(result.InIndexDate);
       const discoveryDate = parseBingDate(result.DiscoveryDate);
+      let derivedInIndex = null;
+      if (result.DocumentSize != null && result.DocumentSize > 0) {
+        derivedInIndex = true;
+      } else if (lastCrawledDate != null) {
+        const httpStatus = result.HttpStatus ?? result.HttpCode;
+        derivedInIndex = httpStatus != null && httpStatus >= 400 ? false : true;
+      } else if (discoveryDate != null) {
+        derivedInIndex = false;
+      }
+      if (derivedInIndex === true) {
+        try {
+          const blockedUrls = await loadBlockingCrawlIssues(conn.apiKey, conn.siteUrl, project.canonicalDomain);
+          if (blockedUrls.has(url)) {
+            derivedInIndex = false;
+          }
+        } catch (e) {
+          bingLog("warn", "inspect-url.crawl-issues-lookup-failed", {
+            domain: project.canonicalDomain,
+            error: e instanceof Error ? e.message : String(e)
+          });
+        }
+      }
       app.db.insert(bingUrlInspections).values({
         id,
         projectId: project.id,

package/dist/cli.js

@@ -38,7 +38,7 @@ import {
   showFirstRunNotice,
   trackEvent,
   usageError
-} from "./chunk-KGOT5OFT.js";
+} from "./chunk-6UY2PETG.js";
 import {
   apiKeys,
   competitors,
@@ -48,7 +48,7 @@ import {
   projects,
   querySnapshots,
   runs
-} from "./chunk-GZF3YIHY.js";
+} from "./chunk-32YTAZBL.js";
 
 // src/cli.ts
 import { pathToFileURL } from "url";
@@ -295,7 +295,7 @@ async function backfillAnswerVisibilityCommand(opts) {
   console.log(`  Errors:   ${providerErrors}`);
 }
 async function backfillInsightsCommand(project, opts) {
-  const { IntelligenceService } = await import("./intelligence-service-KM64AW7J.js");
+  const { IntelligenceService } = await import("./intelligence-service-U7YQ4NXV.js");
   const config = loadConfig();
   const db = createClient(config.database);
   migrate(db);
@@ -2181,6 +2181,7 @@ var COMPETITOR_CLI_COMMANDS = [
 ];
 
 // src/commands/google.ts
+var INDEXING_API_SCOPE_NOTICE = "Note: Google's Indexing API officially supports only pages with JobPosting or BroadcastEvent (livestream VideoObject) structured data. For other URL types, submissions are accepted (HTTP 200) but not guaranteed to be prioritized for crawling. For general pages, submit a sitemap and use URL Inspection to monitor status.";
 function getClient6() {
   return createApiClient();
 }
@@ -2630,27 +2631,32 @@ async function googleRequestIndexing(project, opts) {
       details: { command: "google.request-indexing" }
     });
   }
+  if (opts.format !== "json") {
+    console.error(INDEXING_API_SCOPE_NOTICE);
+    console.error();
+  }
   const result = await client.googleRequestIndexing(project, body);
   let indexingConfirmed = false;
+  const lastInspection = /* @__PURE__ */ new Map();
   if (opts.wait && result.results.some((r) => r.status === "success")) {
     const successUrls = result.results.filter((r) => r.status === "success").map((r) => r.url);
-    const timeout = 10 * 60 * 1e3;
+    const timeout = opts.waitTimeoutMs ?? 10 * 60 * 1e3;
+    const pollInterval = opts.waitPollIntervalMs ?? 1e4;
     const start = Date.now();
-    process.stderr.write("Waiting for indexing confirmation");
+    process.stderr.write("Polling URL Inspection for indexed verdict");
     while (Date.now() - start < timeout) {
-      await new Promise((r) => setTimeout(r, 1e4));
+      await new Promise((r) => setTimeout(r, pollInterval));
       process.stderr.write(".");
       let allIndexed = true;
       for (const url of successUrls) {
         try {
           const inspection = await client.gscInspect(project, url);
-          if (inspection.indexingState !== "INDEXING_ALLOWED") {
+          lastInspection.set(url, inspection);
+          if (inspection.verdict !== "PASS") {
             allIndexed = false;
-            break;
           }
         } catch {
           allIndexed = false;
-          break;
         }
       }
       if (allIndexed) {
@@ -2661,13 +2667,23 @@ async function googleRequestIndexing(project, opts) {
     }
     if (!indexingConfirmed) {
       process.stderr.write("\n");
+      const observed = successUrls.map((url) => {
+        const i = lastInspection.get(url);
+        return {
+          url,
+          verdict: i?.verdict ?? null,
+          coverageState: i?.coverageState ?? null,
+          indexingState: i?.indexingState ?? null
+        };
+      });
       throw new CliError({
         code: "GOOGLE_INDEXING_CONFIRMATION_TIMEOUT",
-        message: "Timed out waiting for indexing confirmation. URLs may still be processing.",
-        displayMessage: "Timed out waiting for indexing confirmation. URLs may still be processing.",
+        message: "Timed out waiting for Google to report verdict=PASS. Google typically takes hours to days to index new URLs, so this is expected and does not mean the submission failed. Re-check later with `canonry google gsc inspect <url>`.",
+        displayMessage: "Timed out waiting for Google to report verdict=PASS. Google typically takes hours to days to index new URLs \u2014 this is not a failure. Re-check later with `canonry google gsc inspect <url>`.",
         details: {
           project,
-          urls: successUrls
+          urls: successUrls,
+          lastObserved: observed
         }
       });
     }
@@ -2692,7 +2708,7 @@ async function googleRequestIndexing(project, opts) {
     console.log(`Summary: ${result.summary.succeeded} succeeded, ${result.summary.failed} failed (${result.summary.total} total)`);
   }
   if (indexingConfirmed) {
-    console.log("All requested URLs are now indexed.");
+    console.log("URL Inspection now reports verdict=PASS for the requested URLs (indexed in Google Search).");
   }
 }
 async function googleRefresh(project, format) {

package/dist/index.js

@@ -1,8 +1,8 @@
 import {
   createServer,
   loadConfig
-} from "./chunk-KGOT5OFT.js";
-import "./chunk-GZF3YIHY.js";
+} from "./chunk-6UY2PETG.js";
+import "./chunk-32YTAZBL.js";
 export {
   createServer,
   loadConfig

package/dist/{intelligence-service-KM64AW7J.js → intelligence-service-U7YQ4NXV.js}

@@ -1,6 +1,6 @@
 import {
   IntelligenceService
-} from "./chunk-GZF3YIHY.js";
+} from "./chunk-32YTAZBL.js";
 export {
   IntelligenceService
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ainyc/canonry",
-  "version": "2.4.3",
+  "version": "2.4.6",
   "type": "module",
   "description": "The ultimate open-source AEO monitoring tool - track how answer engines cite your domain",
   "license": "FSL-1.1-ALv2",
@@ -60,16 +60,16 @@
     "@ainyc/canonry-api-routes": "0.0.0",
     "@ainyc/canonry-config": "0.0.0",
     "@ainyc/canonry-contracts": "0.0.0",
-    "@ainyc/canonry-db": "0.0.0",
-    "@ainyc/canonry-integration-bing": "0.0.0",
     "@ainyc/canonry-intelligence": "0.0.0",
-    "@ainyc/canonry-integration-commoncrawl": "0.0.0",
+    "@ainyc/canonry-db": "0.0.0",
     "@ainyc/canonry-integration-google": "0.0.0",
+    "@ainyc/canonry-integration-bing": "0.0.0",
+    "@ainyc/canonry-provider-cdp": "0.0.0",
     "@ainyc/canonry-integration-wordpress": "0.0.0",
     "@ainyc/canonry-provider-claude": "0.0.0",
-    "@ainyc/canonry-provider-cdp": "0.0.0",
-    "@ainyc/canonry-provider-gemini": "0.0.0",
+    "@ainyc/canonry-integration-commoncrawl": "0.0.0",
     "@ainyc/canonry-provider-local": "0.0.0",
+    "@ainyc/canonry-provider-gemini": "0.0.0",
     "@ainyc/canonry-provider-openai": "0.0.0",
     "@ainyc/canonry-provider-perplexity": "0.0.0"
   },