@ainyc/canonry 2.4.3 → 2.4.5

package/dist/{chunk-GZF3YIHY.js → chunk-32YTAZBL.js}

@@ -1036,7 +1036,22 @@ var MIGRATIONS = [
   // v42: Per-project auto-extract toggle — when a release sync transitions
   // to ready, projects with this flag get a backlink-extract run enqueued.
   // Stored as INTEGER (0/1) to match SQLite boolean convention.
-  `ALTER TABLE projects ADD COLUMN auto_extract_backlinks INTEGER NOT NULL DEFAULT 0`
+  `ALTER TABLE projects ADD COLUMN auto_extract_backlinks INTEGER NOT NULL DEFAULT 0`,
+  // v43: Backfill bing_url_inspections.in_index using the new crawl-signal
+  // decision tree. Legacy rows were classified with the retired Bing `InIndex`
+  // flag plus a DocumentSize>0 check, which mis-classifies URLs that modern
+  // Bing returns with DocumentSize=0 but a valid LastCrawledDate. Use a
+  // created_at cutoff so rows written by the new code (which applies a live
+  // GetCrawlIssues demotion that can't be replayed offline) are preserved.
+  `UPDATE bing_url_inspections
+   SET in_index = CASE
+     WHEN document_size IS NOT NULL AND document_size > 0 THEN 1
+     WHEN last_crawled_date IS NOT NULL AND http_code IS NOT NULL AND http_code >= 400 THEN 0
+     WHEN last_crawled_date IS NOT NULL THEN 1
+     WHEN discovery_date IS NOT NULL THEN 0
+     ELSE NULL
+   END
+   WHERE created_at < '2026-04-22T00:00:00Z'`
 ];
 function isDuplicateColumnError(err) {
   if (!(err instanceof Error)) return false;

package/dist/{chunk-KGOT5OFT.js → chunk-6UY2PETG.js}

@@ -30,7 +30,7 @@ import {
   runs,
   schedules,
   usageCounters
-} from "./chunk-GZF3YIHY.js";
+} from "./chunk-32YTAZBL.js";
 
 // src/config.ts
 import fs from "fs";
@@ -8148,6 +8148,13 @@ async function getKeywordStats(apiKey, siteUrl) {
   const data = await bingFetch(apiKey, `GetQueryStats?siteUrl=${encodedSite}`);
   return data ?? [];
 }
+async function getCrawlIssues(apiKey, siteUrl) {
+  validateApiKey(apiKey);
+  validateSiteUrl2(siteUrl);
+  const encodedSite = encodeURIComponent(siteUrl);
+  const data = await bingFetch(apiKey, `GetCrawlIssues?siteUrl=${encodedSite}`);
+  return data ?? [];
+}
 
 // ../api-routes/src/bing.ts
 function parseBingDate(value) {
@@ -8163,6 +8170,30 @@ function bingLog(level, action, ctx) {
   const stream = level === "error" ? process.stderr : process.stdout;
   stream.write(JSON.stringify(entry) + "\n");
 }
+var CRAWL_ISSUES_CACHE_TTL_MS = 6e4;
+var crawlIssuesCache = /* @__PURE__ */ new Map();
+function isBlockingIssueType(issueType) {
+  if (!issueType) return true;
+  const trimmed = issueType.trim();
+  if (!trimmed) return true;
+  return trimmed.split(/\s+/).some((flag) => !/^(None|Seo(Issues|Concerns))$/i.test(flag));
+}
+async function loadBlockingCrawlIssues(apiKey, siteUrl, domain) {
+  const now = Date.now();
+  const cached = crawlIssuesCache.get(domain);
+  if (cached && now - cached.fetchedAt < CRAWL_ISSUES_CACHE_TTL_MS) {
+    return cached.blockedUrls;
+  }
+  const issues = await getCrawlIssues(apiKey, siteUrl);
+  const blockedUrls = /* @__PURE__ */ new Set();
+  for (const issue of issues) {
+    if (issue.Url && isBlockingIssueType(issue.IssueType ?? null)) {
+      blockedUrls.add(issue.Url);
+    }
+  }
+  crawlIssuesCache.set(domain, { blockedUrls, fetchedAt: now });
+  return blockedUrls;
+}
 async function bingRoutes(app, opts) {
   function requireConnectionStore() {
     if (opts.bingConnectionStore) return opts.bingConnectionStore;
@@ -8411,22 +8442,38 @@ async function bingRoutes(app, opts) {
         domain: project.canonicalDomain,
         url,
         httpStatus: result.HttpStatus ?? result.HttpCode ?? null,
-        inIndex: result.InIndex ?? null,
         documentSize: result.DocumentSize ?? null,
-        lastCrawledDate: result.LastCrawledDate ?? null
+        lastCrawledDate: result.LastCrawledDate ?? null,
+        discoveryDate: result.DiscoveryDate ?? null
       });
       const now = (/* @__PURE__ */ new Date()).toISOString();
       const id = crypto15.randomUUID();
       const httpCode = result.HttpStatus ?? result.HttpCode ?? null;
-      let derivedInIndex = null;
-      if (result.InIndex != null) {
-        derivedInIndex = result.InIndex;
-      } else if (result.DocumentSize != null && result.DocumentSize > 0) {
-        derivedInIndex = true;
-      }
       const lastCrawledDate = parseBingDate(result.LastCrawledDate);
       const inIndexDate = parseBingDate(result.InIndexDate);
       const discoveryDate = parseBingDate(result.DiscoveryDate);
+      let derivedInIndex = null;
+      if (result.DocumentSize != null && result.DocumentSize > 0) {
+        derivedInIndex = true;
+      } else if (lastCrawledDate != null) {
+        const httpStatus = result.HttpStatus ?? result.HttpCode;
+        derivedInIndex = httpStatus != null && httpStatus >= 400 ? false : true;
+      } else if (discoveryDate != null) {
+        derivedInIndex = false;
+      }
+      if (derivedInIndex === true) {
+        try {
+          const blockedUrls = await loadBlockingCrawlIssues(conn.apiKey, conn.siteUrl, project.canonicalDomain);
+          if (blockedUrls.has(url)) {
+            derivedInIndex = false;
+          }
+        } catch (e) {
+          bingLog("warn", "inspect-url.crawl-issues-lookup-failed", {
+            domain: project.canonicalDomain,
+            error: e instanceof Error ? e.message : String(e)
+          });
+        }
+      }
       app.db.insert(bingUrlInspections).values({
         id,
         projectId: project.id,

package/dist/cli.js

@@ -38,7 +38,7 @@ import {
   showFirstRunNotice,
   trackEvent,
   usageError
-} from "./chunk-KGOT5OFT.js";
+} from "./chunk-6UY2PETG.js";
 import {
   apiKeys,
   competitors,
@@ -48,7 +48,7 @@ import {
   projects,
   querySnapshots,
   runs
-} from "./chunk-GZF3YIHY.js";
+} from "./chunk-32YTAZBL.js";
 
 // src/cli.ts
 import { pathToFileURL } from "url";
@@ -295,7 +295,7 @@ async function backfillAnswerVisibilityCommand(opts) {
   console.log(`  Errors:   ${providerErrors}`);
 }
 async function backfillInsightsCommand(project, opts) {
-  const { IntelligenceService } = await import("./intelligence-service-KM64AW7J.js");
+  const { IntelligenceService } = await import("./intelligence-service-U7YQ4NXV.js");
   const config = loadConfig();
   const db = createClient(config.database);
   migrate(db);

package/dist/index.js

@@ -1,8 +1,8 @@
 import {
   createServer,
   loadConfig
-} from "./chunk-KGOT5OFT.js";
-import "./chunk-GZF3YIHY.js";
+} from "./chunk-6UY2PETG.js";
+import "./chunk-32YTAZBL.js";
 export {
   createServer,
   loadConfig

package/dist/{intelligence-service-KM64AW7J.js → intelligence-service-U7YQ4NXV.js}

@@ -1,6 +1,6 @@
 import {
   IntelligenceService
-} from "./chunk-GZF3YIHY.js";
+} from "./chunk-32YTAZBL.js";
 export {
   IntelligenceService
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ainyc/canonry",
-  "version": "2.4.3",
+  "version": "2.4.5",
   "type": "module",
   "description": "The ultimate open-source AEO monitoring tool - track how answer engines cite your domain",
   "license": "FSL-1.1-ALv2",
@@ -57,21 +57,21 @@
     "@types/node-cron": "^3.0.11",
     "tsup": "^8.5.1",
     "tsx": "^4.19.0",
+    "@ainyc/canonry-db": "0.0.0",
     "@ainyc/canonry-api-routes": "0.0.0",
-    "@ainyc/canonry-config": "0.0.0",
+    "@ainyc/canonry-intelligence": "0.0.0",
     "@ainyc/canonry-contracts": "0.0.0",
-    "@ainyc/canonry-db": "0.0.0",
     "@ainyc/canonry-integration-bing": "0.0.0",
-    "@ainyc/canonry-intelligence": "0.0.0",
+    "@ainyc/canonry-config": "0.0.0",
     "@ainyc/canonry-integration-commoncrawl": "0.0.0",
     "@ainyc/canonry-integration-google": "0.0.0",
     "@ainyc/canonry-integration-wordpress": "0.0.0",
-    "@ainyc/canonry-provider-claude": "0.0.0",
-    "@ainyc/canonry-provider-cdp": "0.0.0",
     "@ainyc/canonry-provider-gemini": "0.0.0",
     "@ainyc/canonry-provider-local": "0.0.0",
+    "@ainyc/canonry-provider-cdp": "0.0.0",
     "@ainyc/canonry-provider-openai": "0.0.0",
-    "@ainyc/canonry-provider-perplexity": "0.0.0"
+    "@ainyc/canonry-provider-perplexity": "0.0.0",
+    "@ainyc/canonry-provider-claude": "0.0.0"
   },
   "scripts": {
     "build": "tsx scripts/copy-agent-assets.ts && tsup && tsx build-web.ts",