@ainyc/canonry 3.2.1 → 3.2.2

package/dist/{chunk-QJPPK4WW.js → chunk-QA5HRDRS.js}

@@ -62,7 +62,7 @@ import {
   visibilityStateFromAnswerMentioned,
   windowCutoff,
   wordpressEnvSchema
-} from "./chunk-TUXS2Y6C.js";
+} from "./chunk-TBFRAJU2.js";
 import {
   IntelligenceService,
   agentMemory,
@@ -11033,7 +11033,7 @@ async function wordpressRoutes(app, opts) {
 
 // ../api-routes/src/backlinks.ts
 import crypto18 from "crypto";
-import { and as and9, asc as asc2, desc as desc10, eq as eq21, sql as sql6 } from "drizzle-orm";
+import { and as and10, asc as asc2, desc as desc10, eq as eq21, sql as sql6 } from "drizzle-orm";
 
 // ../integration-commoncrawl/src/constants.ts
 import os2 from "os";
@@ -11429,6 +11429,35 @@ function pruneCachedRelease(release, opts = {}) {
   fs5.rmSync(dir, { recursive: true, force: true });
 }
 
+// ../api-routes/src/backlinks-filter.ts
+import { and as and9, ne, notLike } from "drizzle-orm";
+var BACKLINK_FILTER_PATTERNS = [
+  "*.google.com",
+  "*.googleusercontent.com",
+  "*.translate.goog",
+  "*.bing.com",
+  "*.yandex.com",
+  "*.yandex.ru",
+  "*.baidu.com",
+  "*.duckduckgo.com",
+  "*.archive.org"
+];
+function backlinkCrawlerExclusionClause() {
+  const conditions = [];
+  for (const pattern of BACKLINK_FILTER_PATTERNS) {
+    if (pattern.startsWith("*.")) {
+      const suffix = pattern.slice(2);
+      conditions.push(ne(backlinkDomains.linkingDomain, suffix));
+      conditions.push(notLike(backlinkDomains.linkingDomain, `%.${suffix}`));
+    } else {
+      conditions.push(ne(backlinkDomains.linkingDomain, pattern));
+    }
+  }
+  const combined = and9(...conditions);
+  if (!combined) throw new Error("BACKLINK_FILTER_PATTERNS is unexpectedly empty");
+  return combined;
+}
+
 // ../api-routes/src/backlinks.ts
 var BACKLINKS_UNSUPPORTED_MESSAGE = "Backlinks sync and install are only available from a local canonry install. Run `canonry backlinks install` locally to use this feature.";
 var NON_TERMINAL_SYNC_STATUSES = /* @__PURE__ */ new Set([
@@ -11485,9 +11514,47 @@ function mapRunRow(row) {
   };
 }
 function latestSummaryForProject(db, projectId, release) {
-  const condition = release ? and9(eq21(backlinkSummaries.projectId, projectId), eq21(backlinkSummaries.release, release)) : eq21(backlinkSummaries.projectId, projectId);
+  const condition = release ? and10(eq21(backlinkSummaries.projectId, projectId), eq21(backlinkSummaries.release, release)) : eq21(backlinkSummaries.projectId, projectId);
   return db.select().from(backlinkSummaries).where(condition).orderBy(desc10(backlinkSummaries.queriedAt)).limit(1).get();
 }
+function parseExcludeCrawlers(value) {
+  if (!value) return false;
+  const lower = value.toLowerCase();
+  return lower === "1" || lower === "true" || lower === "yes";
+}
+function computeFilteredSummary(db, base) {
+  const baseDomainCondition = and10(
+    eq21(backlinkDomains.projectId, base.projectId),
+    eq21(backlinkDomains.release, base.release)
+  );
+  const filteredCondition = and10(baseDomainCondition, backlinkCrawlerExclusionClause());
+  const unfilteredAgg = db.select({
+    count: sql6`count(*)`,
+    total: sql6`coalesce(sum(${backlinkDomains.numHosts}), 0)`
+  }).from(backlinkDomains).where(baseDomainCondition).get();
+  const filteredAgg = db.select({
+    count: sql6`count(*)`,
+    total: sql6`coalesce(sum(${backlinkDomains.numHosts}), 0)`
+  }).from(backlinkDomains).where(filteredCondition).get();
+  const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc10(backlinkDomains.numHosts)).limit(10).all();
+  const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
+  const totalHosts = Number(filteredAgg?.total ?? 0);
+  const unfilteredLinkingDomains = Number(unfilteredAgg?.count ?? 0);
+  const unfilteredHosts = Number(unfilteredAgg?.total ?? 0);
+  const top10Sum = top10Rows.reduce((sum, r) => sum + r.numHosts, 0);
+  const top10Share = totalHosts > 0 ? top10Sum / totalHosts : 0;
+  return {
+    projectId: base.projectId,
+    release: base.release,
+    targetDomain: base.targetDomain,
+    totalLinkingDomains,
+    totalHosts,
+    top10HostsShare: top10Share.toFixed(6),
+    queriedAt: base.queriedAt,
+    excludedLinkingDomains: Math.max(0, unfilteredLinkingDomains - totalLinkingDomains),
+    excludedHosts: Math.max(0, unfilteredHosts - totalHosts)
+  };
+}
 async function backlinksRoutes(app, opts) {
   app.get("/backlinks/status", async (_request, reply) => {
     if (!opts.getBacklinksStatus) {
@@ -11620,7 +11687,9 @@ async function backlinksRoutes(app, opts) {
     async (request, reply) => {
       const project = resolveProject(app.db, request.params.name);
       const row = latestSummaryForProject(app.db, project.id, request.query.release);
-      return reply.send(row ? mapSummaryRow(row) : null);
+      if (!row) return reply.send(null);
+      const excludeCrawlers = parseExcludeCrawlers(request.query.excludeCrawlers);
+      return reply.send(excludeCrawlers ? computeFilteredSummary(app.db, row) : mapSummaryRow(row));
     }
   );
   app.get("/projects/:name/backlinks/domains", async (request, reply) => {
@@ -11633,17 +11702,23 @@ async function backlinksRoutes(app, opts) {
     }
     const limit = Math.min(Math.max(parseInt(request.query.limit ?? "50", 10) || 50, 1), 500);
     const offset = Math.max(parseInt(request.query.offset ?? "0", 10) || 0, 0);
-    const domainCondition = and9(
+    const excludeCrawlers = parseExcludeCrawlers(request.query.excludeCrawlers);
+    const baseDomainCondition = and10(
       eq21(backlinkDomains.projectId, project.id),
       eq21(backlinkDomains.release, targetRelease)
     );
+    const domainCondition = excludeCrawlers ? and10(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
     const totalRow = app.db.select({ count: sql6`count(*)` }).from(backlinkDomains).where(domainCondition).get();
     const rows = app.db.select({
       linkingDomain: backlinkDomains.linkingDomain,
       numHosts: backlinkDomains.numHosts
     }).from(backlinkDomains).where(domainCondition).orderBy(desc10(backlinkDomains.numHosts)).limit(limit).offset(offset).all();
+    let summary = null;
+    if (summaryRow) {
+      summary = excludeCrawlers ? computeFilteredSummary(app.db, summaryRow) : mapSummaryRow(summaryRow);
+    }
     const response = {
-      summary: summaryRow ? mapSummaryRow(summaryRow) : null,
+      summary,
       total: Number(totalRow?.count ?? 0),
       rows
     };
@@ -14997,7 +15072,7 @@ import crypto19 from "crypto";
 import fs7 from "fs";
 import path9 from "path";
 import os4 from "os";
-import { and as and10, eq as eq22, inArray as inArray5, sql as sql7 } from "drizzle-orm";
+import { and as and11, eq as eq22, inArray as inArray5, sql as sql7 } from "drizzle-orm";
 
 // src/citation-utils.ts
 function domainMatches(domain, canonicalDomain) {
@@ -15281,7 +15356,7 @@ var JobRunner = class {
         throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
       }
       if (existingRun.status === "queued") {
-        this.db.update(runs).set({ status: "running", startedAt: now }).where(and10(eq22(runs.id, runId), eq22(runs.status, "queued"))).run();
+        this.db.update(runs).set({ status: "running", startedAt: now }).where(and11(eq22(runs.id, runId), eq22(runs.status, "queued"))).run();
       }
       this.throwIfRunCancelled(runId);
       const project = this.db.select().from(projects).where(eq22(projects.id, projectId)).get();
@@ -15584,7 +15659,7 @@ function getCurrentUsageDay() {
 
 // src/gsc-sync.ts
 import crypto20 from "crypto";
-import { eq as eq23, and as and11, sql as sql8 } from "drizzle-orm";
+import { eq as eq23, and as and12, sql as sql8 } from "drizzle-orm";
 var log2 = createLogger("GscSync");
 function formatDate2(d) {
   return d.toISOString().split("T")[0];
@@ -15636,7 +15711,7 @@ async function executeGscSync(db, runId, projectId, opts) {
     });
     log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
     db.delete(gscSearchData).where(
-      and11(
+      and12(
         eq23(gscSearchData.projectId, projectId),
         sql8`${gscSearchData.date} >= ${startDate}`,
         sql8`${gscSearchData.date} <= ${endDate}`
@@ -15725,7 +15800,7 @@ async function executeGscSync(db, runId, projectId, opts) {
       }
     }
     const snapshotDate = formatDate2(/* @__PURE__ */ new Date());
-    db.delete(gscCoverageSnapshots).where(and11(eq23(gscCoverageSnapshots.projectId, projectId), eq23(gscCoverageSnapshots.date, snapshotDate))).run();
+    db.delete(gscCoverageSnapshots).where(and12(eq23(gscCoverageSnapshots.projectId, projectId), eq23(gscCoverageSnapshots.date, snapshotDate))).run();
     db.insert(gscCoverageSnapshots).values({
       id: crypto20.randomUUID(),
       projectId,
@@ -15748,7 +15823,7 @@ async function executeGscSync(db, runId, projectId, opts) {
 
 // src/gsc-inspect-sitemap.ts
 import crypto21 from "crypto";
-import { eq as eq24, and as and12 } from "drizzle-orm";
+import { eq as eq24, and as and13 } from "drizzle-orm";
 
 // src/sitemap-parser.ts
 var log3 = createLogger("SitemapParser");
@@ -15964,7 +16039,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
       }
     }
     const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
-    db.delete(gscCoverageSnapshots).where(and12(eq24(gscCoverageSnapshots.projectId, projectId), eq24(gscCoverageSnapshots.date, snapshotDate))).run();
+    db.delete(gscCoverageSnapshots).where(and13(eq24(gscCoverageSnapshots.projectId, projectId), eq24(gscCoverageSnapshots.date, snapshotDate))).run();
     db.insert(gscCoverageSnapshots).values({
       id: crypto21.randomUUID(),
       projectId,
@@ -16175,7 +16250,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
 // src/commoncrawl-sync.ts
 import crypto23 from "crypto";
 import path10 from "path";
-import { and as and13, eq as eq26, sql as sql9 } from "drizzle-orm";
+import { and as and14, eq as eq26, sql as sql9 } from "drizzle-orm";
 var log6 = createLogger("CommonCrawlSync");
 var INSERT_CHUNK_SIZE = 1e4;
 function defaultDeps() {
@@ -16366,7 +16441,7 @@ function computeSummary(rows) {
 // src/backlink-extract.ts
 import crypto24 from "crypto";
 import fs8 from "fs";
-import { and as and14, desc as desc12, eq as eq27 } from "drizzle-orm";
+import { and as and15, desc as desc12, eq as eq27 } from "drizzle-orm";
 var log7 = createLogger("BacklinkExtract");
 function defaultDeps2() {
   return {
@@ -16412,7 +16487,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
     const targetDomain = project.canonicalDomain;
     db.transaction((tx) => {
       tx.delete(backlinkDomains).where(
-        and14(eq27(backlinkDomains.projectId, projectId), eq27(backlinkDomains.release, release))
+        and15(eq27(backlinkDomains.projectId, projectId), eq27(backlinkDomains.release, release))
       ).run();
       if (rows.length > 0) {
         const values = rows.map((r) => ({
@@ -16668,7 +16743,7 @@ var Scheduler = class {
 };
 
 // src/notifier.ts
-import { eq as eq29, desc as desc13, and as and15, or as or3 } from "drizzle-orm";
+import { eq as eq29, desc as desc13, and as and16, or as or3 } from "drizzle-orm";
 import crypto25 from "crypto";
 var log9 = createLogger("Notifier");
 var Notifier = class {
@@ -16774,7 +16849,7 @@ var Notifier = class {
   }
   computeTransitions(runId, projectId) {
     const recentRuns = this.db.select().from(runs).where(
-      and15(
+      and16(
         eq29(runs.projectId, projectId),
         or3(eq29(runs.status, "completed"), eq29(runs.status, "partial"))
       )
@@ -17260,7 +17335,7 @@ function resolveSessionProviderAndModel(config, opts) {
 
 // src/agent/memory-store.ts
 import crypto26 from "crypto";
-import { and as and16, desc as desc14, eq as eq30, like as like2, sql as sql10 } from "drizzle-orm";
+import { and as and17, desc as desc14, eq as eq30, like as like2, sql as sql10 } from "drizzle-orm";
 var COMPACTION_KEY_PREFIX = "compaction:";
 var COMPACTION_NOTES_PER_SESSION = 3;
 function rowToDto(row) {
@@ -17305,12 +17380,12 @@ function upsertMemoryEntry(db, args) {
       updatedAt: now
     }
   }).run();
-  const row = db.select().from(agentMemory).where(and16(eq30(agentMemory.projectId, args.projectId), eq30(agentMemory.key, args.key))).get();
+  const row = db.select().from(agentMemory).where(and17(eq30(agentMemory.projectId, args.projectId), eq30(agentMemory.key, args.key))).get();
   if (!row) throw new Error("memory upsert produced no row");
   return rowToDto(row);
 }
 function deleteMemoryEntry(db, projectId, key) {
-  const result = db.delete(agentMemory).where(and16(eq30(agentMemory.projectId, projectId), eq30(agentMemory.key, key))).run();
+  const result = db.delete(agentMemory).where(and17(eq30(agentMemory.projectId, projectId), eq30(agentMemory.key, key))).run();
   const changes = result.changes ?? 0;
   return changes > 0;
 }
@@ -17339,7 +17414,7 @@ function writeCompactionNote(db, args) {
     }).run();
     const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
     const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
-      and16(
+      and17(
         eq30(agentMemory.projectId, args.projectId),
         like2(agentMemory.key, `${sessionPrefix}%`)
       )
@@ -17348,7 +17423,7 @@ function writeCompactionNote(db, args) {
     if (stale.length > 0) {
       tx.delete(agentMemory).where(sql10`${agentMemory.id} IN (${sql10.join(stale.map((s) => sql10`${s}`), sql10`, `)})`).run();
     }
-    const row = tx.select().from(agentMemory).where(and16(eq30(agentMemory.projectId, args.projectId), eq30(agentMemory.key, key))).get();
+    const row = tx.select().from(agentMemory).where(and17(eq30(agentMemory.projectId, args.projectId), eq30(agentMemory.key, key))).get();
     if (row) inserted = rowToDto(row);
   });
   if (!inserted) throw new Error("compaction note write produced no row");

package/dist/{chunk-TUXS2Y6C.js → chunk-TBFRAJU2.js}

@@ -1577,7 +1577,12 @@ var backlinkSummaryDtoSchema = z14.object({
   totalLinkingDomains: z14.number().int(),
   totalHosts: z14.number().int(),
   top10HostsShare: z14.string(),
-  queriedAt: z14.string()
+  queriedAt: z14.string(),
+  // Populated when the response is filtered (e.g. ?excludeCrawlers=1).
+  // Counts the rows omitted from totalLinkingDomains/totalHosts so callers
+  // can show "N hidden" hints without re-deriving them.
+  excludedLinkingDomains: z14.number().int().optional(),
+  excludedHosts: z14.number().int().optional()
 });
 var backlinkListResponseSchema = z14.object({
   summary: backlinkSummaryDtoSchema.nullable(),
@@ -2650,15 +2655,19 @@ var ApiClient = class {
   async backlinksExtract(project, release) {
     return this.request("POST", `/projects/${encodeURIComponent(project)}/backlinks/extract`, release ? { release } : {});
   }
-  async backlinksSummary(project, release) {
-    const qs = release ? `?release=${encodeURIComponent(release)}` : "";
-    return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/summary${qs}`);
+  async backlinksSummary(project, opts = {}) {
+    const qs = new URLSearchParams();
+    if (opts.release) qs.set("release", opts.release);
+    if (opts.excludeCrawlers) qs.set("excludeCrawlers", "1");
+    const suffix = qs.toString() ? `?${qs.toString()}` : "";
+    return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/summary${suffix}`);
   }
   async backlinksDomains(project, opts = {}) {
     const qs = new URLSearchParams();
     if (opts.limit !== void 0) qs.set("limit", String(opts.limit));
     if (opts.offset !== void 0) qs.set("offset", String(opts.offset));
     if (opts.release) qs.set("release", opts.release);
+    if (opts.excludeCrawlers) qs.set("excludeCrawlers", "1");
     const suffix = qs.toString() ? `?${qs.toString()}` : "";
     return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/domains${suffix}`);
   }

package/dist/cli.js

@@ -17,7 +17,7 @@ import {
   setGoogleAuthConfig,
   showFirstRunNotice,
   trackEvent
-} from "./chunk-QJPPK4WW.js";
+} from "./chunk-QA5HRDRS.js";
 import {
   CcReleaseSyncStatuses,
   CheckScopes,
@@ -45,7 +45,7 @@ import {
   saveConfig,
   saveConfigPatch,
   usageError
-} from "./chunk-TUXS2Y6C.js";
+} from "./chunk-TBFRAJU2.js";
 import {
   apiKeys,
   competitors,
@@ -728,6 +728,9 @@ function formatSummaryAndDomains(project, response) {
   lines.push(`Linking domains: ${s.totalLinkingDomains}`);
   lines.push(`Total hosts:     ${s.totalHosts}`);
   lines.push(`Top-10 share:    ${s.top10HostsShare}`);
+  if (s.excludedLinkingDomains !== void 0 && s.excludedLinkingDomains > 0) {
+    lines.push(`Excluded:        ${s.excludedLinkingDomains} crawler/proxy domains (${s.excludedHosts ?? 0} hosts)`);
+  }
   if (response.rows.length > 0) {
     lines.push("");
     lines.push(`Top ${response.rows.length} linking domains (of ${response.total}):`);
@@ -861,7 +864,8 @@ async function backlinksList(opts) {
   const client = getClient();
   const response = await client.backlinksDomains(opts.project, {
     limit: opts.limit ?? 50,
-    release: opts.release
+    release: opts.release,
+    excludeCrawlers: opts.excludeCrawlers
   });
   if (opts.format === "json") {
     printJson(response);
@@ -943,16 +947,17 @@ var BACKLINKS_CLI_COMMANDS = [
   },
   {
     path: ["backlinks", "list"],
-    usage: "canonry backlinks list <project> [--limit <n>] [--release <id>] [--format json]",
+    usage: "canonry backlinks list <project> [--limit <n>] [--release <id>] [--exclude-crawlers] [--format json]",
     options: {
       limit: stringOption(),
-      release: stringOption()
+      release: stringOption(),
+      "exclude-crawlers": { type: "boolean" }
     },
     run: async (input) => {
       const project = requireProject(
         input,
         "backlinks list",
-        "canonry backlinks list <project> [--limit <n>] [--release <id>]"
+        "canonry backlinks list <project> [--limit <n>] [--release <id>] [--exclude-crawlers]"
       );
       const limit = parseIntegerOption(input, "limit", {
         message: "--limit must be an integer",
@@ -963,6 +968,7 @@ var BACKLINKS_CLI_COMMANDS = [
         project,
         limit,
         release: getString(input.values, "release"),
+        excludeCrawlers: getBoolean(input.values, "exclude-crawlers"),
         format: input.format
       });
     }

package/dist/index.js

@@ -1,9 +1,9 @@
 import {
   createServer
-} from "./chunk-QJPPK4WW.js";
+} from "./chunk-QA5HRDRS.js";
 import {
   loadConfig
-} from "./chunk-TUXS2Y6C.js";
+} from "./chunk-TBFRAJU2.js";
 import "./chunk-UQH5SKM2.js";
 import "./chunk-MLKGABMK.js";
 export {

package/dist/mcp.js

@@ -2,7 +2,7 @@ import {
   CliError,
   canonryMcpTools,
   createApiClient
-} from "./chunk-TUXS2Y6C.js";
+} from "./chunk-TBFRAJU2.js";
 import "./chunk-MLKGABMK.js";
 
 // src/mcp/cli.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ainyc/canonry",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "type": "module",
   "description": "Agent-first open-source AEO operating platform - track how answer engines cite your domain",
   "license": "FSL-1.1-ALv2",
@@ -59,21 +59,21 @@
     "@types/node-cron": "^3.0.11",
     "tsup": "^8.5.1",
     "tsx": "^4.19.0",
-    "@ainyc/canonry-config": "0.0.0",
     "@ainyc/canonry-api-routes": "0.0.0",
     "@ainyc/canonry-db": "0.0.0",
+    "@ainyc/canonry-config": "0.0.0",
     "@ainyc/canonry-intelligence": "0.0.0",
-    "@ainyc/canonry-integration-google": "0.0.0",
-    "@ainyc/canonry-integration-wordpress": "0.0.0",
     "@ainyc/canonry-contracts": "0.0.0",
-    "@ainyc/canonry-integration-commoncrawl": "0.0.0",
-    "@ainyc/canonry-provider-cdp": "0.0.0",
     "@ainyc/canonry-integration-bing": "0.0.0",
+    "@ainyc/canonry-integration-commoncrawl": "0.0.0",
+    "@ainyc/canonry-integration-google": "0.0.0",
+    "@ainyc/canonry-integration-wordpress": "0.0.0",
     "@ainyc/canonry-provider-claude": "0.0.0",
     "@ainyc/canonry-provider-gemini": "0.0.0",
     "@ainyc/canonry-provider-local": "0.0.0",
-    "@ainyc/canonry-provider-perplexity": "0.0.0",
-    "@ainyc/canonry-provider-openai": "0.0.0"
+    "@ainyc/canonry-provider-openai": "0.0.0",
+    "@ainyc/canonry-provider-cdp": "0.0.0",
+    "@ainyc/canonry-provider-perplexity": "0.0.0"
   },
   "scripts": {
     "build": "tsx scripts/copy-agent-assets.ts && tsup && tsx build-web.ts",