@canonry/canonry 4.125.0 → 4.126.0

package/assets/agent-workspace/skills/canonry/references/canonry-cli.md

@@ -388,6 +388,11 @@ cnry google discover-sitemaps <project> --wait         # auto-discover and inspe
 
 cnry google sync <project>                             # sync GSC data
 cnry google sync <project> --days 30 --full --wait     # full sync with wait
+# `--full` re-fetches 480 days (GSC's 16-month retention ceiling) and is also the
+# BACKFILL path: it repopulates the accurate per-query totals (`dimensions:
+# ['date','query']`, no `page` fan-out) and the property daily totals for the
+# whole retained window, not just recent days. Run it once per project after
+# upgrading to pick up accurate history; a normal sync only covers its own window.
 
 cnry google coverage <project>                         # index coverage summary
 cnry google refresh <project>                         # force-fetch fresh GSC coverage data

package/dist/{chunk-YPRUBNUK.js → chunk-FCGRCCX6.js}

@@ -396,6 +396,7 @@ __export(schema_exports, {
   googleConnections: () => googleConnections,
   gscCoverageSnapshots: () => gscCoverageSnapshots,
   gscDailyTotals: () => gscDailyTotals,
+  gscQueryDailyTotals: () => gscQueryDailyTotals,
   gscSearchData: () => gscSearchData,
   gscUrlInspections: () => gscUrlInspections,
   healthSnapshots: () => healthSnapshots,
@@ -656,6 +657,27 @@ var gscDailyTotals = sqliteTable("gsc_daily_totals", {
   uniqueIndex("idx_gsc_daily_totals_project_date").on(table.projectId, table.date),
   index("idx_gsc_daily_totals_project").on(table.projectId)
 ]);
+var gscQueryDailyTotals = sqliteTable("gsc_query_daily_totals", {
+  id: text("id").primaryKey(),
+  projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
+  date: text("date").notNull(),
+  query: text("query").notNull(),
+  clicks: integer("clicks").notNull().default(0),
+  impressions: integer("impressions").notNull().default(0),
+  /**
+   * Stored as a string like `gscSearchData.position` (parsed on read). CTR is
+   * derived (clicks / impressions) and intentionally not stored.
+   */
+  position: text("position").notNull().default("0"),
+  syncedAt: text("synced_at").notNull(),
+  syncRunId: text("sync_run_id").references(() => runs.id, { onDelete: "cascade" }),
+  createdAt: text("created_at").notNull()
+}, (table) => [
+  uniqueIndex("idx_gsc_query_daily_totals_project_date_query").on(table.projectId, table.date, table.query),
+  index("idx_gsc_query_daily_totals_project_date").on(table.projectId, table.date),
+  index("idx_gsc_query_daily_totals_query").on(table.query),
+  index("idx_gsc_query_daily_totals_run").on(table.syncRunId)
+]);
 var gscUrlInspections = sqliteTable("gsc_url_inspections", {
   id: text("id").primaryKey(),
   projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
@@ -3971,6 +3993,34 @@ var MIGRATION_VERSIONS = [
       `CREATE INDEX IF NOT EXISTS idx_ga_daily_totals_project ON ga_daily_totals(project_id)`,
       `CREATE INDEX IF NOT EXISTS idx_ga_daily_totals_run ON ga_daily_totals(sync_run_id)`
     ]
+  },
+  {
+    // Per-query daily totals fetched WITHOUT the `page` dimension. Summing
+    // `gsc_search_data` by query multiplies impressions by how many of the
+    // site's pages ranked on the same SERP — ~0% for single-page queries but
+    // ~500% for brand+category terms, which reorders a top-queries table.
+    // Dropping `page` makes Google do the dedup. Per-query counterpart to
+    // `gsc_daily_totals`.
+    version: 107,
+    name: "gsc-query-daily-totals",
+    statements: [
+      `CREATE TABLE IF NOT EXISTS gsc_query_daily_totals (
+        id           TEXT PRIMARY KEY,
+        project_id   TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+        date         TEXT NOT NULL,
+        query        TEXT NOT NULL,
+        clicks       INTEGER NOT NULL DEFAULT 0,
+        impressions  INTEGER NOT NULL DEFAULT 0,
+        position     TEXT NOT NULL DEFAULT '0',
+        synced_at    TEXT NOT NULL,
+        sync_run_id  TEXT REFERENCES runs(id) ON DELETE CASCADE,
+        created_at   TEXT NOT NULL
+      )`,
+      `CREATE UNIQUE INDEX IF NOT EXISTS idx_gsc_query_daily_totals_project_date_query ON gsc_query_daily_totals(project_id, date, query)`,
+      `CREATE INDEX IF NOT EXISTS idx_gsc_query_daily_totals_project_date ON gsc_query_daily_totals(project_id, date)`,
+      `CREATE INDEX IF NOT EXISTS idx_gsc_query_daily_totals_query ON gsc_query_daily_totals(query)`,
+      `CREATE INDEX IF NOT EXISTS idx_gsc_query_daily_totals_run ON gsc_query_daily_totals(sync_run_id)`
+    ]
   }
 ];
 function rebuildBacklinkTableWithSource(tx, table) {
@@ -9584,14 +9634,24 @@ function computeQueryChanges(projectQueries, cutoff) {
     label: `+${count2} kp`
   }));
 }
+function pooledRate(buckets, rateKey) {
+  const countKey = rateKey === "citationRate" ? "cited" : "mentionedCount";
+  let numerator = 0;
+  let denominator = 0;
+  for (const bucket of buckets) {
+    numerator += bucket[countKey];
+    denominator += bucket.total;
+  }
+  return denominator > 0 ? numerator / denominator : 0;
+}
 function computeTrend(buckets, rateKey) {
   const nonEmpty = buckets.filter((b) => b.total > 0);
   if (nonEmpty.length < 2) return "stable";
   const mid = Math.floor(nonEmpty.length / 2);
   const firstHalf = nonEmpty.slice(0, mid);
   const secondHalf = nonEmpty.slice(mid);
-  const avgFirst = firstHalf.reduce((s, b) => s + b[rateKey], 0) / firstHalf.length;
-  const avgSecond = secondHalf.reduce((s, b) => s + b[rateKey], 0) / secondHalf.length;
+  const avgFirst = pooledRate(firstHalf, rateKey);
+  const avgSecond = pooledRate(secondHalf, rateKey);
   const diff = avgSecond - avgFirst;
   if (diff > 0.05) return "improving";
   if (diff < -0.05) return "declining";
@@ -10048,7 +10108,12 @@ function buildGaTrafficByPage(db, projectId) {
   return map;
 }
 function sumAiReferralSessions(db, projectId) {
-  const rows = db.select({ sessions: gaAiReferrals.sessions }).from(gaAiReferrals).where(eq13(gaAiReferrals.projectId, projectId)).all();
+  const rows = db.select({ sessions: gaAiReferrals.sessions }).from(gaAiReferrals).where(
+    and9(
+      eq13(gaAiReferrals.projectId, projectId),
+      eq13(gaAiReferrals.sourceDimension, "session")
+    )
+  ).all();
   return rows.reduce((acc, r) => acc + (r.sessions ?? 0), 0);
 }
 function buildCandidateQueries(opts) {
@@ -10636,6 +10701,64 @@ function mergeGscDailyTotalsWithFallback(propertyTotals, dimensionedFallback) {
   for (const row of propertyTotals) byDate.set(row.date, row);
   return [...byDate.values()].sort((a, b) => a.date.localeCompare(b.date));
 }
+function readGscQueryDailyRows(db, projectId, startDate, endDate) {
+  const rows = db.select({
+    date: gscQueryDailyTotals.date,
+    query: gscQueryDailyTotals.query,
+    clicks: gscQueryDailyTotals.clicks,
+    impressions: gscQueryDailyTotals.impressions,
+    position: gscQueryDailyTotals.position
+  }).from(gscQueryDailyTotals).where(
+    and11(
+      eq15(gscQueryDailyTotals.projectId, projectId),
+      sql6`${gscQueryDailyTotals.date} >= ${startDate}`,
+      sql6`${gscQueryDailyTotals.date} <= ${endDate}`
+    )
+  ).all();
+  return rows.map((r) => {
+    const position = Number(r.position);
+    return {
+      date: r.date,
+      query: r.query,
+      clicks: r.clicks,
+      impressions: r.impressions,
+      position: Number.isFinite(position) ? position : 0
+    };
+  });
+}
+function mergeGscQueryTotalsWithFallback(accurateDays, fallbackDays) {
+  const dayKey = (r) => `${r.date}\0${r.query}`;
+  const byDay = /* @__PURE__ */ new Map();
+  for (const row of fallbackDays) byDay.set(dayKey(row), { row, accurate: false });
+  for (const row of accurateDays) byDay.set(dayKey(row), { row, accurate: true });
+  const byQuery = /* @__PURE__ */ new Map();
+  for (const { row, accurate } of byDay.values()) {
+    const acc = byQuery.get(row.query) ?? {
+      clicks: 0,
+      impressions: 0,
+      weightedPositionSum: 0,
+      positionSum: 0,
+      positionDays: 0,
+      sawAccurate: false,
+      sawLegacy: false
+    };
+    acc.clicks += row.clicks;
+    acc.impressions += row.impressions;
+    acc.weightedPositionSum += row.position * row.impressions;
+    acc.positionSum += row.position;
+    acc.positionDays += 1;
+    if (accurate) acc.sawAccurate = true;
+    else acc.sawLegacy = true;
+    byQuery.set(row.query, acc);
+  }
+  return [...byQuery.entries()].map(([query, acc]) => ({
+    query,
+    clicks: acc.clicks,
+    impressions: acc.impressions,
+    position: acc.impressions > 0 ? acc.weightedPositionSum / acc.impressions : acc.positionDays > 0 ? acc.positionSum / acc.positionDays : 0,
+    source: acc.sawAccurate && acc.sawLegacy ? "mixed" : acc.sawAccurate ? "google" : "page-summed"
+  }));
+}
 
 // ../api-routes/src/report-renderer.ts
 var COLORS = {
@@ -12969,15 +13092,16 @@ function buildGscSection(db, projectId, projectBrandNames, canonicalDomain, trac
   const dailyTotals = allDailyTotals.filter((r) => r.date >= startDate && r.date <= maxDate);
   if (rows.length === 0 && dailyTotals.length === 0) return null;
   let dimensionedClicks = 0;
-  const queryAgg = /* @__PURE__ */ new Map();
+  const queryDayAgg = /* @__PURE__ */ new Map();
   const dimensionedTrendAgg = /* @__PURE__ */ new Map();
   for (const r of rows) {
     dimensionedClicks += r.clicks;
-    const q = queryAgg.get(r.query) ?? { clicks: 0, impressions: 0, weightedPositionSum: 0 };
+    const dayKey = `${r.date} ${r.query}`;
+    const q = queryDayAgg.get(dayKey) ?? { date: r.date, query: r.query, clicks: 0, impressions: 0, weightedPositionSum: 0 };
     q.clicks += r.clicks;
     q.impressions += r.impressions;
     q.weightedPositionSum += safeNum(r.position) * r.impressions;
-    queryAgg.set(r.query, q);
+    queryDayAgg.set(dayKey, q);
     const t = dimensionedTrendAgg.get(r.date) ?? { clicks: 0, impressions: 0, weightedPositionSum: 0 };
     t.clicks += r.clicks;
     t.impressions += r.impressions;
@@ -13004,17 +13128,27 @@ function buildGscSection(db, projectId, projectBrandNames, canonicalDomain, trac
   const trend = dailySeries.map((d) => ({ date: d.date, clicks: d.clicks, impressions: d.impressions }));
   const ctr = totalImpressions > 0 ? totalClicks / totalImpressions : 0;
   const avgPosition = totalImpressions > 0 ? weightedPositionSum / totalImpressions : 0;
-  const topQueries = [...queryAgg.entries()].map(([query, agg]) => ({
-    query,
+  const queryTotals = mergeGscQueryTotalsWithFallback(
+    readGscQueryDailyRows(db, projectId, startDate, maxDate),
+    [...queryDayAgg.values()].map((agg) => ({
+      date: agg.date,
+      query: agg.query,
+      clicks: agg.clicks,
+      impressions: agg.impressions,
+      position: agg.impressions > 0 ? agg.weightedPositionSum / agg.impressions : 0
+    }))
+  );
+  const topQueries = queryTotals.map((agg) => ({
+    query: agg.query,
     clicks: agg.clicks,
     impressions: agg.impressions,
     ctr: agg.impressions > 0 ? agg.clicks / agg.impressions : 0,
-    avgPosition: agg.impressions > 0 ? agg.weightedPositionSum / agg.impressions : 0,
-    category: categorizeQuery(query, projectBrandNames, canonicalDomain)
+    avgPosition: agg.position,
+    category: categorizeQuery(agg.query, projectBrandNames, canonicalDomain)
   })).sort((a, b) => b.clicks - a.clicks).slice(0, TOP_QUERIES_LIMIT);
   const categoryAgg = /* @__PURE__ */ new Map();
-  for (const [query, agg] of queryAgg) {
-    const cat = categorizeQuery(query, projectBrandNames, canonicalDomain);
+  for (const agg of queryTotals) {
+    const cat = categorizeQuery(agg.query, projectBrandNames, canonicalDomain);
     const bucket = categoryAgg.get(cat) ?? { clicks: 0, impressions: 0 };
     bucket.clicks += agg.clicks;
     bucket.impressions += agg.impressions;
@@ -13029,9 +13163,9 @@ function buildGscSection(db, projectId, projectBrandNames, canonicalDomain, trac
   const periodStart = trend[0]?.date ?? "";
   const periodEnd = trend.at(-1)?.date ?? "";
   const trackedSet = new Set(trackedQueries.map((q) => q.toLowerCase()));
-  const gscQuerySet = new Set([...queryAgg.keys()].map((q) => q.toLowerCase()));
+  const gscQuerySet = new Set(queryTotals.map((q) => q.query.toLowerCase()));
   const trackedButNoGsc = trackedQueries.filter((q) => !gscQuerySet.has(q.toLowerCase())).sort();
-  const gscButNotTracked = [...queryAgg.entries()].filter(([q]) => !trackedSet.has(q.toLowerCase())).filter(([q]) => categorizeQuery(q, projectBrandNames, canonicalDomain) !== "brand").sort((a, b) => b[1].impressions - a[1].impressions).map(([q]) => q).slice(0, TOP_QUERIES_LIMIT);
+  const gscButNotTracked = [...queryTotals].filter((agg) => !trackedSet.has(agg.query.toLowerCase())).filter((agg) => categorizeQuery(agg.query, projectBrandNames, canonicalDomain) !== "brand").sort((a, b) => b.impressions - a.impressions).map((agg) => agg.query).slice(0, TOP_QUERIES_LIMIT);
   return {
     periodStart,
     periodEnd,
@@ -15754,7 +15888,8 @@ function summarizeTransitionsFromSnapshots(latest, previous, since) {
 }
 function buildSuggestedQueriesFromGsc(app, projectId, trackedQueries) {
   const cutoff = new Date(Date.now() - 28 * 24 * 60 * 60 * 1e3).toISOString().slice(0, 10);
-  const rows = app.db.select({
+  const dimensionedRows = app.db.select({
+    date: gscSearchData.date,
     query: gscSearchData.query,
     impressions: sql8`COALESCE(SUM(${gscSearchData.impressions}), 0)`,
     clicks: sql8`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
@@ -15766,12 +15901,23 @@ function buildSuggestedQueriesFromGsc(app, projectId, trackedQueries) {
     eq20(gscSearchData.projectId, projectId),
     sql8`${gscSearchData.date} >= ${cutoff}`,
     sql8`${gscSearchData.impressions} > 0`
-  )).groupBy(gscSearchData.query).orderBy(sql8`SUM(${gscSearchData.impressions}) DESC`).limit(100).all();
-  const gscRows = rows.map((r) => ({
+  )).groupBy(gscSearchData.date, gscSearchData.query).all();
+  const todayIso = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+  const merged = mergeGscQueryTotalsWithFallback(
+    readGscQueryDailyRows(app.db, projectId, cutoff, todayIso),
+    dimensionedRows.map((r) => ({
+      date: r.date,
+      query: r.query,
+      impressions: Number(r.impressions),
+      clicks: Number(r.clicks),
+      position: Number(r.avgPosition)
+    }))
+  );
+  const gscRows = merged.filter((r) => r.impressions > 0).sort((a, b) => b.impressions - a.impressions).slice(0, 100).map((r) => ({
     query: r.query,
-    impressions: Number(r.impressions),
-    clicks: Number(r.clicks),
-    avgPosition: Number(r.avgPosition)
+    impressions: r.impressions,
+    clicks: r.clicks,
+    avgPosition: r.position
   }));
   return buildSuggestedQueries(gscRows, { trackedQueries });
 }
@@ -42265,6 +42411,7 @@ export {
   notifications,
   gscSearchData,
   gscDailyTotals,
+  gscQueryDailyTotals,
   gscUrlInspections,
   gscCoverageSnapshots,
   siteAuditSnapshots,

package/dist/{chunk-DN5H2ZN7.js → chunk-M6GKSLOJ.js}

@@ -77,6 +77,7 @@ import {
   groupRunsByCreatedAt,
   gscCoverageSnapshots,
   gscDailyTotals,
+  gscQueryDailyTotals,
   gscSearchData,
   gscUrlInspections,
   hashAttributes,
@@ -115,7 +116,7 @@ import {
   siteAuditPages,
   siteAuditSnapshots,
   usageCounters
-} from "./chunk-YPRUBNUK.js";
+} from "./chunk-FCGRCCX6.js";
 import {
   AGENT_MEMORY_VALUE_MAX_BYTES,
   AGENT_PROVIDER_IDS,
@@ -3869,6 +3870,36 @@ async function executeGscSync(db, runId, projectId, opts) {
       }).run();
     }
     log2.info("daily-totals.complete", { runId, projectId, rowCount: totalRows.length });
+    const queryTotalRows = await fetchSearchAnalytics(accessToken, propertyId, {
+      startDate,
+      endDate,
+      dimensions: ["date", "query"]
+    });
+    db.delete(gscQueryDailyTotals).where(
+      and2(
+        eq2(gscQueryDailyTotals.projectId, projectId),
+        sql2`${gscQueryDailyTotals.date} >= ${startDate}`,
+        sql2`${gscQueryDailyTotals.date} <= ${endDate}`
+      )
+    ).run();
+    const queryTotalsNow = (/* @__PURE__ */ new Date()).toISOString();
+    for (const row of queryTotalRows) {
+      const [date, query] = row.keys;
+      if (!date || !query) continue;
+      db.insert(gscQueryDailyTotals).values({
+        id: crypto4.randomUUID(),
+        projectId,
+        date,
+        query,
+        clicks: row.clicks,
+        impressions: row.impressions,
+        position: String(row.position),
+        syncedAt: queryTotalsNow,
+        syncRunId: runId,
+        createdAt: queryTotalsNow
+      }).run();
+    }
+    log2.info("query-totals.complete", { runId, projectId, rowCount: queryTotalRows.length });
     const pageClicks = /* @__PURE__ */ new Map();
     for (const row of rows) {
       const page = row.keys[1];
@@ -6936,7 +6967,7 @@ function readStoredGroundingSources(rawResponse) {
   return result;
 }
 async function backfillInsightsCommand(project, opts) {
-  const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-WJ2ZR7GF.js");
+  const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-K6B3MTDS.js");
   const config = loadConfig();
   const db = createClient(config.database);
   migrate(db);

package/dist/cli.js

@@ -29,7 +29,7 @@ import {
   setTelemetrySource,
   showFirstRunNotice,
   trackEvent
-} from "./chunk-DN5H2ZN7.js";
+} from "./chunk-M6GKSLOJ.js";
 import {
   CliError,
   EXIT_SYSTEM_ERROR,
@@ -54,7 +54,7 @@ import {
   projects,
   queries,
   renderReportHtml
-} from "./chunk-YPRUBNUK.js";
+} from "./chunk-FCGRCCX6.js";
 import {
   AdsOperationKinds,
   AdsOperationStates,

package/dist/index.js

@@ -1,10 +1,10 @@
 import {
   createServer
-} from "./chunk-DN5H2ZN7.js";
+} from "./chunk-M6GKSLOJ.js";
 import {
   loadConfig
 } from "./chunk-WA57WG6W.js";
-import "./chunk-YPRUBNUK.js";
+import "./chunk-FCGRCCX6.js";
 import "./chunk-RFLNTI7Y.js";
 export {
   createServer,

package/dist/{intelligence-service-WJ2ZR7GF.js → intelligence-service-K6B3MTDS.js}

@@ -1,6 +1,6 @@
 import {
   IntelligenceService
-} from "./chunk-YPRUBNUK.js";
+} from "./chunk-FCGRCCX6.js";
 import "./chunk-RFLNTI7Y.js";
 export {
   IntelligenceService

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@canonry/canonry",
-  "version": "4.125.0",
+  "version": "4.126.0",
   "type": "module",
   "description": "Agent-first open-source AEO operating platform - track how answer engines cite your domain",
   "license": "FSL-1.1-ALv2",
@@ -67,25 +67,25 @@
     "tsx": "^4.19.0",
     "@ainyc/canonry-api-client": "0.0.0",
     "@ainyc/canonry-api-routes": "0.0.0",
-    "@ainyc/canonry-config": "0.0.0",
     "@ainyc/canonry-contracts": "0.0.0",
     "@ainyc/canonry-db": "0.0.0",
+    "@ainyc/canonry-config": "0.0.0",
     "@ainyc/canonry-integration-bing": "0.0.0",
-    "@ainyc/canonry-integration-cloud-run": "0.0.0",
-    "@ainyc/canonry-integration-google": "0.0.0",
     "@ainyc/canonry-integration-openai-ads": "0.0.0",
+    "@ainyc/canonry-integration-cloud-run": "0.0.0",
     "@ainyc/canonry-integration-commoncrawl": "0.0.0",
+    "@ainyc/canonry-integration-google": "0.0.0",
     "@ainyc/canonry-integration-google-business-profile": "0.0.0",
     "@ainyc/canonry-integration-google-places": "0.0.0",
+    "@ainyc/canonry-integration-traffic": "0.0.0",
+    "@ainyc/canonry-integration-wordpress": "0.0.0",
     "@ainyc/canonry-intelligence": "0.0.0",
     "@ainyc/canonry-provider-cdp": "0.0.0",
-    "@ainyc/canonry-provider-claude": "0.0.0",
-    "@ainyc/canonry-provider-gemini": "0.0.0",
-    "@ainyc/canonry-integration-wordpress": "0.0.0",
-    "@ainyc/canonry-integration-traffic": "0.0.0",
     "@ainyc/canonry-provider-local": "0.0.0",
     "@ainyc/canonry-provider-openai": "0.0.0",
-    "@ainyc/canonry-provider-perplexity": "0.0.0"
+    "@ainyc/canonry-provider-gemini": "0.0.0",
+    "@ainyc/canonry-provider-perplexity": "0.0.0",
+    "@ainyc/canonry-provider-claude": "0.0.0"
   },
   "scripts": {
     "build": "tsx scripts/copy-agent-assets.ts && tsup && tsx build-web.ts",