@ainyc/canonry 4.51.4 → 4.54.0

package/dist/{chunk-HMZKIOLG.js → chunk-J7MX3YOH.js}

@@ -22,7 +22,7 @@ import {
   trafficConnectVercelRequestSchema,
   trafficConnectWordpressRequestSchema,
   trafficEventKindSchema
-} from "./chunk-FYGBW3SM.js";
+} from "./chunk-VZPDBHBW.js";
 
 // src/config.ts
 import fs from "fs";

package/dist/{chunk-QZ5XSM6C.js → chunk-JHAHNKSN.js}

@@ -10,7 +10,7 @@ import {
   categoryLabel,
   determineAnswerMentioned,
   normalizeProjectDomain
-} from "./chunk-FYGBW3SM.js";
+} from "./chunk-VZPDBHBW.js";
 
 // src/intelligence-service.ts
 import { eq, desc, asc, and, ne, or, inArray } from "drizzle-orm";
@@ -27,6 +27,7 @@ __export(schema_exports, {
   agentMemory: () => agentMemory,
   agentSessions: () => agentSessions,
   aiReferralEventsHourly: () => aiReferralEventsHourly,
+  aiUserFetchEventsHourly: () => aiUserFetchEventsHourly,
   apiKeys: () => apiKeys,
   auditLog: () => auditLog,
   backlinkDomains: () => backlinkDomains,
@@ -643,6 +644,34 @@ var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
   index("idx_crawler_hourly_project_ts").on(table.projectId, table.tsHour),
   index("idx_crawler_hourly_path").on(table.projectId, table.pathNormalized)
 ]);
+var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
+  projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
+  sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
+  tsHour: text("ts_hour").notNull(),
+  botId: text("bot_id").notNull(),
+  operator: text("operator").notNull(),
+  verificationStatus: text("verification_status").notNull(),
+  pathNormalized: text("path_normalized").notNull(),
+  status: integer("status").notNull(),
+  hits: integer("hits").notNull().default(0),
+  sampledUserAgent: text("sampled_user_agent"),
+  createdAt: text("created_at").notNull(),
+  updatedAt: text("updated_at").notNull()
+}, (table) => [
+  primaryKey({
+    columns: [
+      table.projectId,
+      table.sourceId,
+      table.tsHour,
+      table.botId,
+      table.verificationStatus,
+      table.pathNormalized,
+      table.status
+    ]
+  }),
+  index("idx_ai_user_fetch_hourly_project_ts").on(table.projectId, table.tsHour),
+  index("idx_ai_user_fetch_hourly_path").on(table.projectId, table.pathNormalized)
+]);
 var aiReferralEventsHourly = sqliteTable("ai_referral_events_hourly", {
   projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
   sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
@@ -2056,6 +2085,78 @@ var MIGRATION_VERSIONS = [
       `ALTER TABLE audit_log ADD COLUMN user_agent TEXT`,
       `ALTER TABLE audit_log ADD COLUMN actor_session TEXT`
     ]
+  },
+  {
+    version: 64,
+    name: "ai-user-fetch-events-hourly",
+    // Splits per-user fetches (ChatGPT-User, Perplexity-User) out of
+    // crawler_events_hourly so the dashboard / API can distinguish bulk
+    // machine crawl from human-in-the-loop fetch. Bot IDs are pinned to the
+    // two `purpose: 'user-agent'` rules that existed before this change —
+    // future user-fetch UAs land in the new table directly via the
+    // refactored classifier and never need a backfill.
+    //
+    // Statements are idempotent: CREATE/INDEX are IF NOT EXISTS; the
+    // INSERT … SELECT uses ON CONFLICT DO NOTHING (composite PK rows
+    // already moved skip silently); the DELETE keys on `bot_id`, so a
+    // second run is a no-op after the first DELETE drains the source.
+    statements: [
+      `CREATE TABLE IF NOT EXISTS ai_user_fetch_events_hourly (
+         project_id           TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+         source_id            TEXT NOT NULL REFERENCES traffic_sources(id) ON DELETE CASCADE,
+         ts_hour              TEXT NOT NULL,
+         bot_id               TEXT NOT NULL,
+         operator             TEXT NOT NULL,
+         verification_status  TEXT NOT NULL,
+         path_normalized      TEXT NOT NULL,
+         status               INTEGER NOT NULL,
+         hits                 INTEGER NOT NULL DEFAULT 0,
+         sampled_user_agent   TEXT,
+         created_at           TEXT NOT NULL,
+         updated_at           TEXT NOT NULL,
+         PRIMARY KEY (project_id, source_id, ts_hour, bot_id, verification_status, path_normalized, status)
+       )`,
+      `CREATE INDEX IF NOT EXISTS idx_ai_user_fetch_hourly_project_ts ON ai_user_fetch_events_hourly(project_id, ts_hour)`,
+      `CREATE INDEX IF NOT EXISTS idx_ai_user_fetch_hourly_path ON ai_user_fetch_events_hourly(project_id, path_normalized)`,
+      `INSERT INTO ai_user_fetch_events_hourly
+         (project_id, source_id, ts_hour, bot_id, operator, verification_status, path_normalized, status, hits, sampled_user_agent, created_at, updated_at)
+       SELECT project_id, source_id, ts_hour, bot_id, operator, verification_status, path_normalized, status, hits, sampled_user_agent, created_at, updated_at
+         FROM crawler_events_hourly
+        WHERE bot_id IN ('openai-chatgpt-user', 'perplexity-user')
+       ON CONFLICT DO NOTHING`,
+      `DELETE FROM crawler_events_hourly WHERE bot_id IN ('openai-chatgpt-user', 'perplexity-user')`
+    ]
+  },
+  {
+    version: 65,
+    name: "split-mistral-ai-rule",
+    // The pre-existing `mistral-ai` rule matched both `MistralAI-User/*`
+    // (per-user fetch) and `MistralBot/*` (bulk crawl) under one id, so
+    // every historical row landed in crawler_events_hourly with
+    // bot_id='mistral-ai'. The rule is now split into `mistral-ai-user`
+    // (purpose: 'user-agent') and `mistral-bot` (purpose: 'crawl'); this
+    // migration best-effort routes the legacy rows using the bucket's
+    // representative sampled_user_agent.
+    //
+    // Mixed-UA buckets (where a single (project, source, hour, path,
+    // status) accumulated both UAs under the old shared id) are routed
+    // by whichever UA happened to be sampled — the bucket-key granularity
+    // doesn't preserve per-event UAs, so any heuristic has the same
+    // limitation. Going forward the split rules write to disjoint tables.
+    //
+    // Idempotent: the INSERT…SELECT uses ON CONFLICT DO NOTHING; the
+    // UPDATE and DELETE both filter on bot_id='mistral-ai', so a second
+    // run finds no rows after the first apply.
+    statements: [
+      `INSERT INTO ai_user_fetch_events_hourly
+         (project_id, source_id, ts_hour, bot_id, operator, verification_status, path_normalized, status, hits, sampled_user_agent, created_at, updated_at)
+       SELECT project_id, source_id, ts_hour, 'mistral-ai-user', operator, verification_status, path_normalized, status, hits, sampled_user_agent, created_at, updated_at
+         FROM crawler_events_hourly
+        WHERE bot_id = 'mistral-ai' AND sampled_user_agent LIKE '%MistralAI-User%'
+       ON CONFLICT DO NOTHING`,
+      `DELETE FROM crawler_events_hourly WHERE bot_id = 'mistral-ai' AND sampled_user_agent LIKE '%MistralAI-User%'`,
+      `UPDATE crawler_events_hourly SET bot_id = 'mistral-bot' WHERE bot_id = 'mistral-ai'`
+    ]
   }
 ];
 function isDuplicateColumnError(err) {
@@ -4436,6 +4537,7 @@ export {
   agentMemory,
   trafficSources,
   crawlerEventsHourly,
+  aiUserFetchEventsHourly,
   aiReferralEventsHourly,
   rawEventSamples,
   discoverySessions,

package/dist/{chunk-FYGBW3SM.js → chunk-VZPDBHBW.js}

@@ -2646,6 +2646,15 @@ var serverActivitySectionSchema = z21.object({
   verifiedCrawlerHits: z21.object({ current: z21.number(), prior: z21.number(), deltaPct: z21.number().nullable() }),
   /** Last-7d total unverified crawler hits, separated from verified trust metrics. */
   unverifiedCrawlerHits: z21.object({ current: z21.number(), prior: z21.number(), deltaPct: z21.number().nullable() }),
+  /**
+   * Last-7d on-demand per-user fetches from AI surfaces (ChatGPT-User,
+   * Perplexity-User, MistralAI-User). Disjoint from `verifiedCrawlerHits` /
+   * `unverifiedCrawlerHits` — those measure bulk crawl; this measures human
+   * users asking an AI to read a URL. Counts verified + unverified together
+   * because the operational question for user-fetch is "is this happening?"
+   * not "is this a confirmed bot identity?"
+   */
+  aiUserFetchHits: z21.object({ current: z21.number(), prior: z21.number(), deltaPct: z21.number().nullable() }),
   /** Last-7d AI-referral sessions (sessionized from server-side request evidence). */
   referralArrivals: z21.object({ current: z21.number(), prior: z21.number(), deltaPct: z21.number().nullable() }),
   /** Per-AI-operator breakdown (OpenAI, Anthropic, Google AI, Perplexity, …). */
@@ -2654,6 +2663,8 @@ var serverActivitySectionSchema = z21.object({
     verifiedHits: z21.number(),
     /** Shown to agency audience only — claimed-bot UA, rDNS not confirmed. */
     unverifiedHits: z21.number(),
+    /** Per-user fetches from this operator's AI surface (ChatGPT-User, …). */
+    userFetchHits: z21.number(),
     referralArrivals: z21.number(),
     deltaPct: z21.number().nullable()
   })),
@@ -2681,6 +2692,7 @@ var serverActivitySectionSchema = z21.object({
   dailyTrend: z21.array(z21.object({
     date: z21.string(),
     verifiedCrawlerHits: z21.number(),
+    userFetchHits: z21.number(),
     referralArrivals: z21.number()
   })),
   /**
@@ -3198,9 +3210,11 @@ var trafficSyncResponseSchema = z23.object({
   syncedAt: z23.string(),
   pulledEvents: z23.number().int().nonnegative(),
   crawlerHits: z23.number().int().nonnegative(),
+  aiUserFetchHits: z23.number().int().nonnegative(),
   aiReferralHits: z23.number().int().nonnegative(),
   unknownHits: z23.number().int().nonnegative(),
   crawlerBucketRows: z23.number().int().nonnegative(),
+  aiUserFetchBucketRows: z23.number().int().nonnegative(),
   aiReferralBucketRows: z23.number().int().nonnegative(),
   sampleRows: z23.number().int().nonnegative(),
   windowStart: z23.string(),
@@ -3222,6 +3236,7 @@ var trafficBackfillResponseSchema = z23.object({
 });
 var trafficSourceTotalsSchema = z23.object({
   crawlerHits: z23.number().int().nonnegative(),
+  aiUserFetchHits: z23.number().int().nonnegative(),
   aiReferralHits: z23.number().int().nonnegative(),
   sampleCount: z23.number().int().nonnegative()
 });
@@ -3241,7 +3256,7 @@ var trafficSourceDetailDtoSchema = trafficSourceDtoSchema.extend({
 var trafficStatusResponseSchema = z23.object({
   sources: z23.array(trafficSourceDetailDtoSchema)
 });
-var trafficEventKindSchema = z23.enum(["crawler", "ai-referral"]);
+var trafficEventKindSchema = z23.enum(["crawler", "ai-user-fetch", "ai-referral"]);
 var TrafficEventKinds = trafficEventKindSchema.enum;
 var trafficCrawlerEventEntrySchema = z23.object({
   kind: z23.literal(TrafficEventKinds.crawler),
@@ -3254,6 +3269,17 @@ var trafficCrawlerEventEntrySchema = z23.object({
   status: z23.number().int(),
   hits: z23.number().int().nonnegative()
 });
+var trafficAiUserFetchEventEntrySchema = z23.object({
+  kind: z23.literal(TrafficEventKinds["ai-user-fetch"]),
+  sourceId: z23.string(),
+  tsHour: z23.string(),
+  botId: z23.string(),
+  operator: z23.string(),
+  verificationStatus: z23.string(),
+  pathNormalized: z23.string(),
+  status: z23.number().int(),
+  hits: z23.number().int().nonnegative()
+});
 var trafficAiReferralEventEntrySchema = z23.object({
   kind: z23.literal(TrafficEventKinds["ai-referral"]),
   sourceId: z23.string(),
@@ -3268,6 +3294,7 @@ var trafficAiReferralEventEntrySchema = z23.object({
 });
 var trafficEventEntrySchema = z23.discriminatedUnion("kind", [
   trafficCrawlerEventEntrySchema,
+  trafficAiUserFetchEventEntrySchema,
   trafficAiReferralEventEntrySchema
 ]);
 var trafficEventsResponseSchema = z23.object({
@@ -3275,6 +3302,7 @@ var trafficEventsResponseSchema = z23.object({
   windowEnd: z23.string(),
   totals: z23.object({
     crawlerHits: z23.number().int().nonnegative(),
+    aiUserFetchHits: z23.number().int().nonnegative(),
     aiReferralHits: z23.number().int().nonnegative()
   }),
   events: z23.array(trafficEventEntrySchema)

package/dist/cli.js

@@ -23,7 +23,7 @@ import {
   setTelemetrySource,
   showFirstRunNotice,
   trackEvent
-} from "./chunk-WBO5S3IX.js";
+} from "./chunk-CRO6Q25G.js";
 import {
   CliError,
   EXIT_SYSTEM_ERROR,
@@ -39,14 +39,14 @@ import {
   saveConfig,
   saveConfigPatch,
   usageError
-} from "./chunk-HMZKIOLG.js";
+} from "./chunk-J7MX3YOH.js";
 import {
   apiKeys,
   createClient,
   migrate,
   projects,
   queries
-} from "./chunk-QZ5XSM6C.js";
+} from "./chunk-JHAHNKSN.js";
 import {
   CcReleaseSyncStatuses,
   CheckScopes,
@@ -65,7 +65,7 @@ import {
   providerQuotaPolicySchema,
   resolveProviderInput,
   skillsClientSchema
-} from "./chunk-FYGBW3SM.js";
+} from "./chunk-VZPDBHBW.js";
 
 // src/cli.ts
 import { pathToFileURL } from "url";
@@ -3310,33 +3310,45 @@ async function trafficStatus(project, opts) {
   }
 }
 function formatEventLine(event) {
-  if (event.kind === TrafficEventKinds.crawler) {
-    return [
-      event.tsHour,
-      "crawler",
-      event.botId,
-      event.verificationStatus,
-      String(event.status),
-      event.pathNormalized,
-      `${event.hits} hits`
-    ].join("  ");
+  switch (event.kind) {
+    case TrafficEventKinds.crawler:
+      return [
+        event.tsHour,
+        "crawler",
+        event.botId,
+        event.verificationStatus,
+        String(event.status),
+        event.pathNormalized,
+        `${event.hits} hits`
+      ].join("  ");
+    case TrafficEventKinds["ai-user-fetch"]:
+      return [
+        event.tsHour,
+        "ai-user-fetch",
+        event.botId,
+        event.verificationStatus,
+        String(event.status),
+        event.pathNormalized,
+        `${event.hits} hits`
+      ].join("  ");
+    case TrafficEventKinds["ai-referral"]:
+      return [
+        event.tsHour,
+        "ai-referral",
+        event.product,
+        event.evidenceType,
+        event.sourceDomain,
+        event.landingPathNormalized,
+        `${event.hits} hits`
+      ].join("  ");
   }
-  return [
-    event.tsHour,
-    "ai-referral",
-    event.product,
-    event.evidenceType,
-    event.sourceDomain,
-    event.landingPathNormalized,
-    `${event.hits} hits`
-  ].join("  ");
 }
 async function trafficEvents(project, opts) {
-  if (opts.kind && opts.kind !== "all" && opts.kind !== TrafficEventKinds.crawler && opts.kind !== TrafficEventKinds["ai-referral"]) {
+  if (opts.kind && opts.kind !== "all" && opts.kind !== TrafficEventKinds.crawler && opts.kind !== TrafficEventKinds["ai-user-fetch"] && opts.kind !== TrafficEventKinds["ai-referral"]) {
     throw new CliError({
       code: "TRAFFIC_INVALID_KIND",
-      message: `--kind must be one of: all, ${TrafficEventKinds.crawler}, ${TrafficEventKinds["ai-referral"]}`,
-      displayMessage: `Error: --kind must be "all", "${TrafficEventKinds.crawler}", or "${TrafficEventKinds["ai-referral"]}"`,
+      message: `--kind must be one of: all, ${TrafficEventKinds.crawler}, ${TrafficEventKinds["ai-user-fetch"]}, ${TrafficEventKinds["ai-referral"]}`,
+      displayMessage: `Error: --kind must be "all", "${TrafficEventKinds.crawler}", "${TrafficEventKinds["ai-user-fetch"]}", or "${TrafficEventKinds["ai-referral"]}"`,
       details: { project, kind: opts.kind }
     });
   }
@@ -3358,7 +3370,8 @@ async function trafficEvents(project, opts) {
     return;
   }
   console.log(`Traffic events for "${project}"  ${result.windowStart}  \u2192  ${result.windowEnd}`);
-  console.log(`  Crawler hits (window):     ${result.totals.crawlerHits}`);
+  console.log(`  Crawler hits (window):         ${result.totals.crawlerHits}`);
+  console.log(`  AI user-fetch hits (window):   ${result.totals.aiUserFetchHits}`);
   console.log(`  AI referral sessions (window): ${result.totals.aiReferralHits}`);
   console.log("");
   if (result.events.length === 0) {

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   createServer
-} from "./chunk-WBO5S3IX.js";
+} from "./chunk-CRO6Q25G.js";
 import {
   loadConfig
-} from "./chunk-HMZKIOLG.js";
-import "./chunk-QZ5XSM6C.js";
-import "./chunk-FYGBW3SM.js";
+} from "./chunk-J7MX3YOH.js";
+import "./chunk-JHAHNKSN.js";
+import "./chunk-VZPDBHBW.js";
 export {
   createServer,
   loadConfig

package/dist/{intelligence-service-2XL2M7QP.js → intelligence-service-OCREQUCQ.js}

@@ -1,7 +1,7 @@
 import {
   IntelligenceService
-} from "./chunk-QZ5XSM6C.js";
-import "./chunk-FYGBW3SM.js";
+} from "./chunk-JHAHNKSN.js";
+import "./chunk-VZPDBHBW.js";
 export {
   IntelligenceService
 };

package/dist/mcp.js

@@ -2,8 +2,8 @@ import {
   CliError,
   canonryMcpTools,
   createApiClient
-} from "./chunk-HMZKIOLG.js";
-import "./chunk-FYGBW3SM.js";
+} from "./chunk-J7MX3YOH.js";
+import "./chunk-VZPDBHBW.js";
 
 // src/mcp/cli.ts
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ainyc/canonry",
-  "version": "4.51.4",
+  "version": "4.54.0",
   "type": "module",
   "description": "Agent-first open-source AEO operating platform - track how answer engines cite your domain",
   "license": "FSL-1.1-ALv2",
@@ -61,23 +61,23 @@
     "tsup": "^8.5.1",
     "tsx": "^4.19.0",
     "@ainyc/canonry-api-client": "0.0.0",
-    "@ainyc/canonry-config": "0.0.0",
-    "@ainyc/canonry-contracts": "0.0.0",
-    "@ainyc/canonry-db": "0.0.0",
     "@ainyc/canonry-api-routes": "0.0.0",
+    "@ainyc/canonry-db": "0.0.0",
+    "@ainyc/canonry-contracts": "0.0.0",
+    "@ainyc/canonry-config": "0.0.0",
     "@ainyc/canonry-intelligence": "0.0.0",
-    "@ainyc/canonry-integration-commoncrawl": "0.0.0",
-    "@ainyc/canonry-integration-cloud-run": "0.0.0",
     "@ainyc/canonry-integration-bing": "0.0.0",
-    "@ainyc/canonry-integration-traffic": "0.0.0",
-    "@ainyc/canonry-integration-google": "0.0.0",
-    "@ainyc/canonry-integration-wordpress": "0.0.0",
+    "@ainyc/canonry-integration-commoncrawl": "0.0.0",
     "@ainyc/canonry-provider-cdp": "0.0.0",
+    "@ainyc/canonry-integration-traffic": "0.0.0",
+    "@ainyc/canonry-integration-wordpress": "1.0.0",
+    "@ainyc/canonry-integration-cloud-run": "0.0.0",
+    "@ainyc/canonry-provider-gemini": "0.0.0",
     "@ainyc/canonry-provider-claude": "0.0.0",
-    "@ainyc/canonry-provider-local": "0.0.0",
+    "@ainyc/canonry-integration-google": "0.0.0",
+    "@ainyc/canonry-provider-perplexity": "0.0.0",
     "@ainyc/canonry-provider-openai": "0.0.0",
-    "@ainyc/canonry-provider-gemini": "0.0.0",
-    "@ainyc/canonry-provider-perplexity": "0.0.0"
+    "@ainyc/canonry-provider-local": "0.0.0"
   },
   "scripts": {
     "build": "tsx scripts/copy-agent-assets.ts && tsup && tsx build-web.ts",