@ainyc/canonry 4.87.0 → 4.89.0

package/dist/{chunk-MDRDX5R2.js → chunk-LRO4CLPL.js}

@@ -4555,6 +4555,20 @@ function coerceSkillManifest(parsed) {
 
 // ../contracts/src/traffic.ts
 import { z as z30 } from "zod";
+var trafficCrawlerSegmentsSchema = z30.object({
+  content: z30.number().int().nonnegative(),
+  sitemap: z30.number().int().nonnegative(),
+  robots: z30.number().int().nonnegative(),
+  asset: z30.number().int().nonnegative(),
+  other: z30.number().int().nonnegative()
+});
+var trafficPathClassSchema = z30.enum([
+  "content",
+  "sitemap",
+  "robots",
+  "asset",
+  "other"
+]);
 var trafficSourceTypeSchema = z30.enum([
   "cloud-run",
   "wordpress",
@@ -4711,7 +4725,18 @@ var trafficBackfillResponseSchema = z30.object({
   daysApplied: z30.number().int().positive()
 });
 var trafficSourceTotalsSchema = z30.object({
+  /**
+   * Total classified-crawler hits in the window. UNCHANGED contract — still the
+   * full count across every path class. Use `crawlerContentHits` for the
+   * "content was actually crawled" signal.
+   */
   crawlerHits: z30.number().int().nonnegative(),
+  /** Crawler hits against content/document paths only (= `crawlerSegments.content`). */
+  crawlerContentHits: z30.number().int().nonnegative(),
+  /** Infrastructure crawler hits — sitemap + robots + asset fetches (`crawlerSegments.{sitemap,robots,asset}`). */
+  crawlerInfraHits: z30.number().int().nonnegative(),
+  /** Full per-class crawler-hit breakdown; the five buckets sum to `crawlerHits`. */
+  crawlerSegments: trafficCrawlerSegmentsSchema,
   aiUserFetchHits: z30.number().int().nonnegative(),
   aiReferralHits: z30.number().int().nonnegative(),
   sampleCount: z30.number().int().nonnegative()
@@ -4742,6 +4767,8 @@ var trafficCrawlerEventEntrySchema = z30.object({
   operator: z30.string(),
   verificationStatus: z30.string(),
   pathNormalized: z30.string(),
+  /** Coarse class of the fetched path — lets the UI split content crawls from sitemap/robots/asset polling. */
+  pathClass: trafficPathClassSchema,
   status: z30.number().int(),
   hits: z30.number().int().nonnegative()
 });
@@ -4777,13 +4804,151 @@ var trafficEventsResponseSchema = z30.object({
   windowStart: z30.string(),
   windowEnd: z30.string(),
   totals: z30.object({
+    /** Total classified-crawler hits across the window. UNCHANGED contract. */
     crawlerHits: z30.number().int().nonnegative(),
+    /** Crawler hits against content/document paths only (= `crawlerSegments.content`). */
+    crawlerContentHits: z30.number().int().nonnegative(),
+    /** Infrastructure crawler hits — sitemap + robots + asset fetches. */
+    crawlerInfraHits: z30.number().int().nonnegative(),
+    /** Full per-class crawler-hit breakdown; the five buckets sum to `crawlerHits`. */
+    crawlerSegments: trafficCrawlerSegmentsSchema,
     aiUserFetchHits: z30.number().int().nonnegative(),
     aiReferralHits: z30.number().int().nonnegative()
   }),
   events: z30.array(trafficEventEntrySchema)
 });
 
+// ../contracts/src/traffic-path.ts
+var TrafficPathClasses = {
+  content: "content",
+  sitemap: "sitemap",
+  robots: "robots",
+  asset: "asset",
+  other: "other"
+};
+var ROBOTS_BASENAMES = /* @__PURE__ */ new Set(["robots.txt", "llms.txt", "llms-full.txt"]);
+var ASSET_EXTENSIONS = /* @__PURE__ */ new Set([
+  "css",
+  "js",
+  "mjs",
+  "cjs",
+  "map",
+  "json",
+  "png",
+  "jpg",
+  "jpeg",
+  "webp",
+  "avif",
+  "gif",
+  "svg",
+  "ico",
+  "bmp",
+  "tif",
+  "tiff",
+  "woff",
+  "woff2",
+  "ttf",
+  "otf",
+  "eot",
+  "mp4",
+  "webm",
+  "mov",
+  "m4v",
+  "mp3",
+  "wav",
+  "ogg",
+  "flac",
+  "m4a",
+  "wasm"
+]);
+var DOCUMENT_EXTENSIONS = /* @__PURE__ */ new Set([
+  "html",
+  "htm",
+  "xhtml",
+  "shtml",
+  "php",
+  "php5",
+  "php7",
+  "asp",
+  "aspx",
+  "jsp",
+  "jspx",
+  "cfm",
+  "md"
+]);
+var DOWNLOAD_EXTENSIONS = /* @__PURE__ */ new Set([
+  "pdf",
+  "csv",
+  "tsv",
+  "txt",
+  "doc",
+  "docx",
+  "xls",
+  "xlsx",
+  "ppt",
+  "pptx",
+  "odt",
+  "ods",
+  "odp",
+  "rtf",
+  "zip",
+  "gz",
+  "tgz",
+  "tar",
+  "rar",
+  "7z",
+  "bz2",
+  "xz",
+  "rss",
+  "atom",
+  "ics",
+  "vcf",
+  "epub",
+  "mobi",
+  "apk",
+  "dmg",
+  "exe",
+  "bin",
+  "iso",
+  "sql"
+]);
+var SITEMAP_BASENAME = /^sitemap(?:[-_]index)?$/;
+function classifyTrafficPath(pathNormalized) {
+  const raw = (pathNormalized ?? "").trim();
+  if (!raw) return TrafficPathClasses.other;
+  const rawPath = raw.split(/[?#]/)[0] ?? "";
+  const pathOnly = rawPath.length > 1 ? rawPath.replace(/\/+$/, "") || "/" : rawPath;
+  if (!pathOnly) return TrafficPathClasses.other;
+  const lower = pathOnly.toLowerCase();
+  const segments = lower.split("/");
+  const basename = segments[segments.length - 1] ?? "";
+  if (ROBOTS_BASENAMES.has(basename)) return TrafficPathClasses.robots;
+  if (lower.endsWith(".xml") || lower.endsWith(".xml.gz")) return TrafficPathClasses.sitemap;
+  if (SITEMAP_BASENAME.test(basename)) return TrafficPathClasses.sitemap;
+  const dot = basename.lastIndexOf(".");
+  const ext = dot > 0 ? basename.slice(dot + 1) : "";
+  if (ext) {
+    if (ASSET_EXTENSIONS.has(ext)) return TrafficPathClasses.asset;
+    if (DOCUMENT_EXTENSIONS.has(ext)) return TrafficPathClasses.content;
+    if (DOWNLOAD_EXTENSIONS.has(ext)) return TrafficPathClasses.other;
+    return TrafficPathClasses.content;
+  }
+  return TrafficPathClasses.content;
+}
+function emptyCrawlerSegments() {
+  return { content: 0, sitemap: 0, robots: 0, asset: 0, other: 0 };
+}
+function segmentCrawlerHits(rows) {
+  const segments = emptyCrawlerSegments();
+  for (const row of rows) {
+    segments[classifyTrafficPath(row.pathNormalized)] += row.hits;
+  }
+  return segments;
+}
+function sumInfraHits(segments) {
+  return segments.sitemap + segments.robots + segments.asset;
+}
+
 // ../contracts/src/formatting.ts
 function formatRatio(value) {
   if (!Number.isFinite(value) || value === 0) return "0%";
@@ -5018,6 +5183,63 @@ function escapeLikePattern(value) {
   return value.replace(/[\\%_]/g, (match) => `\\${match}`);
 }
 
+// ../contracts/src/embed.ts
+var SELF_TOKEN = "'self'";
+function normalizeFrameOrigin(raw) {
+  const trimmed = raw.trim();
+  if (!trimmed) return null;
+  if (trimmed === SELF_TOKEN) return SELF_TOKEN;
+  if (trimmed.includes("*")) return null;
+  let url;
+  try {
+    url = new URL(trimmed);
+  } catch {
+    return null;
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+  if (url.username || url.password) return null;
+  if (url.pathname && url.pathname !== "/" || url.search || url.hash) return null;
+  if (!url.hostname) return null;
+  return url.origin;
+}
+function splitList(raw) {
+  if (raw === void 0) return [];
+  const tokens = (Array.isArray(raw) ? raw : [raw]).flatMap(
+    (entry) => String(entry).split(/[\s,]+/)
+  );
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const token of tokens) {
+    const trimmed = token.trim();
+    if (!trimmed || seen.has(trimmed)) continue;
+    seen.add(trimmed);
+    out.push(trimmed);
+  }
+  return out;
+}
+function parseOriginList(raw) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const token of splitList(raw)) {
+    const origin = normalizeFrameOrigin(token);
+    if (!origin || seen.has(origin)) continue;
+    seen.add(origin);
+    out.push(origin);
+  }
+  return out;
+}
+function frameAncestorsHeaderValue(origins) {
+  if (origins.length === 0) return "frame-ancestors 'none'";
+  return `frame-ancestors ${origins.join(" ")}`;
+}
+function buildEmbedClientConfig(resolved) {
+  if (!resolved.enabled) return void 0;
+  const client = { enabled: true };
+  if (resolved.views && resolved.views.length > 0) client.views = resolved.views;
+  if (resolved.theme && Object.keys(resolved.theme).length > 0) client.theme = resolved.theme;
+  return client;
+}
+
 export {
   __export,
   apiKeyDtoSchema,
@@ -5285,6 +5507,9 @@ export {
   trafficEventKindSchema,
   TrafficEventKinds,
   trafficEventsResponseSchema,
+  classifyTrafficPath,
+  segmentCrawlerHits,
+  sumInfraHits,
   formatRatio,
   formatNumber,
   formatDate,
@@ -5312,5 +5537,9 @@ export {
   AI_PROVIDER_INFRA_DOMAINS,
   escapeLikePattern,
   READ_ONLY_SCOPE,
-  isReadOnlyKey
+  isReadOnlyKey,
+  splitList,
+  parseOriginList,
+  frameAncestorsHeaderValue,
+  buildEmbedClientConfig
 };

package/dist/{chunk-DUDFNP5Y.js → chunk-MUPIPQBB.js}

@@ -9,7 +9,7 @@ import {
   loadConfig,
   loadConfigRaw,
   saveConfigPatch
-} from "./chunk-6XMXBAEW.js";
+} from "./chunk-PRKBJNAG.js";
 import {
   CC_CACHE_DIR,
   DUCKDB_SPEC,
@@ -104,7 +104,7 @@ import {
   siteAuditPages,
   siteAuditSnapshots,
   usageCounters
-} from "./chunk-5LW7CJAO.js";
+} from "./chunk-WEV4RH25.js";
 import {
   AGENT_MEMORY_VALUE_MAX_BYTES,
   AGENT_PROVIDER_IDS,
@@ -136,6 +136,7 @@ import {
   agentMemoryUpsertRequestSchema,
   authInvalid,
   authRequired,
+  buildEmbedClientConfig,
   buildRunErrorFromMessages,
   classifySkillFile,
   coerceSkillManifest,
@@ -147,20 +148,23 @@ import {
   effectiveBrandNames,
   effectiveDomains,
   factorStatusFromScore,
+  frameAncestorsHeaderValue,
   hostOf,
   isAgentProviderId,
   isBrowserProvider,
   isRetryableHttpError,
   normalizeUrlPath,
   notFound,
+  parseOriginList,
   providerError,
   resolveSnapshotRequestQueries,
   serializeRunError,
   skillsClientSchema,
+  splitList,
   validationError,
   winnabilityClassLabel,
   withRetry
-} from "./chunk-MDRDX5R2.js";
+} from "./chunk-LRO4CLPL.js";
 
 // src/telemetry.ts
 import crypto from "crypto";
@@ -2728,6 +2732,34 @@ var perplexityAdapter = {
   }
 };
 
+// src/embed.ts
+function resolveEmbedConfig(env, config) {
+  const embed = config.embed;
+  const rawEnabled = env.CANONRY_EMBED?.trim();
+  const enabled = rawEnabled ? rawEnabled === "1" || rawEnabled.toLowerCase() === "true" : embed?.enabled === true;
+  const allowedOrigins = env.CANONRY_EMBED_ORIGINS !== void 0 ? parseOriginList(env.CANONRY_EMBED_ORIGINS) : parseOriginList(embed?.allowOrigins);
+  const rawViews = env.CANONRY_EMBED_VIEWS !== void 0 ? splitList(env.CANONRY_EMBED_VIEWS) : splitList(embed?.views);
+  const views = normalizeViews(rawViews);
+  return {
+    enabled,
+    allowedOrigins,
+    ...views ? { views } : {},
+    ...embed?.theme ? { theme: embed.theme } : {}
+  };
+}
+function normalizeViews(raw) {
+  if (raw.length === 0) return void 0;
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const view of raw) {
+    const id = view.toLowerCase();
+    if (seen.has(id)) continue;
+    seen.add(id);
+    out.push(id);
+  }
+  return out.length > 0 ? out : void 0;
+}
+
 // src/places-config.ts
 var DEFAULT_TIER = "atmosphere";
 var DEFAULT_REFRESH_INTERVAL_DAYS = 7;
@@ -6289,7 +6321,7 @@ function readStoredGroundingSources(rawResponse) {
   return result;
 }
 async function backfillInsightsCommand(project, opts) {
-  const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-XUKYOHKL.js");
+  const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-SO46DMGR.js");
   const config = loadConfig();
   const db = createClient(config.database);
   migrate(db);
@@ -10538,6 +10570,8 @@ async function createServer(opts) {
   const rawBasePath = process.env.CANONRY_BASE_PATH ?? opts.config.basePath;
   const normalizedBasePath = rawBasePath ? "/" + rawBasePath.replace(/^\//, "").replace(/\/?$/, "/") : void 0;
   const basePath = normalizedBasePath === "/" ? void 0 : normalizedBasePath;
+  const embed = resolveEmbedConfig(process.env, opts.config);
+  const embedCsp = embed.enabled ? frameAncestorsHeaderValue(embed.allowedOrigins) : void 0;
   const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
   if (opts.config.apiKey) {
     const keyHash = hashApiKey(opts.config.apiKey);
@@ -11173,16 +11207,25 @@ async function createServer(opts) {
     }
   });
   const dirname = path9.dirname(fileURLToPath3(import.meta.url));
-  const assetsDir = path9.join(dirname, "..", "assets");
+  const assetsDir = opts.assetsDir ?? path9.join(dirname, "..", "assets");
   if (fs8.existsSync(assetsDir)) {
     const indexPath = path9.join(assetsDir, "index.html");
     const injectConfig = (html) => {
       const clientConfig = {};
       if (basePath) clientConfig.basePath = basePath;
+      if (embed.enabled) {
+        const embedClient = buildEmbedClientConfig(embed);
+        if (embedClient) clientConfig.embed = embedClient;
+      }
       const configScript = `<script>window.__CANONRY_CONFIG__=${JSON.stringify(clientConfig)}</script>`;
       const baseTag = `<base href="${basePath ?? "/"}">`;
       return html.replace("<head>", `<head>${baseTag}`).replace("</head>", `${configScript}</head>`);
     };
+    const sendSpaDocument = (reply, html) => {
+      reply.header("Cache-Control", "no-cache, must-revalidate");
+      if (embedCsp) reply.header("Content-Security-Policy", embedCsp);
+      return reply.type("text/html").send(injectConfig(html));
+    };
     const fastifyStatic = await import("@fastify/static");
     await app.register(fastifyStatic.default, {
       root: assetsDir,
@@ -11209,7 +11252,7 @@ async function createServer(opts) {
     const serveIndex = (_request, reply) => {
       if (fs8.existsSync(indexPath)) {
         const html = fs8.readFileSync(indexPath, "utf-8");
-        return reply.header("Cache-Control", "no-cache, must-revalidate").type("text/html").send(injectConfig(html));
+        return sendSpaDocument(reply, html);
       }
       return reply.status(404).send({ error: "Dashboard not built" });
     };
@@ -11230,7 +11273,7 @@ async function createServer(opts) {
       }
       if (fs8.existsSync(indexPath)) {
         const html = fs8.readFileSync(indexPath, "utf-8");
-        return reply.header("Cache-Control", "no-cache, must-revalidate").type("text/html").send(injectConfig(html));
+        return sendSpaDocument(reply, html);
       }
       return reply.status(404).send({ error: "Not found" });
     });

package/dist/{chunk-6XMXBAEW.js → chunk-PRKBJNAG.js}

@@ -23,7 +23,7 @@ import {
   trafficConnectVercelRequestSchema,
   trafficConnectWordpressRequestSchema,
   trafficEventKindSchema
-} from "./chunk-MDRDX5R2.js";
+} from "./chunk-LRO4CLPL.js";
 
 // src/config.ts
 import fs from "fs";

package/dist/{chunk-5LW7CJAO.js → chunk-WEV4RH25.js}

@@ -86,6 +86,7 @@ import {
   classifyCitedSurface,
   classifySkillFile,
   classifySurfaceFromCategory,
+  classifyTrafficPath,
   clusterByCosine,
   coerceSkillManifest,
   competitorBatchRequestSchema,
@@ -210,6 +211,7 @@ import {
   scheduleDtoSchema,
   scheduleUpsertRequestSchema,
   seedCollapseWarning,
+  segmentCrawlerHits,
   serializeRunError,
   settingsDtoSchema,
   siteAuditPagesResponseSchema,
@@ -222,6 +224,7 @@ import {
   snapshotReportSchema,
   snapshotRequestSchema,
   sourceBreakdownDtoSchema,
+  sumInfraHits,
   summarizeCheckResults,
   surfaceClassFromCompetitorType,
   surfaceClassLabel,
@@ -255,7 +258,7 @@ import {
   wordpressSchemaDeployResultDtoSchema,
   wordpressSchemaStatusResultDtoSchema,
   wordpressStatusDtoSchema
-} from "./chunk-MDRDX5R2.js";
+} from "./chunk-LRO4CLPL.js";
 
 // src/intelligence-service.ts
 import { eq as eq37, desc as desc18, asc as asc5, and as and27, ne as ne5, or as or5, inArray as inArray14, gte as gte7, lte as lte4 } from "drizzle-orm";
@@ -30625,12 +30628,19 @@ async function trafficRoutes(app, opts) {
     return response;
   });
   function buildSourceDetail(projectId, row, since) {
-    const crawlerTotals = app.db.select({ total: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
+    const crawlerPathRows = app.db.select({
+      pathNormalized: crawlerEventsHourly.pathNormalized,
+      hits: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
+    }).from(crawlerEventsHourly).where(
       and21(
         eq27(crawlerEventsHourly.sourceId, row.id),
         gte4(crawlerEventsHourly.tsHour, since)
       )
-    ).get();
+    ).groupBy(crawlerEventsHourly.pathNormalized).all();
+    const crawlerSegments = segmentCrawlerHits(
+      crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
+    );
+    const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
     const aiUserFetchTotals = app.db.select({ total: sql11`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
       and21(
         eq27(aiUserFetchEventsHourly.sourceId, row.id),
@@ -30659,7 +30669,10 @@ async function trafficRoutes(app, opts) {
     return {
       ...rowToDto(row),
       totals24h: {
-        crawlerHits: Number(crawlerTotals?.total ?? 0),
+        crawlerHits: crawlerTotal,
+        crawlerContentHits: crawlerSegments.content,
+        crawlerInfraHits: sumInfraHits(crawlerSegments),
+        crawlerSegments,
         aiUserFetchHits: Number(aiUserFetchTotals?.total ?? 0),
         aiReferralHits: Number(aiTotals?.total ?? 0),
         sampleCount: Number(sampleTotals?.total ?? 0)
@@ -30776,6 +30789,7 @@ async function trafficRoutes(app, opts) {
     const untilIso = until.toISOString();
     const events = [];
     let crawlerTotal = 0;
+    let crawlerSegments = { content: 0, sitemap: 0, robots: 0, asset: 0, other: 0 };
     let aiUserFetchTotal = 0;
     let aiReferralTotal = 0;
     if (kind === "all" || kind === TrafficEventKinds.crawler) {
@@ -30786,8 +30800,14 @@ async function trafficRoutes(app, opts) {
       ];
       if (sourceIdParam) crawlerFilters.push(eq27(crawlerEventsHourly.sourceId, sourceIdParam));
       const crawlerWhere = and21(...crawlerFilters);
-      const total = app.db.select({ total: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(crawlerWhere).get();
-      crawlerTotal = Number(total?.total ?? 0);
+      const pathTotals = app.db.select({
+        pathNormalized: crawlerEventsHourly.pathNormalized,
+        hits: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
+      }).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
+      crawlerSegments = segmentCrawlerHits(
+        pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
+      );
+      crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
       const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc15(crawlerEventsHourly.tsHour)).limit(limit).all();
       for (const r of rows) {
         events.push({
@@ -30798,6 +30818,7 @@ async function trafficRoutes(app, opts) {
           operator: r.operator,
           verificationStatus: r.verificationStatus,
           pathNormalized: r.pathNormalized,
+          pathClass: classifyTrafficPath(r.pathNormalized),
           status: r.status,
           hits: r.hits
         });
@@ -30861,6 +30882,9 @@ async function trafficRoutes(app, opts) {
       windowEnd: untilIso,
       totals: {
         crawlerHits: crawlerTotal,
+        crawlerContentHits: crawlerSegments.content,
+        crawlerInfraHits: sumInfraHits(crawlerSegments),
+        crawlerSegments,
         aiUserFetchHits: aiUserFetchTotal,
         aiReferralHits: aiReferralTotal
       },