mcp-scraper 0.48.0 → 0.50.0

package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js}

@@ -459,6 +459,12 @@ async function migrateAnalytics() {
     await client.query(
       `ALTER TABLE analytics_events ADD COLUMN IF NOT EXISTS utm_content text`
     );
+    await client.query(
+      `ALTER TABLE analytics_events ADD COLUMN IF NOT EXISTS engaged_ms integer`
+    );
+    await client.query(
+      `ALTER TABLE analytics_events ADD COLUMN IF NOT EXISTS scroll_depth smallint`
+    );
     await client.query(
       `ALTER TABLE analytics_conversions ADD COLUMN IF NOT EXISTS person_id uuid REFERENCES analytics_people(id) ON DELETE SET NULL`
     );
@@ -984,6 +990,14 @@ async function setAnalyticsPixelDomainState(input) {
       404
     );
 }
+var MAX_ENGAGED_MS = 30 * 60 * 1e3;
+var ENGAGED_SESSION_MS = 1e4;
+function boundedEngagedMs(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.min(Math.trunc(value), MAX_ENGAGED_MS) : null;
+}
+function boundedScrollDepth(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.min(Math.trunc(value), 100) : null;
+}
 var blockedPropertyName = /(email|phone|password|token|secret|authorization|cookie|name|address|form|body)/i;
 function normalizeAnalyticsPath(value) {
   if (!value) return "/";
@@ -1172,10 +1186,11 @@ async function ingestAnalyticsEvents(input) {
              id, event_id, site_id, pixel_id, hostname, visitor_id, session_id, event_name,
              occurred_at, path, canonical_url, title, referrer, source, medium, campaign,
              device_class, properties, country_code, region_code, click_ids, person_id,
-             channel_family, platform, campaign_id, ad_set_id, ad_id, creative_id, placement, utm_term, utm_content
+             channel_family, platform, campaign_id, ad_set_id, ad_id, creative_id, placement, utm_term, utm_content,
+             engaged_ms, scroll_depth
            ) VALUES (
              $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18::jsonb, $19, $20, $21::jsonb, $22,
-             $23, $24, $25, $26, $27, $28, $29, $30, $31
+             $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33
            ) ON CONFLICT(site_id, event_id) DO NOTHING`,
           [
             randomUUID(),
@@ -1208,7 +1223,9 @@ async function ingestAnalyticsEvents(input) {
             attribution.creativeId,
             attribution.placement,
             attribution.term,
-            attribution.content
+            attribution.content,
+            boundedEngagedMs(event.engagedMs),
+            boundedScrollDepth(event.scrollDepth)
           ]
         );
         if (inserted.rowCount) accepted += 1;
@@ -1726,10 +1743,37 @@ async function analyticsChannelBreakdown(siteId, userId, filters) {
     dataFreshThrough: (/* @__PURE__ */ new Date()).toISOString()
   };
 }
-async function analyticsContent(siteId, userId, filters, page) {
+var ANALYTICS_CONTENT_SORTS = [
+  "path",
+  "visitors",
+  "sessions",
+  "pageviews",
+  "entrances",
+  "exits",
+  "engagement_events",
+  "conversions",
+  "avg_seconds_on_page",
+  "avg_scroll_depth",
+  "bounce_rate",
+  "exit_rate",
+  "engagement_rate",
+  "pageviews_per_session"
+];
+function normalizeContentOptions(options) {
+  const sort = ANALYTICS_CONTENT_SORTS.includes(options?.sort) ? options.sort : "pageviews";
+  return {
+    sort,
+    direction: options?.direction === "asc" ? "asc" : "desc",
+    search: options?.search?.trim().slice(0, 300) || null
+  };
+}
+async function analyticsContent(siteId, userId, filters, page, options) {
   const db = getAnalyticsPool();
   await accessFor(db, siteId, userId);
   const query = eventFilterSql(siteId, filters);
+  const table = normalizeContentOptions(options);
+  const searchParam = `$${query.values.length + 1}`;
+  const order = table.sort === "path" ? `metrics.path ${table.direction === "asc" ? "ASC" : "DESC"}` : `${table.sort} ${table.direction === "asc" ? "ASC" : "DESC"} NULLS LAST, metrics.path ASC`;
   const result = await db.query(
     `WITH filtered AS (
        SELECT e.* FROM analytics_events e WHERE ${query.where}
@@ -1737,7 +1781,7 @@ async function analyticsContent(siteId, userId, filters, page) {
        SELECT path, count(DISTINCT visitor_id)::int AS visitors,
               count(DISTINCT session_id)::int AS sessions,
               count(*) FILTER (WHERE event_name = 'page_view')::int AS pageviews,
-              count(*) FILTER (WHERE event_name <> 'page_view')::int AS engagement_events,
+              count(*) FILTER (WHERE event_name NOT IN ('page_view', 'page_engagement'))::int AS engagement_events,
               count(*)::int AS events
          FROM filtered GROUP BY path
      ), session_bounds AS (
@@ -1745,35 +1789,98 @@ async function analyticsContent(siteId, userId, filters, page) {
               (array_agg(path ORDER BY occurred_at ASC, id ASC))[1] AS entrance_path,
               (array_agg(path ORDER BY occurred_at DESC, id DESC))[1] AS exit_path
          FROM filtered WHERE session_id IS NOT NULL GROUP BY session_id
+     ), session_pageviews AS (
+       SELECT session_id, count(*) FILTER (WHERE event_name = 'page_view')::int AS pageviews
+         FROM filtered WHERE session_id IS NOT NULL GROUP BY session_id
+     ), page_dwell AS (
+       SELECT session_id, path,
+              sum(engaged_ms)::bigint AS engaged_ms,
+              max(scroll_depth)::int AS scroll_depth
+         FROM filtered
+        WHERE event_name = 'page_engagement' AND session_id IS NOT NULL
+        GROUP BY session_id, path
+     ), dwell_metrics AS (
+       SELECT path,
+              avg(engaged_ms) FILTER (WHERE engaged_ms IS NOT NULL) AS avg_engaged_ms,
+              avg(scroll_depth) FILTER (WHERE scroll_depth IS NOT NULL) AS avg_scroll,
+              count(*) FILTER (WHERE engaged_ms IS NOT NULL)::int AS measured_pageviews
+         FROM page_dwell GROUP BY path
      ), entrances AS (
        SELECT entrance_path AS path, count(*)::int AS entrances FROM session_bounds GROUP BY entrance_path
      ), exits AS (
        SELECT exit_path AS path, count(*)::int AS exits FROM session_bounds GROUP BY exit_path
+     ), bounces AS (
+       SELECT b.entrance_path AS path, count(*)::int AS bounces
+         FROM session_bounds b JOIN session_pageviews p USING(session_id)
+        WHERE p.pageviews <= 1
+        GROUP BY b.entrance_path
+     ), converted_sessions AS (
+       SELECT DISTINCT c.session_id
+         FROM analytics_conversions c
+        WHERE c.site_id = $1 AND c.occurred_at >= $2::timestamptz AND c.occurred_at < $3::timestamptz
+          AND c.session_id IS NOT NULL
+     ), path_sessions AS (
+       SELECT DISTINCT path, session_id FROM filtered WHERE session_id IS NOT NULL
+     ), engaged AS (
+       SELECT ps.path,
+              count(*) FILTER (
+                WHERE sp.pageviews >= 2
+                   OR COALESCE(pd.engaged_ms, 0) >= ${ENGAGED_SESSION_MS}
+                   OR cs.session_id IS NOT NULL
+              )::int AS engaged_sessions
+         FROM path_sessions ps
+         JOIN session_pageviews sp USING(session_id)
+         LEFT JOIN page_dwell pd ON pd.session_id = ps.session_id AND pd.path = ps.path
+         LEFT JOIN converted_sessions cs ON cs.session_id = ps.session_id
+        GROUP BY ps.path
      ), assisted AS (
        SELECT f.path, count(DISTINCT c.id)::int AS conversions
          FROM analytics_conversions c JOIN filtered f ON f.session_id = c.session_id
         WHERE c.site_id = $1 AND c.occurred_at >= $2::timestamptz AND c.occurred_at < $3::timestamptz
         GROUP BY f.path
      )
-     SELECT metrics.*, COALESCE(entrances.entrances, 0)::int AS entrances,
+     SELECT metrics.*,
+            COALESCE(entrances.entrances, 0)::int AS entrances,
             COALESCE(exits.exits, 0)::int AS exits,
-            COALESCE(assisted.conversions, 0)::int AS conversions
+            COALESCE(assisted.conversions, 0)::int AS conversions,
+            COALESCE(dwell.measured_pageviews, 0)::int AS measured_pageviews,
+            CASE WHEN dwell.avg_engaged_ms IS NULL THEN NULL
+                 ELSE round((dwell.avg_engaged_ms / 1000.0)::numeric, 1) END AS avg_seconds_on_page,
+            CASE WHEN dwell.avg_scroll IS NULL THEN NULL
+                 ELSE round(dwell.avg_scroll::numeric, 1) END AS avg_scroll_depth,
+            CASE WHEN COALESCE(entrances.entrances, 0) = 0 THEN NULL
+                 ELSE round(COALESCE(bounces.bounces, 0)::numeric * 100 / entrances.entrances, 1) END AS bounce_rate,
+            CASE WHEN metrics.pageviews = 0 THEN NULL
+                 ELSE round(COALESCE(exits.exits, 0)::numeric * 100 / metrics.pageviews, 1) END AS exit_rate,
+            CASE WHEN metrics.sessions = 0 THEN NULL
+                 ELSE round(COALESCE(engaged.engaged_sessions, 0)::numeric * 100 / metrics.sessions, 1) END AS engagement_rate,
+            CASE WHEN metrics.sessions = 0 THEN NULL
+                 ELSE round(metrics.pageviews::numeric / metrics.sessions, 2) END AS pageviews_per_session
        FROM page_metrics metrics
        LEFT JOIN entrances USING(path)
        LEFT JOIN exits USING(path)
        LEFT JOIN assisted USING(path)
-      ORDER BY metrics.pageviews DESC, metrics.events DESC, metrics.path ASC
+       LEFT JOIN bounces USING(path)
+       LEFT JOIN engaged USING(path)
+       LEFT JOIN dwell_metrics dwell USING(path)
+      WHERE ${searchParam}::text IS NULL OR metrics.path ILIKE '%' || ${searchParam} || '%'
+      ORDER BY ${order}
       LIMIT 1000`,
-    query.values
+    [...query.values, table.search]
   );
   const paged = paginateRows(
     result.rows,
     page,
-    pageFingerprint("content", siteId, filters)
+    pageFingerprint(
+      `content:${table.sort}:${table.direction}:${table.search ?? ""}`,
+      siteId,
+      filters
+    )
   );
   return {
     content: paged.items,
     pageInfo: paged.pageInfo,
+    table,
     dataFreshThrough: (/* @__PURE__ */ new Date()).toISOString()
   };
 }
@@ -2612,7 +2719,7 @@ async function refreshAnalyticsDailyRollups(input) {
   try {
     await client.query("BEGIN");
     await client.query(
-      `DELETE FROM analytics_rollups_daily WHERE bucket >= $1::date AND bucket < $2::date`,
+      `DELETE FROM analytics_rollups_daily WHERE bucket >= $1::date AND bucket <= $2::date`,
       [input.start, input.end]
     );
     const inserted = await client.query(
@@ -2629,7 +2736,14 @@ async function refreshAnalyticsDailyRollups(input) {
               now()
          FROM analytics_events
         WHERE occurred_at >= $1::timestamptz AND occurred_at < $2::timestamptz
-        GROUP BY 1, 2, 3, 4, 5, 6, 7`,
+        GROUP BY 1, 2, 3, 4, 5, 6, 7
+       ON CONFLICT (site_id, pixel_id, hostname, bucket, source, medium, campaign)
+       DO UPDATE SET
+         pageviews = EXCLUDED.pageviews,
+         events = EXCLUDED.events,
+         visitors = EXCLUDED.visitors,
+         sessions = EXCLUDED.sessions,
+         refreshed_at = EXCLUDED.refreshed_at`,
       [input.start, input.end]
     );
     await client.query("COMMIT");
@@ -2641,9 +2755,17 @@ async function refreshAnalyticsDailyRollups(input) {
     return { runId, rows };
   } catch (error) {
     await client.query("ROLLBACK");
+    const pgCode = error?.code;
+    const detail = error instanceof Error ? error.message : String(error);
+    console.error(JSON.stringify({
+      event: "analytics_rollup_failed",
+      runId,
+      pgCode: pgCode ?? null,
+      message: detail.slice(0, 400)
+    }));
     await db.query(
-      `UPDATE analytics_rollup_runs SET status = 'failed', completed_at = now(), error_code = 'rollup_failed' WHERE id = $1`,
-      [runId]
+      `UPDATE analytics_rollup_runs SET status = 'failed', completed_at = now(), error_code = $2 WHERE id = $1`,
+      [runId, (pgCode ? `pg_${pgCode}` : "rollup_failed").slice(0, 60)]
     );
     throw error;
   } finally {
@@ -2696,6 +2818,24 @@ function rowsToCsv(rows) {
     )
   ].join("\n");
 }
+function markdownHeading(column) {
+  return column.split("_").map((word, index) => index ? word : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
+}
+function markdownCell(value) {
+  if (value === null || value === void 0) return "\u2014";
+  return String(value).replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+function rowsToMarkdown(rows) {
+  const columns = [...new Set(rows.flatMap((row) => Object.keys(row)))];
+  if (!columns.length) return ["No rows matched this window and filter set."];
+  return [
+    `| ${columns.map(markdownHeading).join(" | ")} |`,
+    `| ${columns.map(() => "---").join(" | ")} |`,
+    ...rows.map(
+      (row) => `| ${columns.map((column) => markdownCell(row[column])).join(" | ")} |`
+    )
+  ];
+}
 async function createAnalyticsExport(input) {
   const site = (await listAnalyticsSites(input.userId)).find(
     (candidate) => candidate.id === input.siteId
@@ -2763,6 +2903,7 @@ async function createAnalyticsExport(input) {
       `- Verified conversions: ${overview.conversions}`,
       `- Verified revenue: ${(overview.revenueMinor / 100).toFixed(2)}`,
       "",
+      ...input.report === "overview" ? [] : [`## ${markdownHeading(input.report)} rows`, "", ...rowsToMarkdown(rows), ""],
       `Generated ${(/* @__PURE__ */ new Date()).toISOString()} from the governed ${input.report} report contract.`
     ].join("\n");
   }
@@ -2820,6 +2961,8 @@ export {
   listAnalyticsPixels,
   updateAnalyticsPixel,
   setAnalyticsPixelDomainState,
+  MAX_ENGAGED_MS,
+  ENGAGED_SESSION_MS,
   normalizeAnalyticsPath,
   normalizeAnalyticsUrl,
   sanitizeAnalyticsProperties,
@@ -2829,6 +2972,8 @@ export {
   analyticsTimeseries,
   analyticsAcquisition,
   analyticsChannelBreakdown,
+  ANALYTICS_CONTENT_SORTS,
+  normalizeContentOptions,
   analyticsContent,
   analyticsEventCounts,
   analyticsConversions,
@@ -2862,4 +3007,4 @@ export {
   refreshAnalyticsDailyRollupsIfDue,
   createAnalyticsExport
 };
-//# sourceMappingURL=chunk-YW2LXLDU.js.map
+//# sourceMappingURL=chunk-5BEPOFKG.js.map