core-outline 1.1.21 → 1.1.23

package/docs/changelog/270626-143000_tracking_pipeline_changelog.md

@@ -0,0 +1,44 @@
+# Tracking Pipeline — react-component Changes
+Date: 2026-06-27
+
+## Summary
+Replaced Socket.IO with direct HTTP calls to the Hermes ingest API. Added warehouse_id-based routing, improved event schema, and removed dev artifacts.
+
+## Breaking Changes
+- `<CoreOutline>` now requires a `warehouse_id` prop (in addition to existing `data_source_id` + `data_source_secret`)
+- The `data_source_secret` is now validated against Hermes's Vault store; the old `api.coreoutline.com/data-source/authorize` endpoint is no longer used
+
+## New Files
+- `src/components/CoreOutline/helpers.js` — browser utilities: `getAnonymousId`, `getSessionId`, `getUtmParams`, `detectDeviceType`, `detectOS`, `getBrowserName`, `getPagePath`
+- `src/components/CoreOutline/ingest.js` — HTTP ingest client: `initIngest`, `trackEvent`, `flush`, `sendRrwebBatch`, `stopFlushInterval`
+
+## Modified Files
+
+### `src/components/CoreOutline/CoreOutline.js`
+- Added `warehouse_id` prop
+- Replaced all `socket.send()` calls with `trackEvent()` from `ingest.js`
+- Events now include: UTM params, device type, OS, page_url, page_path, anonymous_id (persistent cross-session), session_id (per-tab via sessionStorage)
+- Removed `getLocation()` / geolocation (replaced by server-side IP geolocation)
+- Removed dev-only `<button>Save Events Locally</button>` from rendered output
+- `flag_item_clicked(item_id)` and `flag_item_purchased(item_id, price)` rewritten to use `trackEvent()`; `flag_item_purchased` now accepts an optional `price` parameter mapped to `value_num`
+- Events auto-flush every 5 seconds and on page unload/visibility change
+- rrweb recordings uploaded via `sendRrwebBatch` at session end
+
+### `package.json`
+- Removed: `socket.io-client` (~100KB bundle savings)
+- Added: `uuid` as explicit dependency (was transitive via rrweb)
+
+## Event Types Emitted
+| event_type | event_category | Destination table |
+|---|---|---|
+| `session_start` | `session` | `fact_session` |
+| `session_end` | `session` | `fact_session` (update via ReplacingMergeTree) |
+| `pageview` | `navigation` | `fact_pageview` + `fact_event` |
+| `item_clicked` | `interaction` | `fact_event` |
+| `product_clicked` | `commerce` | `fact_event` |
+| `product_purchased` | `commerce` | `fact_event` (value_num = price) |
+
+## Data Flow
+react-component → `POST https://api.coreoutline.com/api/ingest/authorize` (get JWT)
+→ `POST https://api.coreoutline.com/api/ingest/events` (batch, every 5s + on unload)
+→ `POST https://api.coreoutline.com/api/ingest/rrweb` (session end)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "core-outline",
-  "version": "1.1.21",
+  "version": "1.1.23",
   "description": "A React component for Core&Outline",
   "main": "dist/index.js",
   "module": "dist/index.es.js",
@@ -47,6 +47,6 @@
     "react-error-boundary": "^5.0.0",
     "react-tracking": "^9.3.2",
     "rrweb": "^2.0.0-alpha.4",
-    "socket.io-client": "^4.8.1"
+    "uuid": "^9.0.0"
   }
 }

package/src/components/CoreOutline/CoreOutline.js

@@ -1,171 +1,125 @@
-import React, { use, useEffect, useRef, useState } from 'react';
+import React, { useEffect, useRef, useState } from 'react';
 import { record } from 'rrweb';
-import { v4 } from 'uuid';
-import socket from './socket';
-
 import {
   EventType,
-  eventWithTime,
   IncrementalSource,
   MouseInteractions,
 } from 'rrweb';
-
-const CoreOutline = ({ children, data_source_id, data_source_secret }) => {
-  const [events, setEvents] = useState([]);
+import {
+  getAnonymousId,
+  getSessionId,
+  getUtmParams,
+  detectDeviceType,
+  detectOS,
+  getBrowserName,
+  getPagePath,
+} from './helpers';
+import {
+  initIngest,
+  trackEvent,
+  flush,
+  sendRrwebBatch,
+  stopFlushInterval,
+} from './ingest';
+
+const CoreOutline = ({ children, data_source_id, data_source_secret, warehouse_id }) => {
   const [replayEvents, setReplayEvents] = useState([]);
   const recorderActive = useRef(false);
   const [currentPage, setCurrentPage] = useState(window.location.href);
-  const [location, setLocation] = useState({ latitude: null, longitude: null });
-  const [browser, setBrowser] = useState(getBrowserInfo());
-  const [appToken, setAppToken] = useState('');
-  const [sessionId, setSessionId] = useState(v4());
+  const pageviewCountRef = useRef(0);
+  const eventCountRef = useRef(0);
+  const sessionStartRef = useRef(Date.now());
+
+  const sessionId = getSessionId();
+  const anonymousId = getAnonymousId();
 
-  async function authorizeApp(data_source_id, data_source_secret) {
-    const requestOptions = {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        data_source_id: data_source_id,
-        data_source_secret: data_source_secret,
-      }),
+  useEffect(() => {
+    const utmParams = getUtmParams();
+    const deviceType = detectDeviceType();
+    const os = detectOS();
+    const browser = getBrowserName();
+    const referrer = document.referrer || 'direct';
+
+    const basePayload = {
+      session_id: sessionId,
+      anonymous_id: anonymousId,
+      account_id: data_source_id,
+      platform: 'web',
+      device_type: deviceType,
+      os,
+      browser,
+      referrer,
+      page_url: window.location.href,
+      page_path: getPagePath(),
+      ...utmParams,
     };
-    const response = fetch(
-      `http://api.coreoutline.com/data-source/authorize`,
-      requestOptions
-    )
-      .then((response) => response.json())
-      .then((data) => {
-        setAppToken(data.app_token);
-        return data;
-      })
-      .catch((error) => {
-        console.error('Error:', error);
-        return error;
-      });
-    return response;
-  }
 
-  const offLoadFunctionality = () => {
-    const formattedEvents = events;
-    const rawEvents = replayEvents;
-    const data = JSON.stringify({
-      data_source_id: data_source_id,
-      sessionId: sessionId,
-      formatted_events: formattedEvents,
-      raw_events: rawEvents,
-    });
+    let initialized = false;
 
-    socket.send({
-      topic: 'session-data',
-      data_source_id: data_source_id,
-      session_id: localStorage.getItem('coreOutlineSessionId'),
-      start_date: new Date().toLocaleString(),
-      latitude: location.latitude,
-      longitude: location.longitude,
-      browser: browser,
-      event_id: 'SESSION_END',
-    });
+    const init = async () => {
+      initialized = await initIngest(warehouse_id, data_source_id, data_source_secret);
+      if (!initialized) return;
 
-    if (navigator.sendBeacon) {
-      navigator.sendBeacon(`http://data.coreoutline.com/upload-events`, data);
-    } else {
-      fetch(`http://data.coreoutline.com/upload-events`, {
-        method: 'POST',
-        body: data,
-        keepalive: true,
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      }).catch((err) => console.error('Upload failed:', err));
-    }
-  };
-  useEffect(() => {}, [events, replayEvents]);
+      pageviewCountRef.current += 1;
+      trackEvent('pageview', { ...basePayload, event_name: 'pageview', event_category: 'navigation' });
+      trackEvent('session_start', {
+        ...basePayload,
+        event_name: 'session_start',
+        event_category: 'session',
+        session_start_ts: new Date().toISOString(),
+      });
+    };
 
-  useEffect(async () => {
-    const handleBeforeUnload = () => {
-      offLoadFunctionality();
+    init();
+
+    const handleSessionEnd = () => {
+      if (!initialized) return;
+      const durationMs = Date.now() - sessionStartRef.current;
+      trackEvent('session_end', {
+        ...basePayload,
+        event_name: 'session_end',
+        event_category: 'session',
+        session_end_ts: new Date().toISOString(),
+        pageviews: pageviewCountRef.current,
+        events_count: eventCountRef.current,
+        duration_ms: durationMs,
+      });
+      flush();
+      stopFlushInterval();
+      sendRrwebBatch(replayEvents, sessionId);
     };
 
     const handleVisibilityChange = () => {
-      if (document.visibilityState === 'hidden') {
-        offLoadFunctionality();
-      }
+      if (document.visibilityState === 'hidden') handleSessionEnd();
     };
-    window.addEventListener('beforeunload', handleBeforeUnload);
+
+    window.addEventListener('beforeunload', handleSessionEnd);
     document.addEventListener('visibilitychange', handleVisibilityChange);
-    await authorizeApp(data_source_id, data_source_secret);
-    localStorage.setItem('coreOutlineSessionId', sessionId);
-    localStorage.setItem('coreOutlineDataSourceIdId', data_source_id);
-    console.log('Session', sessionId);
-    socket.send({
-      topic: 'session-data',
-      data_source_id: data_source_id,
-      session_id: localStorage.getItem('coreOutlineSessionId'),
-      start_date: new Date().toLocaleString(),
-      latitude: location.latitude,
-      longitude: location.longitude,
-      browser: browser,
-      event_id: 'SESSION_START',
-    });
 
     return () => {
-      offLoadFunctionality();
-      window.removeEventListener('beforeunload', handleBeforeUnload);
+      handleSessionEnd();
+      window.removeEventListener('beforeunload', handleSessionEnd);
       document.removeEventListener('visibilitychange', handleVisibilityChange);
-      socket.send({
-        topic: 'session-data',
-        data_source_id: data_source_id,
-        session_id: localStorage.getItem('coreOutlineSessionId'),
-        start_date: new Date().toLocaleString(),
-        latitude: location.latitude,
-        longitude: location.longitude,
-        browser: browser,
-        event_id: 'SESSION_END',
-      });
     };
   }, []);
 
   useEffect(() => {
-    const navigationEvent = {
-      type: 4,
-      data: {
-        href: window.location.href,
-        width: 1536,
-        height: 730,
-      },
-      timestamp: Date.now(),
-    };
-    const startTime = performance.now();
-    const observer = new MutationObserver(() => {
-      const endTime = performance.now();
-      const loadTime = endTime - startTime;
-      navigationEvent.data.loadTime = loadTime;
-      const hours = Math.floor(loadTime / 3600000);
-      const minutes = Math.floor((loadTime % 3600000) / 60000);
-      const seconds = Math.floor((loadTime % 60000) / 1000);
-      navigationEvent.data.loadTimeFormatted = `${hours} hours ${minutes} minutes ${seconds} seconds`;
-      observer.disconnect();
-    });
-    observer.observe(document, { childList: true, subtree: true });
-    setEvents((prevEvents) => [...prevEvents, navigationEvent]);
-    localStorage.setItem(
-      'coreoutlineFormattedEvents',
-      JSON.stringify((prevEvents) => [...prevEvents, navigationEvent])
-    );
+    const utmParams = getUtmParams();
+
     const handleNavigation = () => {
       setCurrentPage(window.location.href);
-      socket.send({
-        topic: 'session-data',
-        data_source_id: data_source_id,
-        session_id: localStorage.getItem('coreOutlineSessionId'),
-        start_date: new Date().toLocaleString(),
-        latitude: location.latitude,
-        longitude: location.longitude,
-        browser: browser,
-        page_name: window.location.href,
-        event_id: 'PAGE_NAVIGATION',
+      pageviewCountRef.current += 1;
+      trackEvent('pageview', {
+        session_id: sessionId,
+        anonymous_id: anonymousId,
+        account_id: data_source_id,
+        platform: 'web',
+        page_url: window.location.href,
+        page_path: getPagePath(),
+        referrer: document.referrer || 'direct',
+        event_name: 'pageview',
+        event_category: 'navigation',
+        ...utmParams,
       });
     };
 
@@ -182,76 +136,22 @@ const CoreOutline = ({ children, data_source_id, data_source_secret }) => {
     };
   }, [currentPage]);
 
-  useEffect(() => {
-    console.log('Location', location);
-    getLocation();
-  }, []);
-
   useEffect(() => {
     let stopFn;
     if (!recorderActive.current) {
       stopFn = record({
         emit(event) {
-          setReplayEvents((prevEvents) => [...prevEvents, event]);
-          localStorage.setItem(
-            'coreoutlineRawEvents',
-            JSON.stringify((prevEvents) => [...replayEvents, event])
-          );
+          setReplayEvents((prev) => [...prev, event]);
         },
         maskAllInputs: false,
         maskTextSelector: null,
       });
-
       recorderActive.current = true;
     }
 
     return () => {
       if (stopFn) {
         stopFn();
-
-        recorderActive.current = false;
-      }
-    };
-  }, []);
-
-  useEffect(() => {
-    let stopFn;
-    if (!recorderActive.current) {
-      stopFn = record({
-        emit(event) {
-          console.log('Event:', event);
-          if (event.type == 3 && event.data.type == 2) {
-            return;
-          }
-          if (event.type == 4) {
-            return;
-          }
-          if (event.type == 3 && event.data.source == 0) {
-            return;
-          }
-          if (event.type == 3 && event.data.source == 1) {
-            return;
-          }
-          if (event.type == 3 && event.data.type == 3) {
-            console.log('Contextmenu Event:', event);
-          }
-          setEvents((prevEvents) => [...prevEvents, event]);
-          localStorage.setItem(
-            'coreoutlineFormattedEvents',
-            JSON.stringify((prevEvents) => [...prevEvents, event])
-          );
-        },
-        maskAllInputs: false,
-        maskTextSelector: null,
-      });
-
-      recorderActive.current = true;
-    }
-
-    return () => {
-      if (stopFn) {
-        stopFn();
-
         recorderActive.current = false;
       }
     };
@@ -259,180 +159,63 @@ const CoreOutline = ({ children, data_source_id, data_source_secret }) => {
 
   useEffect(() => {
     const handleClick = (event) => {
-      console.log('Inner text:', event.target.innerText);
-      const clickEvent = {
-        type: EventType.IncrementalSnapshot,
-        data: {
-          source: IncrementalSource.MouseInteraction,
-          type: MouseInteractions.Click,
-          id: event.target.id,
-          x: event.clientX,
-          y: event.clientY,
-        },
-        timestamp: Date.now(),
-      };
-      const clickedElement = document.elementFromPoint(
-        clickEvent.data.x,
-        clickEvent.data.y
-      );
-
-      if (clickedElement) {
-        clickEvent.data.metadata = {
-          label:
-            clickedElement.getAttribute('data-label') ||
-            clickedElement.innerText ||
-            null,
-          value:
-            clickedElement.getAttribute('value') ||
-            clickedElement.innerText ||
-            null,
-          id:
-            clickedElement.getAttribute('id') ||
-            clickedElement.innerText ||
-            null,
-          name:
-            clickedElement.getAttribute('name') ||
-            clickedElement.innerText ||
-            null,
-          class:
-            clickedElement.getAttribute('class') ||
-            clickedElement.innerText ||
-            null,
-          tag: clickedElement.tagName,
-          html: String(clickedElement.getHTML()),
-          innerText: event.target.innerText,
-        };
-      }
-      setEvents((prevEvents) => [...prevEvents, clickEvent]);
-      localStorage.setItem(
-        'coreoutlineFormattedEvents',
-        JSON.stringify((prevEvents) => [...prevEvents, clickEvent])
-      );
-      socket.send({
-        topic: 'session-data',
-        data_source_id: data_source_id,
-        session_id: localStorage.getItem('coreOutlineSessionId'),
-        start_date: new Date().toLocaleString(),
-        latitude: location.latitude,
-        longitude: location.longitude,
-        browser: browser,
-        event_id: 'ITEM_CLICKED',
-        // item_clicked: event.target,
+      const clickedElement = document.elementFromPoint(event.clientX, event.clientY);
+      const metadata = clickedElement
+        ? {
+            label: clickedElement.getAttribute('data-label') || clickedElement.innerText || null,
+            value: clickedElement.getAttribute('value') || clickedElement.innerText || null,
+            id: clickedElement.getAttribute('id') || null,
+            name: clickedElement.getAttribute('name') || null,
+            class: clickedElement.getAttribute('class') || null,
+            tag: clickedElement.tagName,
+            inner_text: event.target.innerText || null,
+          }
+        : {};
+
+      eventCountRef.current += 1;
+      trackEvent('item_clicked', {
+        session_id: sessionId,
+        anonymous_id: anonymousId,
+        account_id: data_source_id,
+        platform: 'web',
+        page_url: window.location.href,
+        page_path: getPagePath(),
+        event_name: 'item_clicked',
+        event_category: 'interaction',
+        properties_json: JSON.stringify(metadata),
       });
     };
 
     document.addEventListener('click', handleClick);
-
-    return () => {
-      document.removeEventListener('click', handleClick);
-    };
+    return () => document.removeEventListener('click', handleClick);
   }, []);
 
-  function saveEventsLocally(type = 'formatted' || 'raw') {
-    let blob = '';
-    if (type === 'formatted') {
-      blob = new Blob([JSON.stringify(events, null, 2)], {
-        type: 'application/json',
-      });
-    } else if (type === 'raw') {
-      blob = new Blob([JSON.stringify(replayEvents, null, 2)], {
-        type: 'application/json',
-      });
-    }
-    const url = URL.createObjectURL(blob);
-    const a = document.createElement('a');
-    a.href = url;
-    a.download = `${type}_events.json`;
-    a.click();
-    URL.revokeObjectURL(url);
-  }
-
-  function getBrowserInfo() {
-    const userAgent = navigator.userAgent;
-    let browserName = 'Unknown';
-
-    if (userAgent.indexOf('Firefox') > -1) {
-      browserName = 'Firefox';
-    } else if (
-      userAgent.indexOf('Opera') > -1 ||
-      userAgent.indexOf('OPR') > -1
-    ) {
-      browserName = 'Opera';
-    } else if (userAgent.indexOf('Chrome') > -1) {
-      browserName = 'Chrome';
-    } else if (userAgent.indexOf('Safari') > -1) {
-      browserName = 'Safari';
-    } else if (
-      userAgent.indexOf('MSIE') > -1 ||
-      userAgent.indexOf('Trident/') > -1
-    ) {
-      browserName = 'Internet Explorer';
-    }
-
-    return browserName;
-  }
-
-  function getLocation() {
-    if (navigator.geolocation) {
-      navigator.geolocation.getCurrentPosition(
-        (position) => {
-          setLocation({
-            latitude: position.coords.latitude,
-            longitude: position.coords.longitude,
-          });
-        },
-        (error) => {
-          console.error('Error getting location:', error);
-        }
-      );
-    } else {
-      console.error('Geolocation is not supported by this browser.');
-    }
-  }
-
-  return (
-    <div>
-      <button
-        onClick={() => {
-          saveEventsLocally('formatted');
-          saveEventsLocally('raw');
-        }}
-      >
-        Save Events Locally
-      </button>
-      {children}
-    </div>
-  );
+  return <>{children}</>;
 };
 
 export const flag_item_clicked = (item_id) => {
-  const sessionId = localStorage.getItem('coreOutlineSessionId');
-  const dataSourceId = localStorage.getItem('coreOutlineDataSourceIdId');
-  console.log('Session', sessionId);
-  console.log('Data Source', dataSourceId);
-  socket.send({
-    topic: 'session-data',
-    data_source_id: dataSourceId,
-    session_id: localStorage.getItem('coreOutlineSessionId'),
-    start_date: new Date().toLocaleString(),
-    event_id: 'PRODUCT_CLICKED',
-    item_clicked: item_id,
+  const sessionId = sessionStorage.getItem('co_session_id') || '';
+  const anonymousId = localStorage.getItem('co_anon_id') || '';
+  trackEvent('product_clicked', {
+    session_id: sessionId,
+    anonymous_id: anonymousId,
+    event_name: 'product_clicked',
+    event_category: 'commerce',
+    feature_key: item_id,
   });
   return { status: 'success', message: 'Item click flagged successfully.' };
 };
 
-export const flag_item_purchased = (item_id) => {
-  const sessionId = localStorage.getItem('coreOutlineSessionId');
-  const dataSourceId = localStorage.getItem('coreOutlineDataSourceIdId');
-  console.log('Session', sessionId);
-  console.log('Data Source', dataSourceId);
-  socket.send({
-    topic: 'session-data',
-    data_source_id: dataSourceId,
-    session_id: localStorage.getItem('coreOutlineSessionId'),
-    start_date: new Date().toLocaleString(),
-    event_id: 'PRODUCT_PURCHASED',
-    item_clicked: item_id,
+export const flag_item_purchased = (item_id, price = null) => {
+  const sessionId = sessionStorage.getItem('co_session_id') || '';
+  const anonymousId = localStorage.getItem('co_anon_id') || '';
+  trackEvent('product_purchased', {
+    session_id: sessionId,
+    anonymous_id: anonymousId,
+    event_name: 'product_purchased',
+    event_category: 'commerce',
+    feature_key: item_id,
+    value_num: price != null ? Number(price) : null,
   });
   return { status: 'success', message: 'Item purchase flagged successfully.' };
 };