dashcam 0.8.4 → 1.0.1-beta.11

package/lib/extension-logs/index.js

@@ -0,0 +1,347 @@
+import path from 'path';
+import { logger } from '../logger.js';
+import { jsonl } from '../utilities/jsonl.js';
+import {
+  verifyPattern,
+  updateTabsState,
+  shouldCountEvent,
+  eventTypeToStatType,
+  sanitizeWebLogEventPayload,
+} from './helpers.js';
+
+// Simple ref implementation for reactivity (adapted from Vue's ref)
+function ref(value) {
+  return {
+    value,
+    _isRef: true
+  };
+}
+
+function computed(fn) {
+  // Don't cache - always recompute to ensure reactivity
+  return {
+    get value() {
+      return fn();
+    },
+    _isComputed: true
+  };
+}
+
+function effect(fn) {
+  fn();
+  return () => {}; // Simplified cleanup
+}
+
+// Simple getStats function (same as used in FileTracker)
+function getStats(eventTimes = []) {
+  const endTime = Date.now();
+  const startTime = Date.now() - 60000;
+
+  let startIndex = 0;
+  let count = 0;
+
+  for (const time of eventTimes) {
+    if (time < startTime) startIndex++;
+    else if (time <= endTime) {
+      count++;
+    } else break;
+  }
+
+  return {
+    eventTimes: eventTimes.slice(startIndex),
+    count,
+  };
+}
+
+const filename = 'dashcam_logs_web_events.jsonl';
+
+class WebLogsTracker {
+  // Omitting a directory puts it in a watch only mode where it
+  // only collect per minute stats
+  constructor({ config, webTrackerManager, directory }) {
+    logger.debug('WebLogsTracker: Initializing', { 
+      isWatchOnly: !directory, 
+      configKeys: Object.keys(config),
+      configCount: Object.keys(config).length
+    });
+    
+    // if watchOnly, the configs in the argument will be a ref
+    // but if tracked for a recording the configs will be static
+    // we wrap it in a ref just for a consistent way to access it
+    this.tabs = {};
+    this.events = new Set();
+    this.config = ref(config);
+    this.trackedPatterns = {};
+    this.eventTimesByUrl = {};
+    this.lastActiveTabId = null;
+    this.isWatchOnly = !directory;
+    this.webTrackerManager = webTrackerManager;
+    this.fullLogs = { tabs: {}, activeTabs: [] };
+    this.fileLocation = this.isWatchOnly ? '' : path.join(directory, filename);
+    
+    this.patterns = computed(
+      () =>
+        new Set(
+          Object.values(this.config.value)
+            .map((config) => config.patterns)
+            .flat()
+        )
+    );
+
+    logger.debug('WebLogsTracker: Subscribing to global tabs and navigation events');
+    this.unsubscribeFromGlobalTabsAndNavigation =
+      this.webTrackerManager.subscribeToGlobalTabsAndNavigation((event) =>
+        this.#handleEvent(event)
+      );
+
+    this.watchPatternsEffect = effect(() => {
+      const patternsToRemove = Object.keys(this.trackedPatterns).filter(
+        (pattern) => !this.patterns.value.has(pattern)
+      );
+      const newPatterns = [...this.patterns.value].filter(
+        (pattern) => !this.trackedPatterns[pattern]
+      );
+
+      logger.debug('WebLogsTracker: Updating pattern subscriptions', { 
+        patternsToRemove: patternsToRemove.length, 
+        newPatterns: newPatterns.length,
+        totalPatterns: this.patterns.value.size
+      });
+
+      newPatterns.forEach((pattern) => this.#startWatchingPattern(pattern));
+      patternsToRemove.forEach((pattern) => this.#stopWatchingPattern(pattern));
+    });
+  }
+
+  updateConfig(config) {
+    this.config.value = config;
+  }
+
+  #startWatchingPattern(pattern) {
+    if (this.trackedPatterns[pattern]) {
+      logger.debug('WebLogsTracker: Pattern already being watched', { pattern });
+      return;
+    }
+    
+    logger.debug('WebLogsTracker: Starting to watch pattern', { pattern });
+    this.trackedPatterns[pattern] = this.webTrackerManager.subscribe(
+      pattern,
+      (event) => this.#handleEvent(event)
+    );
+  }
+
+  #stopWatchingPattern(pattern) {
+    const unsubscribe = this.trackedPatterns[pattern];
+    if (unsubscribe) {
+      logger.debug('WebLogsTracker: Stopping watch for pattern', { pattern });
+      unsubscribe();
+      Object.keys(this.eventTimesByUrl)
+        .filter((shortUrl) => verifyPattern(pattern, shortUrl))
+        .forEach((shortUrl) => {
+          logger.debug('WebLogsTracker: Removing event times for URL', { shortUrl, pattern });
+          delete this.eventTimesByUrl[shortUrl];
+        });
+      delete this.trackedPatterns[pattern];
+    } else {
+      logger.debug('WebLogsTracker: Pattern not being watched', { pattern });
+    }
+  }
+
+  getStatus() {
+    const statuses = Object.values(this.config.value).map(
+      ({ id, type, name, patterns }) => {
+        const items = Object.values(this.eventTimesByUrl)
+          .filter(({ shortUrl }) =>
+            patterns.some((pattern) => verifyPattern(pattern, shortUrl))
+          )
+          .map(({ stats, shortUrl, logsCount, errorsCount, networkCount }) => {
+            if (this.isWatchOnly) {
+              const { eventTimes: l, count: lCount } = getStats(stats.logs);
+              const { eventTimes: e, count: eCount } = getStats(stats.errors);
+              const { eventTimes: n, count: nCount } = getStats(stats.network);
+
+              stats.logs = l;
+              stats.errors = e;
+              stats.network = n;
+              logsCount = lCount;
+              errorsCount = eCount;
+              networkCount = nCount;
+            }
+
+            return {
+              item: shortUrl,
+              count: logsCount + errorsCount + networkCount,
+              counts: {
+                logs: logsCount,
+                errors: errorsCount,
+                network: networkCount,
+              },
+            };
+          });
+
+        return {
+          id,
+          type,
+          name,
+          items,
+          fileLocation: this.fileLocation,
+          count: items.reduce((sum, item) => sum + item.count, 0),
+        };
+      }
+    );
+
+    return statuses;
+  }
+
+  #handleEvent(event) {
+    if (this.events.has(event)) {
+      logger.debug('WebLogsTracker: Ignoring duplicate event', { type: event.type });
+      return;
+    }
+    this.events.add(event);
+
+    logger.debug('WebLogsTracker: Processing event', { 
+      type: event.type, 
+      time: event.time,
+      tabId: event.payload?.tabId,
+      url: event.payload?.url?.substring(0, 100)
+    });
+
+    this.tabs = updateTabsState(event, this.tabs);
+
+    const { type, time, payload } = event;
+    const patterns = [...this.patterns.value];
+
+    let newEvent = { ...event };
+    switch (type) {
+      case 'INITIAL_TABS':
+        newEvent.payload = event.payload.filter((tab) =>
+          patterns.some((pattern) => verifyPattern(pattern, tab.url))
+        );
+        logger.debug('WebLogsTracker: Filtered initial tabs', { 
+          originalCount: event.payload.length, 
+          filteredCount: newEvent.payload.length 
+        });
+        break;
+
+      case 'TAB_REMOVED':
+        logger.debug('WebLogsTracker: Tab removed event, marking as null');
+        newEvent = null;
+        break;
+
+      case 'TAB_ACTIVATED':
+        const tabId = patterns.some((pattern) =>
+          verifyPattern(pattern, this.tabs[payload.tabId]?.url)
+        )
+          ? payload.tabId
+          : null;
+
+        if (tabId !== this.lastActiveTabId) {
+          logger.debug('WebLogsTracker: Active tab changed', { 
+            from: this.lastActiveTabId, 
+            to: tabId,
+            url: this.tabs[tabId]?.url?.substring(0, 100)
+          });
+          this.lastActiveTabId = tabId;
+          if (!tabId) {
+            newEvent.payload = { tabId: null, windowId: null };
+          }
+        } else {
+          logger.debug('WebLogsTracker: Active tab unchanged, ignoring event');
+          newEvent = null;
+        }
+        break;
+
+      case 'NAVIGATION_STARTED':
+      case 'NAVIGATION_COMPLETED':
+        if (
+          !patterns.some((ptrn) =>
+            verifyPattern(ptrn, this.tabs[payload.tabId]?.url)
+          )
+        ) {
+          logger.debug('WebLogsTracker: Navigation event does not match patterns, ignoring', { 
+            type, 
+            url: this.tabs[payload.tabId]?.url?.substring(0, 100)
+          });
+          newEvent = null;
+        } else {
+          logger.debug('WebLogsTracker: Navigation event matches pattern', { 
+            type, 
+            url: this.tabs[payload.tabId]?.url?.substring(0, 100)
+          });
+        }
+        break;
+
+      case 'LOG_EVENT':
+      case 'LOG_ERROR':
+      case 'NETWORK_BEFORE_REQUEST':
+      case 'NETWORK_COMPLETED_REQUEST':
+      case 'NETWORK_RESPONSE':
+      case 'NETWORK_RESPONSE_BODY':
+        // These events are already filtered by pattern in WebTrackerManager
+        // They should be written as-is
+        logger.debug('WebLogsTracker: Processing web event', { 
+          type, 
+          tabId: payload?.tabId,
+          url: this.tabs[payload.tabId]?.url?.substring(0, 100)
+        });
+        break;
+        
+      default:
+        logger.debug('WebLogsTracker: Unknown event type, passing through', { type });
+        break;
+    }
+
+    if (!newEvent) {
+      logger.debug('WebLogsTracker: Event filtered out, not processing further');
+      return;
+    }
+
+    if (!this.isWatchOnly) {
+      logger.debug('WebLogsTracker: Writing event to file', { fileLocation: this.fileLocation });
+      jsonl.append(this.fileLocation, {
+        ...newEvent,
+        payload: sanitizeWebLogEventPayload(newEvent.payload),
+      });
+    }
+
+    if (this.tabs[payload.tabId] && shouldCountEvent(type)) {
+      const shortUrl = this.tabs[payload.tabId].url.split('?')[0];
+      this.eventTimesByUrl[shortUrl] ??= {
+        shortUrl,
+        stats: {
+          logs: [],
+          errors: [],
+          network: [],
+        },
+        logsCount: 0,
+        errorsCount: 0,
+        networkCount: 0,
+      };
+
+      const statType = eventTypeToStatType[type];
+      if (statType) {
+        this.eventTimesByUrl[shortUrl][`${statType}Count`]++;
+        this.eventTimesByUrl[shortUrl].stats[statType].push(time);
+        logger.debug('WebLogsTracker: Updated stats for URL', { 
+          shortUrl, 
+          statType, 
+          newCount: this.eventTimesByUrl[shortUrl][`${statType}Count`]
+        });
+      }
+    }
+  }
+
+  destroy() {
+    const status = this.getStatus();
+    // stop(this.watchPatternsEffect); // Simplified for CLI
+    this.events.clear();
+    this.unsubscribeFromGlobalTabsAndNavigation();
+    Object.keys(this.trackedPatterns).forEach((pattern) => {
+      this.#stopWatchingPattern(pattern);
+    });
+    return status;
+  }
+}
+
+export { WebLogsTracker };

package/lib/extension-logs/manager.js

@@ -0,0 +1,344 @@
+import { ref } from '../websocket/server.js';
+import { verifyPattern, updateTabsState } from './helpers.js';
+import { logger } from '../logger.js';
+
+const endMessage = {
+  type: 'STOP_RECORDING',
+};
+
+// Simple reactive implementations (simplified versions for CLI)
+function reactive(obj) {
+  return obj;
+}
+
+function computed(fn) {
+  return { get value() { return fn(); } };
+}
+
+function effect(fn) {
+  fn();
+  return fn;
+}
+
+class WebTrackerManager {
+  constructor(server) {
+    this.tabs = {};
+    this.cleanups = [];
+    this.server = server;
+    this.isListening = ref(server.isListening?.value || false);
+    this.patternsByCallback = reactive(new Map());
+    this.globalTabsAndNavigationCallbacks = reactive(new Set());
+    this.eventCallbacks = computed(
+      () => new Set(this.patternsByCallback.keys())
+    );
+
+    this.watchCallbacksSizeEffect = effect(() => {
+      const size =
+        this.eventCallbacks.value.size +
+        this.globalTabsAndNavigationCallbacks.size;
+      const isListening = this.isListening.value;
+      logger.debug('WebTrackerManager: Effect triggered', { size, isListening, shouldStart: isListening && size > 0 });
+      if (!isListening || size === 0) {
+        this.#stop();
+      } else this.#start();
+    });
+  }
+
+  get isStarted() {
+    return this.cleanups.length > 0 && this.isListening.value;
+  }
+
+  #start() {
+    if (!this.isListening.value) {
+      logger.debug('WebTrackerManager: Not starting because WebSocket server is not listening');
+      return;
+    }
+    
+    if (this.cleanups.length > 0) {
+      logger.debug('WebTrackerManager: Already started, skipping duplicate start');
+      return;
+    }
+    
+    // Collect all patterns from the patternsByCallback map
+    const patterns = Array.from(new Set(this.patternsByCallback.values()));
+    logger.info('WebTrackerManager: Starting tracking...', { patterns, callbackCount: this.patternsByCallback.size });
+    
+    const startMessage = {
+      type: 'START_RECORDING',
+      payload: patterns.length > 0 ? patterns : ['*'], // Default to all if no patterns
+    };
+    
+    this.server.broadcast(startMessage);
+    logger.info('WebTrackerManager: Broadcasted start message to all clients', { patterns });
+
+    const messageCleanup = this.server.on('message', (event) => {
+      logger.debug('WebTrackerManager: Received message from server', { eventType: event.type, hasPayload: !!event.payload });
+      this.#handleEvent(event);
+    });
+    const connectionCleanup = this.server.on('connection', (client) => {
+      // Send current patterns to the new client
+      const patterns = Array.from(new Set(this.patternsByCallback.values()));
+      logger.info('WebTrackerManager: New client connected, sending start message', { 
+        patterns, 
+        callbackCount: this.patternsByCallback.size 
+      });
+      const startMessage = {
+        type: 'START_RECORDING',
+        payload: patterns.length > 0 ? patterns : ['*'],
+      };
+      this.server.send(client, startMessage);
+    });
+
+    this.cleanups.push(messageCleanup, connectionCleanup);
+    logger.debug('WebTrackerManager: Started successfully, registered event handlers');
+  }
+
+  #stop() {
+    logger.debug('WebTrackerManager: Stopping tracking...');
+    if (this.isListening.value) {
+      this.server.broadcast(endMessage);
+      logger.debug('WebTrackerManager: Broadcasted stop message to all clients');
+    }
+    
+    this.cleanups.forEach((cleanupFn) => cleanupFn());
+    this.cleanups = [];
+    logger.debug('WebTrackerManager: Stopped and cleaned up event handlers');
+  }
+
+  subscribeToGlobalTabsAndNavigation(callback) {
+    logger.debug('WebTrackerManager: Subscribing to global tabs/navigation events');
+    this.globalTabsAndNavigationCallbacks.add(callback);
+    this.#tryCallback(callback, this.#getInitialTabs());
+    // Manually trigger effect since we're using a simplified effect implementation
+    const size =
+      this.eventCallbacks.value.size +
+      this.globalTabsAndNavigationCallbacks.size;
+    if (this.isListening.value && size > 0) {
+      this.#start();
+    }
+    return () => this.unsubscribeFromGlobalTabsAndNavigation(callback);
+  }
+
+  unsubscribeFromGlobalTabsAndNavigation(callback) {
+    logger.debug('WebTrackerManager: Unsubscribing from global tabs/navigation events');
+    this.globalTabsAndNavigationCallbacks.delete(callback);
+  }
+
+  subscribe(pattern, callback) {
+    logger.info('WebTrackerManager: Subscribing to pattern', { pattern, isStarted: this.isStarted });
+    this.patternsByCallback.set(callback, pattern);
+    
+    // If already started, send updated patterns to all clients
+    if (this.cleanups.length > 0 && this.isListening.value) {
+      const patterns = Array.from(new Set(this.patternsByCallback.values()));
+      logger.info('WebTrackerManager: Sending updated patterns to clients', { patterns, callbackCount: this.patternsByCallback.size });
+      const startMessage = {
+        type: 'START_RECORDING',
+        payload: patterns,
+      };
+      this.server.broadcast(startMessage);
+    }
+    
+    // Manually trigger effect since we're using a simplified effect implementation
+    const size =
+      this.eventCallbacks.value.size +
+      this.globalTabsAndNavigationCallbacks.size;
+    if (this.isListening.value && size > 0) {
+      this.#start();
+    }
+    return () => this.unsubscribe(callback);
+  }
+
+  unsubscribe(callback) {
+    const pattern = this.patternsByCallback.get(callback);
+    logger.debug('WebTrackerManager: Unsubscribing from pattern', { pattern });
+    this.patternsByCallback.delete(callback);
+  }
+
+  /**
+   * Get the DOM from a webpage through the Chrome extension
+   * @param {Object} options - Execution options
+   * @param {number} [options.tabId] - Specific tab ID (optional, uses active tab if not provided)
+   * @param {string} [options.url] - URL pattern to match (optional)
+   * @param {boolean} [options.allFrames=false] - Get from all frames
+   * @returns {Promise<Object>} - Resolves with {html, title, url} from the page
+   */
+  getPageDOM({ tabId, url, allFrames = false } = {}) {
+    return new Promise((resolve, reject) => {
+      if (!this.isListening.value) {
+        logger.error('WebTrackerManager: Cannot execute script, WebSocket server is not listening');
+        return reject(new Error('WebSocket server is not listening'));
+      }
+
+      const requestId = `getdom_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+      const message = {
+        type: 'GET_PAGE_DOM',
+        requestId,
+        payload: {
+          tabId,
+          url,
+          allFrames,
+        },
+      };
+
+      logger.info('WebTrackerManager: Sending get page DOM request', { 
+        requestId, 
+        tabId, 
+        url
+      });
+
+      // Set up one-time listener for the response
+      const timeout = setTimeout(() => {
+        logger.error('WebTrackerManager: Script execution timed out', { requestId });
+        cleanup();
+        reject(new Error('Script execution timed out'));
+      }, 30000); // 30 second timeout
+
+      const responseHandler = (event) => {
+        logger.debug('WebTrackerManager: Received message in responseHandler', { 
+          type: event.type, 
+          requestId: event.requestId,
+          matchesOurRequest: event.requestId === requestId 
+        });
+        
+        if (event.type === 'DOM_RESULT' && event.requestId === requestId) {
+          clearTimeout(timeout);
+          cleanup();
+          
+          if (event.error) {
+            logger.error('WebTrackerManager: Failed to get DOM', { 
+              requestId, 
+              error: event.error 
+            });
+            reject(new Error(event.error));
+          } else {
+            logger.info('WebTrackerManager: Got DOM successfully', { requestId });
+            resolve(event.result);
+          }
+        }
+      };
+
+      const cleanup = this.server.on('message', responseHandler);
+      this.server.broadcast(message);
+    });
+  }
+
+  #getInitialTabs(time = Date.now()) {
+    return {
+      time,
+      type: 'INITIAL_TABS',
+      payload: Object.values(this.tabs).map(({ previousUrl, ...tab }) => tab),
+    };
+  }
+
+  #shouldSendEvent(pattern, event, cache = {}) {
+    if (cache[pattern] !== undefined) return cache[pattern];
+
+    const tab = this.tabs[event.payload.tabId];
+    if (tab) {
+      const urlMatches = verifyPattern(pattern, tab.url);
+      const previousUrlMatches = verifyPattern(pattern, tab.previousUrl);
+      
+      logger.info('WebTrackerManager: Checking pattern match', {
+        pattern,
+        tabUrl: tab.url,
+        previousUrl: tab.previousUrl,
+        urlMatches,
+        previousUrlMatches,
+        willMatch: urlMatches || previousUrlMatches
+      });
+      
+      if (urlMatches || previousUrlMatches) {
+        cache[pattern] = true;
+      } else {
+        cache[pattern] = false;
+      }
+    } else {
+      cache[pattern] = false;
+      logger.debug('WebTrackerManager: Tab not found in tabs map', {
+        tabId: event.payload.tabId,
+        availableTabs: Object.keys(this.tabs).length
+      });
+    }
+
+    return cache[pattern];
+  }
+
+  #handleEvent(event) {
+    logger.info('WebTrackerManager: Received event', { 
+      type: event.type, 
+      tabId: event.payload?.tabId,
+      url: event.payload?.url?.substring(0, 100),
+      requestUrl: event.payload?.requestUrl?.substring(0, 100)
+    });
+    
+    this.tabs = updateTabsState(event, this.tabs);
+    const { payload, type } = event;
+    switch (type) {
+      case 'INITIAL_TABS':
+      case 'TAB_REMOVED':
+      case 'TAB_ACTIVATED':
+      case 'NAVIGATION_STARTED':
+      case 'NAVIGATION_COMPLETED':
+        logger.debug('WebTrackerManager: Broadcasting global tabs/navigation event', { type, tabCount: event.payload?.length || 1 });
+        this.globalTabsAndNavigationCallbacks.forEach((callback) => {
+          this.#tryCallback(callback, event);
+        });
+        break;
+
+      default:
+        const cache = {};
+        let matchedCallbacks = 0;
+        
+        logger.info('WebTrackerManager: Processing non-nav event', {
+          type,
+          callbackCount: this.eventCallbacks.value.size,
+          patternCount: this.patternsByCallback.size
+        });
+        
+        this.eventCallbacks.value.forEach((callback) => {
+          const pattern = this.patternsByCallback.get(callback);
+          logger.info('WebTrackerManager: Checking callback', { pattern, hasPattern: !!pattern });
+          if (pattern && this.#shouldSendEvent(pattern, event, cache)) {
+            logger.debug('WebTrackerManager: Event matches pattern, sending to callback', { 
+              type, 
+              pattern,
+              tabUrl: this.tabs[event.payload?.tabId]?.url?.substring(0, 100)
+            });
+            this.#tryCallback(callback, event);
+            matchedCallbacks++;
+          }
+        });
+        
+        if (matchedCallbacks === 0) {
+          const availablePatterns = Array.from(this.patternsByCallback.values());
+          logger.debug('WebTrackerManager: Event did not match any patterns', { 
+            type, 
+            tabId: event.payload?.tabId,
+            tabUrl: this.tabs[event.payload?.tabId]?.url,
+            availablePatterns,
+            totalTabs: Object.keys(this.tabs).length
+          });
+        }
+    }
+  }
+
+  #tryCallback(callback, event) {
+    try {
+      callback(event);
+    } catch (error) {
+      logger.error(
+        'Failed sending ExtensionTracker event',
+        { event, error }
+      );
+    }
+  }
+
+  destroy() {
+    this.#stop();
+    this.patternsByCallback.clear();
+    this.globalTabsAndNavigationCallbacks.clear();
+  }
+}
+
+export { WebTrackerManager, reactive, computed, effect };