@nebulacomponents/citable 1.18.2 → 1.19.0

package/src/observations/browser/runtimeEventCollector.js

@@ -0,0 +1,337 @@
+import { safeJsonClone, isSensitiveKey, isSensitiveValue } from './sanitizer.js';
+
+export class RuntimeEventCollector {
+  constructor(options = {}) {
+    const {
+      runtimeEventPolicy = {},
+      correlationTracker,
+      planId,
+      profileId,
+    } = options;
+
+    this.policy = runtimeEventPolicy;
+    this.enabled = Boolean(this.policy.enabled);
+    this.planId = planId || 'plan-unspecified';
+    this.profileId = profileId || 'profile-unspecified';
+    this.tracker = correlationTracker;
+
+    this.sources = this.policy.sources || [];
+    this.allowedEventNames = this.policy.allowed_event_names ? new Set(this.policy.allowed_event_names) : null;
+    this.allowedFields = this.policy.allowed_fields || null;
+    this.hashFields = this.policy.hash_fields || null;
+    this.redactFields = this.policy.redact_fields || null;
+    this.maxEvents = Math.min(this.policy.max_events || 200, 1000);
+    this.maxPayloadDepth = Math.min(this.policy.max_payload_depth || 5, 10);
+    this.maxFieldSize = Math.min(this.policy.max_field_size || 1024, 4096);
+
+    this.events = [];
+    this.eventCountsByName = {};
+    this.eventCountsBySource = {};
+    this.summary = {
+      total_events: 0,
+      filtered_events: 0,
+      by_source: {},
+      by_event_name: {},
+      collector_status: 'complete',
+      truncated: false,
+      truncation_reason: null,
+      collector_health: {
+        status: 'active',
+        queue_replaced: false,
+        push_replaced: false,
+        polling_fallback_used: false,
+        supported_contexts: ['top_level_main_frame'],
+        unsupported_contexts: ['cross_origin_iframes', 'web_workers', 'service_workers'],
+      },
+    };
+  }
+
+  /**
+   * Generates initialization script to be added to browser context or page.
+   */
+  getInitScript() {
+    if (!this.enabled || !this.sources.length) return null;
+    const sourcesConfig = JSON.stringify(this.sources);
+
+    return `
+      (function() {
+        window.__citable_events = window.__citable_events || [];
+        window.__citable_health = window.__citable_health || {
+          queue_replaced: false,
+          push_replaced: false,
+          polling_fallback_used: false,
+        };
+        const sources = ${sourcesConfig};
+
+        function safeExtract(obj, depth, seen) {
+          if (depth <= 0) return '[DEPTH_LIMIT]';
+          if (obj === null || obj === undefined) return obj;
+          const t = typeof obj;
+          if (t === 'number' || t === 'boolean' || t === 'string') return obj;
+          if (t !== 'object') return String(obj);
+          if (seen.has(obj)) return '[CIRCULAR]';
+          seen.add(obj);
+
+          if (Array.isArray(obj)) {
+            return obj.map(function(item) { return safeExtract(item, depth - 1, seen); });
+          }
+
+          const res = {};
+          const keys = Object.keys(obj);
+          for (var i = 0; i < keys.length; i++) {
+            var k = keys[i];
+            if (k === '__proto__' || k === 'constructor' || k === 'prototype') continue;
+            try {
+              res[k] = safeExtract(obj[k], depth - 1, seen);
+            } catch (err) {
+              res[k] = '[GETTER_ERROR]';
+            }
+          }
+          return res;
+        }
+
+        for (var i = 0; i < sources.length; i++) {
+          var src = sources[i];
+          if (src.type === 'global_array' && src.global_name) {
+            var propName = src.global_name.replace(/^window\\./, '');
+
+            (function(sourceConfig, prop) {
+              var currentArray = window[prop];
+              if (!Array.isArray(currentArray)) {
+                currentArray = [];
+                window[prop] = currentArray;
+              }
+
+              for (var j = 0; j < currentArray.length; j++) {
+                try {
+                  window.__citable_events.push({
+                    source_id: sourceConfig.source_id,
+                    payload: safeExtract(currentArray[j], 5, new WeakSet()),
+                    ts: Date.now()
+                  });
+                } catch (e) {}
+              }
+              currentArray.__citable_drained_count = currentArray.length;
+
+              function wrapPush(arr) {
+                var origPush = arr.push;
+                arr.push = function(...items) {
+                  for (var k = 0; k < items.length; k++) {
+                    try {
+                      window.__citable_events.push({
+                        source_id: sourceConfig.source_id,
+                        payload: safeExtract(items[k], 5, new WeakSet()),
+                        ts: Date.now()
+                      });
+                    } catch (e) {}
+                  }
+                  if (typeof origPush === 'function') {
+                    return origPush.apply(this, items);
+                  }
+                };
+              }
+
+              wrapPush(currentArray);
+
+              try {
+                var storedArray = currentArray;
+                Object.defineProperty(window, prop, {
+                  configurable: true,
+                  enumerable: true,
+                  get: function() { return storedArray; },
+                  set: function(newVal) {
+                    window.__citable_health.queue_replaced = true;
+                    storedArray = newVal;
+                    if (Array.isArray(newVal)) {
+                      wrapPush(newVal);
+                      for (var m = 0; m < newVal.length; m++) {
+                        try {
+                          window.__citable_events.push({
+                            source_id: sourceConfig.source_id,
+                            payload: safeExtract(newVal[m], 5, new WeakSet()),
+                            ts: Date.now()
+                          });
+                        } catch (e) {}
+                      }
+                      newVal.__citable_drained_count = newVal.length;
+                    }
+                  }
+                });
+              } catch (e) {}
+            })(src, propName);
+
+          } else if (src.type === 'window_event' && src.event_name) {
+            (function(sourceConfig) {
+              window.addEventListener(sourceConfig.event_name, function(e) {
+                try {
+                  window.__citable_events.push({
+                    source_id: sourceConfig.source_id,
+                    payload: {
+                      type: e.type,
+                      detail: safeExtract(e.detail, 5, new WeakSet()) || null,
+                    },
+                    ts: Date.now()
+                  });
+                } catch (err) {}
+              }, true);
+            })(src);
+          }
+        }
+      })();
+    `;
+  }
+
+  async installOnPage(page) {
+    if (!this.enabled || !this.sources.length) return;
+    const script = this.getInitScript();
+    if (!script) return;
+    try {
+      await page.evaluate(script);
+    } catch {
+      // Ignored if page is not ready yet
+    }
+  }
+
+  async drainEvents(page) {
+    if (!this.enabled || !this.sources.length) return;
+
+    let drainResult = null;
+    try {
+      drainResult = await page.evaluate((sources) => {
+        const events = window.__citable_events ? window.__citable_events.splice(0) : [];
+        const health = window.__citable_health || {
+          queue_replaced: false,
+          push_replaced: false,
+          polling_fallback_used: false,
+        };
+
+        // Fallback polling for arrays where push was replaced or unhooked
+        for (const src of sources) {
+          if (src.type === 'global_array' && src.global_name) {
+            const prop = src.global_name.replace(/^window\./, '');
+            const arr = window[prop];
+            if (Array.isArray(arr)) {
+              const startIdx = arr.__citable_drained_count || 0;
+              if (arr.length > startIdx) {
+                health.polling_fallback_used = true;
+                for (let i = startIdx; i < arr.length; i++) {
+                  events.push({
+                    source_id: src.source_id,
+                    payload: arr[i],
+                    ts: Date.now(),
+                  });
+                }
+                arr.__citable_drained_count = arr.length;
+              }
+            }
+          }
+        }
+
+        return { events, health };
+      }, this.sources);
+    } catch {
+      return;
+    }
+
+    const eventsList = Array.isArray(drainResult)
+      ? drainResult
+      : (drainResult && Array.isArray(drainResult.events) ? drainResult.events : []);
+
+    if (!eventsList || eventsList.length === 0) return;
+
+    if (drainResult && !Array.isArray(drainResult) && drainResult.health) {
+      if (drainResult.health.queue_replaced) this.summary.collector_health.queue_replaced = true;
+      if (drainResult.health.push_replaced) this.summary.collector_health.push_replaced = true;
+      if (drainResult.health.polling_fallback_used) this.summary.collector_health.polling_fallback_used = true;
+    }
+
+    for (const raw of eventsList) {
+      if (this.events.length >= this.maxEvents) {
+        this.summary.truncated = true;
+        this.summary.truncation_reason = 'max_events_exceeded';
+        break;
+      }
+
+      const sourceId = raw.source_id || 'unknown';
+      const sourceConfig = this.sources.find((s) => s.source_id === sourceId) || {};
+      const payload = raw.payload;
+
+      // Extract event name
+      let eventName = 'unnamed_event';
+      if (typeof payload === 'object' && payload !== null) {
+        const keyName = sourceConfig.event_name_key || 'event';
+        if (typeof payload[keyName] === 'string') {
+          eventName = payload[keyName];
+        } else if (typeof payload.type === 'string') {
+          eventName = payload.type;
+        } else if (sourceConfig.event_name) {
+          eventName = sourceConfig.event_name;
+        }
+      } else if (typeof payload === 'string') {
+        eventName = payload;
+      }
+
+      // Check allowed event names
+      const isAllowed = !this.allowedEventNames || this.allowedEventNames.has(eventName);
+      if (!isAllowed) {
+        this.summary.filtered_events++;
+        continue; // Filtered out
+      }
+
+      this.eventCountsByName[eventName] = (this.eventCountsByName[eventName] || 0) + 1;
+      this.eventCountsBySource[sourceId] = (this.eventCountsBySource[sourceId] || 0) + 1;
+      const occurrenceIndex = this.eventCountsByName[eventName];
+
+      const seq = this.tracker.nextSequence();
+      const timing = this.tracker.timing();
+      const correlation = this.tracker.getCorrelation();
+
+      // Sanitize payload
+      const stats = { redactedCount: 0, filteredFields: [] };
+      const sanitizedPayload = typeof payload === 'object' && payload !== null
+        ? safeJsonClone(payload, {
+            maxDepth: this.maxPayloadDepth,
+            maxFieldSize: this.maxFieldSize,
+            allowedFields: this.allowedFields,
+            hashFields: this.hashFields,
+            redactFields: this.redactFields,
+            stats,
+          })
+        : { value: String(payload) };
+
+      const eventRecord = {
+        sequence: seq,
+        source_id: sourceId,
+        event_name: eventName,
+        occurrence_index: occurrenceIndex,
+        timing,
+        correlation,
+        payload: sanitizedPayload,
+        disposition: {
+          capture_status: stats.redactedCount > 0 ? 'redacted' : 'captured',
+          redacted_count: stats.redactedCount,
+          filtered_fields: stats.filteredFields,
+        },
+      };
+
+      this.events.push(eventRecord);
+      this.summary.total_events++;
+    }
+
+    this.summary.by_source = { ...this.eventCountsBySource };
+    this.summary.by_event_name = { ...this.eventCountsByName };
+  }
+
+  toArtifact() {
+    this.summary.collector_status = this.summary.truncated ? 'truncated' : 'complete';
+    return {
+      schema_version: 1,
+      plan_id: this.planId,
+      profile_id: this.profileId,
+      collected_at: this.tracker.timing().timestamp,
+      capture_policy: this.policy,
+      summary: this.summary,
+      events: this.events,
+    };
+  }
+}

package/src/observations/browser/sanitizer.js

@@ -0,0 +1,322 @@
+import { sha256 } from '../../shared/io.js';
+
+// Sensitive key patterns: headers, storage keys, query parameters, payload fields
+export const SENSITIVE_KEY_PATTERN = /(?:^|[-_])(?:authorization|bearer|token|secret|apiKey|api[-_]?key|password|credential|access[-_]?token|refresh[-_]?token|jwt|session(?:[-_]?id)?|csrf|xsrf|cvv|cvc|ssn|credit[-_]?card|passwd|auth[-_]?token|oauth[-_]?token|auth[-_]?code|code)(?:$|[-_])|^(?:cookie|set[-_]?cookie)$/i;
+
+// Sensitive value patterns: bearer tokens, common API keys, JWT tokens, card numbers
+export const SENSITIVE_VALUE_PATTERN = /(?:bearer\s+[a-zA-Z0-9_\-.]{10,}|(?:ghp|gho|glpat|ya29|sk-[a-zA-Z0-9]{20,})|eyJ[a-zA-Z0-9_-]{10,}\.eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}|\b(?:\d[ -]*?){13,16}\b)/i;
+
+export const REDACTED_MARKER = '[REDACTED_SECRET]';
+export const FILTERED_MARKER = '[FILTERED]';
+export const POLICY_RESTRICTED_MARKER = '[POLICY_RESTRICTED]';
+
+/**
+ * Check if a field/header/cookie/param key is sensitive.
+ */
+export function isSensitiveKey(key) {
+  if (typeof key !== 'string') return false;
+  return SENSITIVE_KEY_PATTERN.test(key);
+}
+
+/**
+ * Check if a string value contains sensitive tokens or patterns.
+ */
+export function isSensitiveValue(value) {
+  if (typeof value !== 'string') return false;
+  return SENSITIVE_VALUE_PATTERN.test(value);
+}
+
+/**
+ * Sanitize headers against allowlist and sensitive key/value rules.
+ * Even if a sensitive header is allowlisted, it MUST be redacted.
+ */
+export function sanitizeHeaders(headers, allowedNames = null) {
+  if (!headers || typeof headers !== 'object') {
+    return { headers: null, status: 'not_requested', redactedCount: 0, filteredCount: 0 };
+  }
+  const allowedSet = Array.isArray(allowedNames) ? new Set(allowedNames.map((n) => n.toLowerCase())) : null;
+  const result = {};
+  let redactedCount = 0;
+  let filteredCount = 0;
+
+  for (const [rawKey, rawVal] of Object.entries(headers)) {
+    const key = rawKey.toLowerCase();
+    // Sensitive keys are unconditionally redacted or omitted
+    if (isSensitiveKey(key)) {
+      if (allowedSet && allowedSet.has(key)) {
+        result[key] = REDACTED_MARKER;
+        redactedCount++;
+      } else {
+        filteredCount++;
+      }
+      continue;
+    }
+
+    // Non-sensitive keys: must be explicitly allowed if allowedSet is provided
+    if (allowedSet && !allowedSet.has(key)) {
+      filteredCount++;
+      continue;
+    }
+
+    // Value sanitization
+    const strVal = String(rawVal ?? '');
+    if (isSensitiveValue(strVal)) {
+      result[key] = REDACTED_MARKER;
+      redactedCount++;
+    } else {
+      result[key] = strVal;
+    }
+  }
+
+  const hasEntries = Object.keys(result).length > 0;
+  let status = 'captured';
+  if (!hasEntries) {
+    status = filteredCount > 0 ? 'filtered' : 'not_requested';
+  } else if (redactedCount > 0) {
+    status = 'redacted';
+  }
+
+  return {
+    headers: hasEntries ? result : null,
+    status,
+    redactedCount,
+    filteredCount,
+  };
+}
+
+/**
+ * Sanitize URL query parameters according to allowlist.
+ */
+export function sanitizeQueryParams(urlStr, allowedParams = null) {
+  if (!urlStr) return { query: null, status: 'none_present', redactedCount: 0, filteredFields: [] };
+  if (!allowedParams || !Array.isArray(allowedParams) || allowedParams.length === 0) {
+    return { query: null, status: 'not_requested', redactedCount: 0, filteredFields: [] };
+  }
+  let parsed;
+  try {
+    parsed = new URL(urlStr);
+  } catch {
+    return { query: null, status: 'none_present', redactedCount: 0, filteredFields: [] };
+  }
+
+  const searchParams = parsed.searchParams;
+  if (!searchParams || Array.from(searchParams.keys()).length === 0) {
+    return { query: null, status: 'none_present', redactedCount: 0, filteredFields: [] };
+  }
+
+  const allowedSet = new Set(allowedParams.map((p) => p.toLowerCase()));
+  const query = {};
+  let redactedCount = 0;
+  const filteredFields = [];
+
+  for (const [key, val] of searchParams.entries()) {
+    const lowerKey = key.toLowerCase();
+    if (isSensitiveKey(lowerKey)) {
+      if (allowedSet.has(lowerKey)) {
+        query[key] = REDACTED_MARKER;
+        redactedCount++;
+      } else {
+        filteredFields.push(key);
+      }
+      continue;
+    }
+
+    if (!allowedSet.has(lowerKey)) {
+      filteredFields.push(key);
+      continue;
+    }
+
+    if (isSensitiveValue(val)) {
+      query[key] = REDACTED_MARKER;
+      redactedCount++;
+    } else {
+      query[key] = val;
+    }
+  }
+
+  const hasEntries = Object.keys(query).length > 0;
+  let status = 'captured';
+  if (!hasEntries) {
+    status = filteredFields.length > 0 ? 'filtered' : 'none_present';
+  } else if (redactedCount > 0) {
+    status = 'redacted';
+  }
+
+  return {
+    query: hasEntries ? query : null,
+    status,
+    redactedCount,
+    filteredFields,
+  };
+}
+
+/**
+ * Safely deep-clone an object with cycle detection, depth limits, field allowlists,
+ * and sensitive-value redaction.
+ */
+export function safeJsonClone(value, options = {}) {
+  const {
+    maxDepth = 5,
+    maxBytes = 4096,
+    maxFieldSize = 1024,
+    allowedFields = null,
+    hashFields = null,
+    redactFields = null,
+    seen = new WeakSet(),
+    depth = 0,
+    stats = { redactedCount: 0, filteredFields: [] },
+  } = options;
+
+  if (value === null || value === undefined) return value;
+  if (typeof value === 'boolean' || typeof value === 'number') return value;
+
+  if (typeof value === 'string') {
+    if (isSensitiveValue(value)) {
+      stats.redactedCount++;
+      return REDACTED_MARKER;
+    }
+    if (value.length > maxFieldSize) {
+      return value.slice(0, maxFieldSize) + '...[TRUNCATED]';
+    }
+    return value;
+  }
+
+  if (depth >= maxDepth) return '[DEPTH_LIMIT]';
+
+  if (typeof value === 'object') {
+    if (seen.has(value)) return '[CIRCULAR]';
+    seen.add(value);
+
+    if (Array.isArray(value)) {
+      return value.map((item) => safeJsonClone(item, { ...options, depth: depth + 1, seen, stats }));
+    }
+
+    const copy = {};
+    const allowedSet = allowedFields ? new Set(allowedFields) : null;
+    const hashSet = hashFields ? new Set(hashFields) : null;
+    const redactSet = redactFields ? new Set(redactFields) : null;
+
+    for (const k of Object.getOwnPropertyNames(value)) {
+      // Check allowlist first: if not allowed, filter it out
+      if (allowedSet && !allowedSet.has(k)) {
+        stats.filteredFields.push(k);
+        continue;
+      }
+
+      // If allowed (or no allowlist specified), check if k is sensitive or in redactSet
+      if (isSensitiveKey(k) || (redactSet && redactSet.has(k))) {
+        copy[k] = REDACTED_MARKER;
+        stats.redactedCount++;
+        continue;
+      }
+
+      let v;
+      try {
+        v = value[k];
+      } catch {
+        v = '[GETTER_ERROR]';
+      }
+
+      // Check hash fields
+      if (hashSet && hashSet.has(k)) {
+        copy[k] = `sha256:${sha256(typeof v === 'string' ? v : JSON.stringify(v))}`;
+        continue;
+      }
+
+      copy[k] = safeJsonClone(v, { ...options, depth: depth + 1, seen, stats });
+    }
+    return copy;
+  }
+
+  return String(value);
+}
+
+/**
+ * Sanitize request or response payloads (body).
+ */
+export function sanitizePayload(payload, options = {}) {
+  const {
+    allowedFields = null,
+    hashOnly = false,
+    maxBytes = 4096,
+    maxDepth = 5,
+  } = options;
+
+  if (payload == null || payload === '') {
+    return { sanitized: null, hash: null, bytes: 0, status: 'unavailable', redactedCount: 0, filteredFields: [] };
+  }
+
+  // Handle binary buffers without unsafe UTF-8 string coercion
+  if (Buffer.isBuffer(payload)) {
+    const isBinary = payload.some((byte) => byte === 0);
+    if (isBinary) {
+      const hash = sha256(payload);
+      return {
+        sanitized: null,
+        hash,
+        bytes: payload.length,
+        status: hashOnly ? 'hash_only' : 'unavailable',
+        redactedCount: 0,
+        filteredFields: [],
+      };
+    }
+  }
+
+  const rawString = typeof payload === 'string' ? payload : Buffer.isBuffer(payload) ? payload.toString('utf8') : JSON.stringify(payload);
+  const bytes = Buffer.byteLength(rawString, 'utf8');
+  const hash = sha256(rawString);
+
+  if (hashOnly) {
+    return { sanitized: null, hash, bytes, status: 'hash_only', redactedCount: 0, filteredFields: [] };
+  }
+
+  // Try parsing JSON if allowedFields or structured redaction is requested
+  let parsedJson = null;
+  try {
+    parsedJson = JSON.parse(rawString);
+  } catch {
+    parsedJson = null;
+  }
+
+  const stats = { redactedCount: 0, filteredFields: [] };
+
+  if (parsedJson && typeof parsedJson === 'object') {
+    const cloned = safeJsonClone(parsedJson, {
+      maxDepth,
+      maxBytes,
+      allowedFields,
+      stats,
+    });
+    const hasAllowedEntries = cloned !== null && typeof cloned === 'object' && Object.keys(cloned).length > 0;
+    let status = 'captured';
+    if (!hasAllowedEntries && allowedFields && allowedFields.length > 0) {
+      status = 'filtered';
+    } else if (rawString.length > maxBytes) {
+      status = 'truncated';
+    }
+    return {
+      sanitized: cloned,
+      hash,
+      bytes,
+      status,
+      redactedCount: stats.redactedCount,
+      filteredFields: stats.filteredFields,
+    };
+  }
+
+  // Raw text payload
+  if (isSensitiveValue(rawString)) {
+    return { sanitized: REDACTED_MARKER, hash, bytes, status: 'captured', redactedCount: 1, filteredFields: [] };
+  }
+
+  const isTruncated = rawString.length > maxBytes;
+  const truncated = isTruncated ? rawString.slice(0, maxBytes) + '...[TRUNCATED]' : rawString;
+  return {
+    sanitized: truncated,
+    hash,
+    bytes,
+    status: isTruncated ? 'truncated' : 'captured',
+    redactedCount: 0,
+    filteredFields: [],
+  };
+}