@leanbase-giangnd/js 0.0.0

package/src/extensions/replay/external/config.ts

@@ -0,0 +1,278 @@
+import { CapturedNetworkRequest, LeanbaseConfig, NetworkRecordOptions } from '../../../types'
+import { isArray, isBoolean, isFunction, isNullish, isString, isUndefined } from '@posthog/core'
+import { convertToURL } from '../../../utils/request-utils'
+import { logger } from '../../../utils/logger'
+import { shouldCaptureValue } from '../../../autocapture-utils'
+import { each } from '../../../utils'
+
+const LOGGER_PREFIX = '[SessionRecording]'
+
+const REDACTED = 'redacted'
+
+export const defaultNetworkOptions: Required<NetworkRecordOptions> = {
+    initiatorTypes: [
+        'audio',
+        'beacon',
+        'body',
+        'css',
+        'early-hints',
+        'embed',
+        'fetch',
+        'frame',
+        'iframe',
+        'image',
+        'img',
+        'input',
+        'link',
+        'navigation',
+        'object',
+        'ping',
+        'script',
+        'track',
+        'video',
+        'xmlhttprequest',
+    ],
+    maskRequestFn: (data: CapturedNetworkRequest) => data,
+    recordHeaders: false,
+    recordBody: false,
+    recordInitialRequests: false,
+    recordPerformance: false,
+    performanceEntryTypeToObserve: [
+        // 'event', // This is too noisy as it covers all browser events
+        'first-input',
+        // 'mark', // Mark is used too liberally. We would need to filter for specific marks
+        // 'measure', // Measure is used too liberally. We would need to filter for specific measures
+        'navigation',
+        'paint',
+        'resource',
+    ],
+    payloadSizeLimitBytes: 1000000,
+    payloadHostDenyList: [
+        '.lr-ingest.io',
+        '.ingest.sentry.io',
+        '.clarity.ms',
+        // NB no leading dot here
+        'analytics.google.com',
+        'bam.nr-data.net',
+    ],
+}
+
+const HEADER_DENY_LIST = [
+    'authorization',
+    'x-forwarded-for',
+    'authorization',
+    'cookie',
+    'set-cookie',
+    'x-api-key',
+    'x-real-ip',
+    'remote-addr',
+    'forwarded',
+    'proxy-authorization',
+    'x-csrf-token',
+    'x-csrftoken',
+    'x-xsrf-token',
+]
+
+const PAYLOAD_CONTENT_DENY_LIST = [
+    'password',
+    'secret',
+    'passwd',
+    'api_key',
+    'apikey',
+    'auth',
+    'credentials',
+    'mysql_pwd',
+    'privatekey',
+    'private_key',
+    'token',
+]
+
+// we always remove headers on the deny list because we never want to capture this sensitive data
+const removeAuthorizationHeader = (data: CapturedNetworkRequest): CapturedNetworkRequest => {
+    const headers = data.requestHeaders
+    if (!isNullish(headers)) {
+        const mutableHeaders: Record<string, any> = isArray(headers)
+            ? Object.fromEntries(headers as any)
+            : (headers as any)
+
+        each(Object.keys(mutableHeaders ?? {}), (header) => {
+            if (HEADER_DENY_LIST.includes(header.toLowerCase())) {
+                mutableHeaders[header] = REDACTED
+            }
+        })
+
+        data.requestHeaders = mutableHeaders as any
+    }
+    return data
+}
+
+const POSTHOG_PATHS_TO_IGNORE = ['/s/', '/e/', '/i/']
+// want to ignore posthog paths when capturing requests, or we can get trapped in a loop
+// because calls to PostHog would be reported using a call to PostHog which would be reported....
+const ignorePostHogPaths = (
+    data: CapturedNetworkRequest,
+    apiHostConfig: LeanbaseConfig['host']
+): CapturedNetworkRequest | undefined => {
+    const url = convertToURL(data.name)
+
+    const host = apiHostConfig || ''
+    let replaceValue = host.indexOf('http') === 0 ? convertToURL(host)?.pathname : host
+    if (replaceValue === '/') {
+        replaceValue = ''
+    }
+    const pathname = url?.pathname.replace(replaceValue || '', '')
+
+    if (url && pathname && POSTHOG_PATHS_TO_IGNORE.some((path) => pathname.indexOf(path) === 0)) {
+        return undefined
+    }
+    return data
+}
+
+function estimateBytes(payload: string): number {
+    return new Blob([payload]).size
+}
+
+function enforcePayloadSizeLimit(
+    payload: string | null | undefined,
+    headers: Record<string, any> | undefined,
+    limit: number,
+    description: string
+): string | null | undefined {
+    if (isNullish(payload)) {
+        return payload
+    }
+
+    let requestContentLength: string | number = headers?.['content-length'] || estimateBytes(payload)
+    if (isString(requestContentLength)) {
+        requestContentLength = parseInt(requestContentLength)
+    }
+
+    if (requestContentLength > limit) {
+        return LOGGER_PREFIX + ` ${description} body too large to record (${requestContentLength} bytes)`
+    }
+
+    return payload
+}
+
+// people can have arbitrarily large payloads on their site, but we don't want to ingest them
+const limitPayloadSize = (
+    options: NetworkRecordOptions
+): ((data: CapturedNetworkRequest | undefined) => CapturedNetworkRequest | undefined) => {
+    // the smallest of 1MB or the specified limit if there is one
+    const limit = Math.min(1000000, options.payloadSizeLimitBytes ?? 1000000)
+
+    return (data) => {
+        if (data?.requestBody) {
+            data.requestBody = enforcePayloadSizeLimit(data.requestBody, data.requestHeaders, limit, 'Request')
+        }
+
+        if (data?.responseBody) {
+            data.responseBody = enforcePayloadSizeLimit(data.responseBody, data.responseHeaders, limit, 'Response')
+        }
+
+        return data
+    }
+}
+
+function scrubPayload(payload: string | null | undefined, label: 'Request' | 'Response'): string | null | undefined {
+    if (isNullish(payload)) {
+        return payload
+    }
+    let scrubbed = payload
+
+    if (!shouldCaptureValue(scrubbed, false)) {
+        scrubbed = LOGGER_PREFIX + ' ' + label + ' body ' + REDACTED
+    }
+    each(PAYLOAD_CONTENT_DENY_LIST, (text) => {
+        if (scrubbed?.length && scrubbed?.indexOf(text) !== -1) {
+            scrubbed = LOGGER_PREFIX + ' ' + label + ' body ' + REDACTED + ' as might contain: ' + text
+        }
+    })
+
+    return scrubbed
+}
+
+function scrubPayloads(capturedRequest: CapturedNetworkRequest | undefined): CapturedNetworkRequest | undefined {
+    if (isUndefined(capturedRequest)) {
+        return undefined
+    }
+
+    capturedRequest.requestBody = scrubPayload(capturedRequest.requestBody, 'Request')
+    capturedRequest.responseBody = scrubPayload(capturedRequest.responseBody, 'Response')
+
+    return capturedRequest
+}
+
+/**
+ *  whether a maskRequestFn is provided or not,
+ *  we ensure that we remove the denied header from requests
+ *  we _never_ want to record that header by accident
+ *  if someone complains then we'll add an opt-in to let them override it
+ */
+export const buildNetworkRequestOptions = (
+    instanceConfig: LeanbaseConfig,
+    remoteNetworkOptions: Pick<
+        NetworkRecordOptions,
+        'recordHeaders' | 'recordBody' | 'recordPerformance' | 'payloadHostDenyList'
+    > = {}
+): NetworkRecordOptions => {
+    const remoteOptions = remoteNetworkOptions || {}
+    const config: NetworkRecordOptions = {
+        payloadSizeLimitBytes: defaultNetworkOptions.payloadSizeLimitBytes,
+        performanceEntryTypeToObserve: [...defaultNetworkOptions.performanceEntryTypeToObserve],
+        payloadHostDenyList: [
+            ...(remoteOptions.payloadHostDenyList || []),
+            ...defaultNetworkOptions.payloadHostDenyList,
+        ],
+    }
+    // client can always disable despite remote options
+    const sessionRecordingConfig = instanceConfig.session_recording || {}
+    const capturePerformanceConfig = instanceConfig.capture_performance
+    const userPerformanceOptIn = isBoolean(capturePerformanceConfig)
+        ? capturePerformanceConfig
+        : !!capturePerformanceConfig?.network_timing
+    const canRecordHeaders = sessionRecordingConfig.recordHeaders === true && !!remoteOptions.recordHeaders
+    const canRecordBody = sessionRecordingConfig.recordBody === true && !!remoteOptions.recordBody
+    const canRecordPerformance = userPerformanceOptIn && !!remoteOptions.recordPerformance
+
+    const payloadLimiter = limitPayloadSize(config)
+
+    const enforcedCleaningFn: NetworkRecordOptions['maskRequestFn'] = (d: CapturedNetworkRequest) =>
+        payloadLimiter(ignorePostHogPaths(removeAuthorizationHeader(d), instanceConfig.host || ''))
+
+    const hasDeprecatedMaskFunction = isFunction(sessionRecordingConfig.maskNetworkRequestFn)
+
+    if (hasDeprecatedMaskFunction && isFunction(sessionRecordingConfig.maskCapturedNetworkRequestFn)) {
+        logger.warn(
+            'Both `maskNetworkRequestFn` and `maskCapturedNetworkRequestFn` are defined. `maskNetworkRequestFn` will be ignored.'
+        )
+    }
+
+    if (hasDeprecatedMaskFunction) {
+        sessionRecordingConfig.maskCapturedNetworkRequestFn = (data: CapturedNetworkRequest) => {
+            const cleanedURL = sessionRecordingConfig.maskNetworkRequestFn!({ url: data.name })
+            return {
+                ...data,
+                name: cleanedURL?.url,
+            } as CapturedNetworkRequest
+        }
+    }
+
+    config.maskRequestFn = isFunction(sessionRecordingConfig.maskCapturedNetworkRequestFn)
+        ? (data) => {
+              const cleanedRequest = enforcedCleaningFn(data)
+              return cleanedRequest
+                  ? (sessionRecordingConfig.maskCapturedNetworkRequestFn?.(cleanedRequest) ?? undefined)
+                  : undefined
+          }
+        : (data) => scrubPayloads(enforcedCleaningFn(data))
+
+    return {
+        ...defaultNetworkOptions,
+        ...config,
+        recordHeaders: canRecordHeaders,
+        recordBody: canRecordBody,
+        recordPerformance: canRecordPerformance,
+        recordInitialRequests: canRecordPerformance,
+    }
+}

package/src/extensions/replay/external/denylist.ts

@@ -0,0 +1,32 @@
+import { NetworkRecordOptions } from '../../../types'
+
+function hostnameFromURL(url: string | URL | RequestInfo): string | null {
+    try {
+        if (typeof url === 'string') {
+            return new URL(url).hostname
+        }
+        if ('url' in url) {
+            return new URL(url.url).hostname
+        }
+        return url.hostname
+    } catch {
+        return null
+    }
+}
+
+export function isHostOnDenyList(url: string | URL | Request, options: NetworkRecordOptions) {
+    const hostname = hostnameFromURL(url)
+    const defaultNotDenied = { hostname, isHostDenied: false }
+
+    if (!options.payloadHostDenyList?.length || !hostname?.trim().length) {
+        return defaultNotDenied
+    }
+
+    for (const deny of options.payloadHostDenyList) {
+        if (hostname.endsWith(deny)) {
+            return { hostname, isHostDenied: true }
+        }
+    }
+
+    return defaultNotDenied
+}