@friggframework/core 2.0.0-next.111 → 2.0.0-next.113

package/logs/record.js

@@ -0,0 +1,276 @@
+const { LEVELS } = require('./levels');
+const { serializeValue, serializeError, isError } = require('./serialize');
+const { isPlainObject } = require('./context');
+const {
+    RESOURCE_KEYS,
+    DROPPED_KEYS_FIELD,
+    OWNED_KEYS,
+    RESERVED_KEYS,
+    PAYLOAD_KEYS,
+} = require('./contract-keys');
+
+const MAX_RECORD_BYTES = 16 * 1024;
+const DEFAULT_LOGGER_NAME = 'frigg.unknown';
+
+function deepFreeze(value) {
+    if (value && typeof value === 'object' && !Object.isFrozen(value)) {
+        for (const key of Object.keys(value)) deepFreeze(value[key]);
+        Object.freeze(value);
+    }
+    return value;
+}
+
+function readFields(source) {
+    const result = {};
+    if (!source || typeof source !== 'object') return result;
+    let keys;
+    try {
+        keys = Object.keys(source);
+    } catch {
+        return result;
+    }
+    for (const key of keys) {
+        try {
+            result[key] = source[key];
+        } catch {
+            result[key] = '[Getter threw]';
+        }
+    }
+    return result;
+}
+
+// Returns the message text. An Error message is serialized once; when the
+// call site has no `error`, that result becomes the record's `error`.
+function buildMessage(message, callSite) {
+    if (typeof message === 'string') {
+        return { text: serializeValue(message) };
+    }
+    if (message === undefined || message === null) return { text: '' };
+    if (isError(message)) {
+        const error = serializeError(message);
+        const text = typeof error?.message === 'string' ? error.message : '';
+        if (callSite.error !== undefined) return { text };
+        callSite.error = message;
+        return { text, error };
+    }
+    if (callSite.value === undefined) callSite.value = message;
+    return { text: '[non-string message]' };
+}
+
+const INVOCATION_KEY = 'invocation';
+
+// A boundary adds request detail to the scope's invocation; scope keys win.
+function extendInvocation(scoped, detail) {
+    const merged = { ...scoped };
+    for (const key of Object.keys(detail)) {
+        if (!(key in merged)) merged[key] = detail[key];
+    }
+    return merged;
+}
+
+function traceFields(spanContext) {
+    if (
+        !spanContext ||
+        typeof spanContext.traceId !== 'string' ||
+        typeof spanContext.spanId !== 'string'
+    ) {
+        return {};
+    }
+    const flags = Number(spanContext.traceFlags) || 0;
+    return {
+        trace_id: spanContext.traceId,
+        span_id: spanContext.spanId,
+        trace_flags: (flags & 0xff).toString(16).padStart(2, '0'),
+    };
+}
+
+function sameValue(a, b) {
+    if (a === b) return true;
+    return (
+        typeof a === 'object' &&
+        typeof b === 'object' &&
+        JSON.stringify(a) === JSON.stringify(b)
+    );
+}
+
+function byteLength(record) {
+    return Buffer.byteLength(JSON.stringify(record));
+}
+
+function isDroppedKey(key, dropPayloads) {
+    return (
+        OWNED_KEYS.has(key) ||
+        RESERVED_KEYS.has(key) ||
+        (dropPayloads && PAYLOAD_KEYS.has(key))
+    );
+}
+
+// Filters and serializes one source. `error` is serialized on its own, once
+// (or taken from `preSerializedError`), and never walked by serializeValue.
+function prepareSource(source, dropPayloads, preSerializedError) {
+    const accepted = {};
+    const keys = [];
+    const dropped = [];
+    let rawError;
+    for (const [key, value] of Object.entries(source)) {
+        if (value === undefined || value === null) continue;
+        if (isDroppedKey(key, dropPayloads)) {
+            dropped.push(key);
+            continue;
+        }
+        keys.push(key);
+        if (key === 'error') rawError = value;
+        else accepted[key] = value;
+    }
+    // One pass over the object, so key rules (denied keys, digests,
+    // headers, OAuth shapes) apply to top-level fields too.
+    const safe = serializeValue(accepted);
+    const entries = [];
+    for (const key of keys) {
+        const value =
+            key === 'error'
+                ? preSerializedError ?? serializeError(rawError)
+                : safe?.[key];
+        if (value !== undefined && value !== null) entries.push([key, value]);
+    }
+    return { entries, dropped };
+}
+
+// The scope object is frozen and shared by every record of one scope.
+const preparedScopes = new WeakMap();
+
+function prepareScope(scope, dropPayloads) {
+    if (!scope || typeof scope !== 'object') return { entries: [], dropped: [] };
+    let byLevel = preparedScopes.get(scope);
+    if (!byLevel) {
+        byLevel = {};
+        preparedScopes.set(scope, byLevel);
+    }
+    const slot = dropPayloads ? 'info' : 'debug';
+    if (!byLevel[slot]) byLevel[slot] = prepareSource(scope, dropPayloads);
+    return byLevel[slot];
+}
+
+function buildRecord({
+    level,
+    logger,
+    message,
+    fields,
+    bindings,
+    scope,
+    spanContext,
+    resource = {},
+    now = new Date(),
+} = {}) {
+    const dropPayloads = LEVELS[level] >= LEVELS.INFO;
+    // log.error('msg', err): own props of an Error (axios config, request) must not become fields.
+    const callSite = readFields(isError(fields) ? { error: fields } : fields);
+    const built = buildMessage(message, callSite);
+    const record = {
+        timestamp: now.toISOString(),
+        level,
+        message: built.text,
+        logger: typeof logger === 'string' && logger ? logger : DEFAULT_LOGGER_NAME,
+    };
+    for (const key of RESOURCE_KEYS) {
+        if (typeof resource[key] === 'string' && resource[key]) {
+            record[key] = resource[key];
+        }
+    }
+    Object.assign(record, traceFields(spanContext));
+
+    const dropped = new Set();
+    const callSiteKeys = [];
+    // Call-site keys merged into the scope invocation; the size cap must reach them too.
+    const invocationDetailKeys = [];
+    const place = ({ entries, dropped: droppedHere }, isCallSite) => {
+        for (const key of droppedHere) dropped.add(key);
+        for (const [key, value] of entries) {
+            if (Object.prototype.hasOwnProperty.call(record, key)) {
+                if (
+                    key === INVOCATION_KEY &&
+                    isPlainObject(record[key]) &&
+                    isPlainObject(value)
+                ) {
+                    const merged = extendInvocation(record[key], value);
+                    if (isCallSite) {
+                        for (const detailKey of Object.keys(value)) {
+                            if (!(detailKey in record[key])) invocationDetailKeys.push(detailKey);
+                        }
+                    }
+                    record[key] = merged;
+                } else if (!sameValue(record[key], value)) {
+                    // A repeat of the winning value is no conflict.
+                    dropped.add(key);
+                }
+                continue;
+            }
+            record[key] = value;
+            if (isCallSite) callSiteKeys.push(key);
+        }
+    };
+    place(prepareScope(scope, dropPayloads), false);
+    place(prepareSource(bindings || {}, dropPayloads), false);
+    place(prepareSource(callSite, dropPayloads, built.error), true);
+
+    const setDroppedKeys = () => {
+        delete record[DROPPED_KEYS_FIELD];
+        if (dropped.size) record[DROPPED_KEYS_FIELD] = [...dropped];
+    };
+    setDroppedKeys();
+    let size = byteLength(record);
+
+    if (size > MAX_RECORD_BYTES) {
+        const candidates = [
+            ...callSiteKeys
+                .filter((key) => key !== 'error')
+                .map((key) => ({
+                    name: key,
+                    value: record[key],
+                    remove: () => delete record[key],
+                })),
+            ...invocationDetailKeys.map((key) => ({
+                name: `${INVOCATION_KEY}.${key}`,
+                value: record[INVOCATION_KEY][key],
+                remove: () => {
+                    const rest = { ...record[INVOCATION_KEY] };
+                    delete rest[key];
+                    record[INVOCATION_KEY] = rest;
+                },
+            })),
+        ];
+        const bySize = candidates
+            .map((candidate) => [candidate, Buffer.byteLength(JSON.stringify(candidate.value))])
+            .sort((a, b) => b[1] - a[1]);
+        for (const [candidate] of bySize) {
+            candidate.remove();
+            dropped.add(candidate.name);
+            setDroppedKeys();
+            size = byteLength(record);
+            if (size <= MAX_RECORD_BYTES) break;
+        }
+    }
+    for (const part of ['stack', 'cause']) {
+        if (
+            size > MAX_RECORD_BYTES &&
+            record.error &&
+            typeof record.error === 'object' &&
+            record.error[part] !== undefined
+        ) {
+            const { [part]: _removed, ...rest } = record.error;
+            record.error = rest;
+            size = byteLength(record);
+        }
+    }
+
+    return deepFreeze(record);
+}
+
+module.exports = {
+    buildRecord,
+    deepFreeze,
+    MAX_RECORD_BYTES,
+    RESERVED_KEYS,
+    PAYLOAD_KEYS,
+    DEFAULT_LOGGER_NAME,
+};

package/logs/redact.js

@@ -0,0 +1,328 @@
+const { CONTRACT_KEYS } = require('./contract-keys');
+
+const DENIED_SUFFIXES = [
+    'token',
+    'secret',
+    'password',
+    'apikey',
+    'privatekey',
+    'signature',
+    'authorization',
+    'cookie',
+    'pwd',
+    'passphrase',
+    'credentials',
+    'sessionid',
+];
+
+// Core encryption registry leaves and exact names the suffix rule does not
+// cover. `header` is Node's raw `_header` request text. `domain` is not
+// seeded (too common); a module can still register it as a credential field.
+const deniedKeys = new Set([
+    'hashword',
+    'cookies',
+    'apikeyvalue',
+    'mapping',
+    'auth',
+    'header',
+]);
+
+// Record-contract fields. A credential field with one of these names must not
+// blank the field in every record.
+const PROTECTED_KEYS = new Set(CONTRACT_KEYS.map(normalizeKey));
+
+const DENIED_SUFFIX = new RegExp(`(?:${DENIED_SUFFIXES.join('|')})$`);
+const PAIR_KEYS = new Set(['code', 'codeverifier']);
+const DIGEST_KEY = /(sha(1|256)|hash|digest|checksum)$/;
+
+const MAX_SCRUB_INPUT = 65536;
+
+const URL_IN_TEXT = /\b[a-zA-Z][a-zA-Z0-9+.-]*:\/\/[^\s"'<>`]+/g;
+const URL_TRAILING_PUNCT = /[.,;:!?)\]}]+$/;
+const ABSOLUTE_URL = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//;
+const RAW_ABSOLUTE_PATH = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\/[^/?#]*([^?#]*)/;
+const RAW_RELATIVE_PATH = /^[^?#]*/;
+const PATH_TOKEN_SEGMENT = /^[A-Za-z0-9_.~-]{20,}$/;
+const PREFIXED_TOKEN =
+    /(?<![A-Za-z0-9_-])(?:(?:sk|rk)_(?:live|test)_|whsec_|xox[abprs]-|gh[opsu]_|github_pat_|shp(?:at|ss)_|glpat-|npm_)[A-Za-z0-9_-]{16,}|\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
+const JWT = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*/g;
+const AUTH_SCHEME = /\b(Bearer|bearer|BEARER|Basic|basic|BASIC|Token|token|TOKEN)(\s+)([A-Za-z0-9._~+/=-]{8,})/g;
+const AUTH_SCHEME_WORD = /^(bearer|basic|token)$/i;
+const COLON_PAIR = /(?<![A-Za-z0-9_-])([A-Za-z_][A-Za-z0-9_.-]{0,99})(["']?)(\s*:\s*)(["']?)([^\s"',;{}]+)/g;
+const COOKIE_LINE = /(?<![A-Za-z0-9_-])((?:set-)?cookie)(\s*:\s*)([^\r\n]+)/gi;
+const CODE_PAIR = /(?<![A-Za-z0-9_.-])code=/;
+const JSON_PAIR = /"([^"\\]{1,100})"(\s*:\s*)"((?:[^"\\]|\\.)*)"/g;
+const TEXT_PAIR = /(?<![A-Za-z0-9_.-])([A-Za-z0-9_.-]{1,100})=([^\s&"'<>,;)]+)/g;
+const HEX_RUN = /(?<![A-Za-z0-9])[0-9a-fA-F]{40,}(?![A-Za-z0-9])/g;
+const BASE64_RUN = /(?<![A-Za-z0-9+/=_-])[A-Za-z0-9+/_-]{40,}={0,2}(?![A-Za-z0-9+/=_-])/g;
+const FORM_TEXT = /^[A-Za-z0-9_.%[\]-]+=[^&\s]*(?:&[A-Za-z0-9_.%[\]-]+=[^&\s]*)+$/;
+const PATH_SEGMENT = /^[A-Za-z0-9_-]*$/;
+
+function redacted(value) {
+    return `[REDACTED:${value.length}]`;
+}
+
+function normalizeKey(key) {
+    if (typeof key !== 'string') {
+        if (typeof key !== 'number') return '';
+        key = String(key);
+    }
+    return key.toLowerCase().replace(/[-_\s]/g, '');
+}
+
+// The *Normalized variants take a normalizeKey() result, so a walk
+// normalizes each key once.
+function isDeniedNormalized(normalized) {
+    if (!normalized || PROTECTED_KEYS.has(normalized)) return false;
+    return deniedKeys.has(normalized) || DENIED_SUFFIX.test(normalized);
+}
+
+function isDeniedKey(key) {
+    return isDeniedNormalized(normalizeKey(key));
+}
+
+function isDigestNormalized(normalized) {
+    return DIGEST_KEY.test(normalized);
+}
+
+function isDigestKey(key) {
+    return isDigestNormalized(normalizeKey(key));
+}
+
+function isPairKey(key, { withState = false } = {}) {
+    const normalized = normalizeKey(key);
+    return (
+        isDeniedKey(key) ||
+        PAIR_KEYS.has(normalized) ||
+        (withState && normalized === 'state')
+    );
+}
+
+// Returns the leaves it skipped as record-contract keys, so a caller can warn.
+function addDeniedKeys(keys) {
+    const ignored = [];
+    if (!Array.isArray(keys)) return ignored;
+    for (const key of keys) {
+        if (typeof key !== 'string') continue;
+        const rawLeaf = key.split('.').pop();
+        const leaf = normalizeKey(rawLeaf);
+        if (!leaf) continue;
+        if (PROTECTED_KEYS.has(leaf)) ignored.push(rawLeaf);
+        else deniedKeys.add(leaf);
+    }
+    return ignored;
+}
+
+function hasMixedClasses(run) {
+    return /[A-Z]/.test(run) && /[a-z]/.test(run) && /[0-9]/.test(run);
+}
+
+function isPathLike(run) {
+    if (!run.startsWith('/')) return false;
+    const segments = run.split('/');
+    if (segments.length < 3) return false;
+    return segments.every(
+        (segment) => segment.length < 40 && PATH_SEGMENT.test(segment)
+    );
+}
+
+function isRedactedValue(value) {
+    return value === 'REDACTED' || value.startsWith('[REDACTED');
+}
+
+function scrubTokens(text, { allowHex = false } = {}) {
+    let out = text.replace(PREFIXED_TOKEN, redacted);
+    out = out.replace(JWT, redacted);
+    out = out.replace(AUTH_SCHEME, (match, scheme, space, credential) => {
+        const classes = [/[A-Z]/, /[a-z]/, /[0-9]/, /[._~+/=-]/].filter((re) =>
+            re.test(credential)
+        ).length;
+        return classes >= 2 ? `${scheme}${space}${redacted(credential)}` : match;
+    });
+    out = out.replace(JSON_PAIR, (match, key, colon, value) =>
+        isDeniedKey(key) || normalizeKey(key) === 'codeverifier'
+            ? `"${key}"${colon}"${redacted(value)}"`
+            : match
+    );
+    out = out.replace(COOKIE_LINE, (match, key, colon, value) =>
+        isRedactedValue(value) ? match : `${key}${colon}${redacted(value)}`
+    );
+    out = out.replace(COLON_PAIR, (match, key, close, colon, open, value) =>
+        isDeniedKey(key) &&
+        !isRedactedValue(value) &&
+        !AUTH_SCHEME_WORD.test(value)
+            ? `${key}${close}${colon}${open}${redacted(value)}`
+            : match
+    );
+    const withState = CODE_PAIR.test(out);
+    out = out.replace(TEXT_PAIR, (match, key, value) =>
+        isPairKey(key, { withState }) && !isRedactedValue(value)
+            ? `${key}=${redacted(value)}`
+            : match
+    );
+    if (!allowHex) out = out.replace(HEX_RUN, redacted);
+    out = out.replace(BASE64_RUN, (run) =>
+        hasMixedClasses(run) && !isPathLike(run)
+            ? redacted(run)
+            : run
+    );
+    return out;
+}
+
+function queryKeys(search) {
+    return search
+        .replace(/^[?#]/, '')
+        .split('&')
+        .map((pair) => pair.split('=')[0])
+        .filter(Boolean);
+}
+
+function redactParams(search, prefix) {
+    const keys = queryKeys(search);
+    if (keys.length === 0) return '';
+    return `${prefix}${keys.map((key) => `${key}=REDACTED`).join('&')}`;
+}
+
+function redactHash(hash) {
+    if (!hash) return '';
+    if (hash.includes('=')) return redactParams(hash, '#');
+    return scrubTokens(hash);
+}
+
+// A long segment mixing upper case, lower case and digits is an opaque
+// token (webhook secrets live in paths); hex ids and UUIDs lack one class.
+function redactPathSegment(segment) {
+    if (PATH_TOKEN_SEGMENT.test(segment) && hasMixedClasses(segment)) {
+        return redacted(segment);
+    }
+    return scrubTokens(segment);
+}
+
+// The raw path, not URL#pathname, which percent-encodes `{proxy+}`.
+function redactPath(pathname) {
+    return pathname.split('/').map(redactPathSegment).join('/');
+}
+
+function redactUrlFallback(raw) {
+    const out = raw
+        .replace(/(:\/\/)[^/?#@\s]*@/, '$1')
+        .replace(/([?&#][^=&#\s]+)=[^&#\s]*/g, '$1=REDACTED');
+    return scrubTokens(out);
+}
+
+function redactUrl(url) {
+    if (url === undefined || url === null) return '';
+    let raw;
+    try {
+        raw = url instanceof URL ? url.href : String(url);
+    } catch {
+        return '';
+    }
+    try {
+        if (ABSOLUTE_URL.test(raw)) {
+            const u = new URL(raw);
+            const path = raw.match(RAW_ABSOLUTE_PATH)[1];
+            return `${u.protocol}//${u.host}${redactPath(path)}${redactParams(
+                u.search,
+                '?'
+            )}${redactHash(u.hash)}`;
+        }
+        if (raw.startsWith('/')) {
+            const u = new URL(raw, 'https://relative.invalid');
+            return `${redactPath(raw.match(RAW_RELATIVE_PATH)[0])}${redactParams(
+                u.search,
+                '?'
+            )}${redactHash(u.hash)}`;
+        }
+    } catch {
+        return redactUrlFallback(raw);
+    }
+    return redactUrlFallback(raw);
+}
+
+function looksLikeJson(trimmed) {
+    if (trimmed.startsWith('{')) return trimmed.endsWith('}');
+    return /^\[\s*[{["\]\d-]/.test(trimmed) && trimmed.endsWith(']');
+}
+
+function scrubJsonText(text, originalLength) {
+    try {
+        return JSON.stringify(redactValue(JSON.parse(text)));
+    } catch {
+        return `[unparsed:${originalLength}]`;
+    }
+}
+
+function scrubFormText(text) {
+    const pairs = text.split('&').map((pair) => {
+        const index = pair.indexOf('=');
+        return [pair.slice(0, index), pair.slice(index + 1)];
+    });
+    const decodedKeys = pairs.map(([key]) => {
+        try {
+            return decodeURIComponent(key);
+        } catch {
+            return key;
+        }
+    });
+    const dropAll = isOAuthCallbackShape(decodedKeys);
+    return pairs
+        .map(([key, value], i) => {
+            if (dropAll || isPairKey(decodedKeys[i])) {
+                return `${key}=${redacted(value)}`;
+            }
+            return `${key}=${scrubTokens(value)}`;
+        })
+        .join('&');
+}
+
+function scrubText(text, options) {
+    const withUrls = text.replace(URL_IN_TEXT, (match) => {
+        const trailing = match.match(URL_TRAILING_PUNCT)?.[0] ?? '';
+        const core = trailing ? match.slice(0, -trailing.length) : match;
+        return `${redactUrl(core)}${trailing}`;
+    });
+    return scrubTokens(withUrls, options);
+}
+
+function scrubString(str, { allowHex = false } = {}) {
+    if (typeof str !== 'string' || str.length === 0) return str;
+    try {
+        const text =
+            str.length > MAX_SCRUB_INPUT ? str.slice(0, MAX_SCRUB_INPUT) : str;
+        const trimmed = text.trim();
+        if (looksLikeJson(trimmed)) return scrubJsonText(trimmed, str.length);
+        if (FORM_TEXT.test(trimmed)) return scrubFormText(trimmed);
+        return scrubText(text, { allowHex });
+    } catch {
+        return redacted(str);
+    }
+}
+
+function isOAuthCallbackShapeNormalized(normalizedKeys) {
+    const keys = new Set(normalizedKeys);
+    return keys.has('code') && (keys.has('state') || keys.has('codeverifier'));
+}
+
+function isOAuthCallbackShape(keys) {
+    return isOAuthCallbackShapeNormalized(keys.map(normalizeKey));
+}
+
+function redactValue(value) {
+    const { serializeValue } = require('./serialize');
+    return serializeValue(value);
+}
+
+module.exports = {
+    normalizeKey,
+    isDeniedKey,
+    isDeniedNormalized,
+    isDigestKey,
+    isDigestNormalized,
+    isOAuthCallbackShape,
+    isOAuthCallbackShapeNormalized,
+    addDeniedKeys,
+    redactUrl,
+    scrubString,
+    redactValue,
+};