@turns/dev 0.0.1

package/dist/secret-redaction.js

@@ -0,0 +1,181 @@
+/**
+ * Machine-local secret filtering for outbound transcript data.
+ *
+ * The uploader keeps the original rows for local parsing, prompt matching, and
+ * harness operation. Only values about to cross the turns.dev socket are
+ * rewritten. This is intentionally a credential filter, not a PII filter:
+ * names, email addresses, paths, identifiers, and public keys remain intact.
+ */
+export const REDACTION_VERSION = 1;
+export const REDACTED_SECRET = '[REDACTED_SECRET]';
+const PLACEHOLDER = /^(?:x{8,}|\*{4,}|\.{3,}|<[^>]*>|\[[^\]]*\]|(?:your|my|the|some|example|sample|test|fake|dummy|placeholder|changeme|redacted)[-_a-z0-9]*)$/iu;
+const MIN_CONTEXT_VALUE_LENGTH = 16;
+const MIN_CONTEXT_ENTROPY = 3;
+const STRUCTURAL_RULES = [
+    {
+        name: 'private_key_block',
+        pattern: /-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----[\s\S]*?(?:-----END (?:[A-Z ]+ )?PRIVATE KEY-----|$)/gu,
+        replacement: () => '[REDACTED_PRIVATE_KEY]',
+    },
+    {
+        name: 'url_credentials',
+        pattern: /\b([a-z][a-z0-9+.-]*:\/\/)([^\s/:@]+):([^\s/@]+)@/giu,
+        replacement: (_match, scheme, user) => `${scheme}${user}:${REDACTED_SECRET}@`,
+    },
+];
+const PREFIX_RULES = [
+    ['anthropic_key', /\bsk-ant-[A-Za-z0-9_-]{20,}/gu],
+    ['openai_project_key', /\bsk-proj-[A-Za-z0-9_-]{20,}/gu],
+    ['openai_key', /\bsk-[A-Za-z0-9_-]{20,}/gu],
+    ['github_pat', /\bgithub_pat_[A-Za-z0-9_]{20,}/gu],
+    ['github_token', /\bgh[ouspr]_[A-Za-z0-9]{20,}/gu],
+    ['npm_token', /\bnpm_[A-Za-z0-9]{30,}/gu],
+    ['slack_token', /\bxox[baprs]-[A-Za-z0-9-]{10,}/gu],
+    ['gitlab_pat', /\bglpat-[A-Za-z0-9_-]{16,}/gu],
+    ['aws_access_key', /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/gu],
+    ['google_api_key', /\bAIza[A-Za-z0-9_-]{35}\b/gu],
+    ['google_oauth_token', /\bya29\.[A-Za-z0-9_-]{20,}/gu],
+    ['huggingface_token', /\bhf_[A-Za-z0-9]{30,}/gu],
+    ['digitalocean_token', /\bdop_v1_[a-f0-9]{60,}/gu],
+    ['stripe_key', /\b[rs]k_(?:live|test)_[A-Za-z0-9]{20,}/gu],
+    ['jwt', /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/gu],
+];
+const NON_SECRET_KEY = new RegExp([
+    '(?:^|[._-])(?:pub|public)[._-]?key',
+    'site[._-]?key',
+    'key[._-]?(?:id|name|file|path|type|size|length|algorithm|alg)\\b',
+    '(?:^|[._-])key[._-]?(?:pair|store|ring)',
+    'id$',
+    'fingerprint|thumbprint|checksum|digest|(?:^|[._-])hash',
+].join('|'), 'iu');
+const CREDENTIAL_KEY = /[A-Za-z0-9_.-]*(?:token|secret|key|password|passwd|pwd|credential|apikey|auth)[A-Za-z0-9_.-]*/iu;
+const CREDENTIAL_WORD = /token|secret|key|password|passwd|pwd|credential|apikey|auth/iu;
+const CONTEXT_RULE = new RegExp(`(["']?)(${CREDENTIAL_KEY.source})\\1(\\s*[:=]\\s*)(["']?)([^\\s"'\`,;)}\\]]{${MIN_CONTEXT_VALUE_LENGTH},})\\4`, 'giu');
+const OPAQUE_VALUE = /^[A-Za-z0-9_\-+/.~]+={0,2}$/u;
+const DOTTED_IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)+$/u;
+export function shannonEntropy(value) {
+    if (!value)
+        return 0;
+    const counts = new Map();
+    for (const character of value)
+        counts.set(character, (counts.get(character) || 0) + 1);
+    let entropy = 0;
+    for (const count of counts.values()) {
+        const probability = count / value.length;
+        entropy -= probability * Math.log2(probability);
+    }
+    return entropy;
+}
+function looksLikeIdentifier(value) {
+    const segments = value.split(/[_\-.]+|(?<=[a-z0-9])(?=[A-Z])/u).filter(Boolean);
+    return segments.length > 0 && segments.every((segment) => /^[A-Za-z]{2,}$/u.test(segment));
+}
+export function looksLikeSecretValue(value) {
+    if (typeof value !== 'string' || value.length < MIN_CONTEXT_VALUE_LENGTH)
+        return false;
+    if (PLACEHOLDER.test(value) || value.startsWith('[REDACTED'))
+        return false;
+    if (!OPAQUE_VALUE.test(value) || DOTTED_IDENTIFIER.test(value) || looksLikeIdentifier(value))
+        return false;
+    if (/^(?:[~.]{0,2}\/|[A-Za-z]:[\\/])/u.test(value))
+        return false;
+    if (/^[a-z][a-z0-9+.-]*:\/\//iu.test(value))
+        return false;
+    return shannonEntropy(value) >= MIN_CONTEXT_ENTROPY;
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+}
+/** Build a reusable, idempotent secret redactor. */
+export function createSecretRedactor({ literals = [], onFinding = null, detectOnly = false } = {}) {
+    const knownLiterals = new Set();
+    let exactRules = [];
+    const addLiterals = (values) => {
+        for (const value of values || []) {
+            const literal = String(value || '');
+            if (literal.length >= 8)
+                knownLiterals.add(literal);
+        }
+        exactRules = [...knownLiterals]
+            .sort((left, right) => right.length - left.length)
+            .map((value, index) => [`known_literal_${index}`, new RegExp(escapeRegExp(value), 'gu')]);
+    };
+    addLiterals(literals);
+    const report = (rule, count) => {
+        if (count && typeof onFinding === 'function')
+            onFinding({ rule, count });
+    };
+    const redactText = (text) => {
+        if (typeof text !== 'string' || !text)
+            return text;
+        let output = text;
+        for (const [name, pattern] of exactRules) {
+            let count = 0;
+            output = output.replace(pattern, () => {
+                count += 1;
+                return REDACTED_SECRET;
+            });
+            report(name, count);
+        }
+        for (const rule of STRUCTURAL_RULES) {
+            let count = 0;
+            output = output.replace(rule.pattern, (...arguments_) => {
+                count += 1;
+                return rule.replacement(...arguments_);
+            });
+            report(rule.name, count);
+        }
+        for (const [name, pattern] of PREFIX_RULES) {
+            let count = 0;
+            output = output.replace(pattern, () => {
+                count += 1;
+                return REDACTED_SECRET;
+            });
+            report(name, count);
+        }
+        let contextual = 0;
+        // Avoid the contextual expression entirely for large tool output that does
+        // not even contain a credential word. Besides being the common case, this
+        // keeps filtering linear for multi-megabyte homogeneous stdout frames.
+        if (CREDENTIAL_WORD.test(output)) {
+            output = output.replace(CONTEXT_RULE, (match, keyQuote, key, separator, valueQuote, value) => {
+                if (NON_SECRET_KEY.test(key) || !looksLikeSecretValue(value))
+                    return match;
+                contextual += 1;
+                return `${keyQuote}${key}${keyQuote}${separator}${valueQuote}${REDACTED_SECRET}${valueQuote}`;
+            });
+        }
+        report('assigned_credential', contextual);
+        return detectOnly ? text : output;
+    };
+    const redactValue = (value, depth = 0, key = '') => {
+        if (depth > 24)
+            return value;
+        if (typeof value === 'string') {
+            if (CREDENTIAL_KEY.test(key) && !NON_SECRET_KEY.test(key) && looksLikeSecretValue(value)) {
+                report('assigned_credential', 1);
+                return detectOnly ? value : REDACTED_SECRET;
+            }
+            return redactText(value);
+        }
+        if (Array.isArray(value))
+            return value.map((item) => redactValue(item, depth + 1));
+        if (!value || typeof value !== 'object')
+            return value;
+        return Object.fromEntries(Object.entries(value).map(([childKey, child]) => [childKey, redactValue(child, depth + 1, childKey)]));
+    };
+    const redactJsonLine = (raw) => {
+        if (typeof raw !== 'string' || !raw)
+            return raw;
+        const trimmed = raw.trimStart();
+        if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+            try {
+                const redacted = redactValue(JSON.parse(raw));
+                return detectOnly ? raw : JSON.stringify(redacted);
+            }
+            catch { }
+        }
+        return redactText(raw);
+    };
+    return { redactText, redactValue, redactJsonLine, addLiterals, detectOnly };
+}