engineering-memory 1.11.9 → 1.11.11

package/runtime/dist/src/runtime/offline-outbox.js

@@ -3,140 +3,9 @@ import { readdir } from 'node:fs/promises';
 import { join } from 'node:path';
 import { assertManagedPath, ensureManagedDirectory, readJson, removeFile, writeJson, } from '../utilities/files.js';
 import { sha256, stableStringify } from '../utilities/hash.js';
-const sensitiveKeyPattern = /(authorization|password|passcode|secret|token|cookie|email|phone|mobile|customer|account|user.?id|user.?name)/i;
-const emailSource = '(?:^|[^\\p{L}\\p{N}._%+-])[\\p{L}\\p{N}._%+-]{1,64}@([\\p{L}\\p{N}.-]+\\.[\\p{L}]{2,63})(?=$|[^\\p{L}\\p{N}.-])';
-const emailPattern = new RegExp(emailSource, 'iu');
-const emailMatchPattern = new RegExp(emailSource, 'giu');
-const phonePattern = /\+\d[\d ()-]{7,}\d/;
-const repositoryPathFields = new Set([
-    'path',
-    'paths',
-    'originalPath',
-    'changedPaths',
-    'pathPatterns',
-    'evidencePaths',
-    'screenPathPatterns',
-    'componentPathPatterns',
-    'pathChanges',
-    'templatePath',
-]);
-const topLevelDomains = new Set([
-    'com',
-    'net',
-    'org',
-    'edu',
-    'gov',
-    'mil',
-    'int',
-    'info',
-    'biz',
-    'name',
-    'pro',
-    'mobi',
-    'tel',
-    'asia',
-    'jobs',
-    'coop',
-    'aero',
-    'museum',
-    'travel',
-    'app',
-    'dev',
-    'cloud',
-    'online',
-    'site',
-    'store',
-    'tech',
-    'space',
-    'website',
-    'digital',
-    'agency',
-    'systems',
-    'solutions',
-    'email',
-    'xyz',
-    'top',
-    'club',
-    'live',
-    'life',
-    'world',
-    'today',
-    'network',
-    'group',
-    'company',
-    'center',
-    'media',
-    'studio',
-    'design',
-    'software',
-    'tools',
-    'works',
-    'team',
-    'zone',
-    'one',
-    'ltd',
-    'inc',
-    'llc',
-    'gmbh',
-    'shop',
-    'page',
-    'blog',
-    'wiki',
-    'news',
-    'art',
-    'link',
-    'host',
-    'press',
-    'fyi',
-    'test',
-    'example',
-    'invalid',
-    'local',
-    'localhost',
-    'internal',
-]);
-const countryCodeTopLevelDomains = new Set('ac ad ae af ag ai al am ao aq ar as at au aw ax az ba bb bd be bf bg bh bi bj bm bn bo br bs bt bw by bz ca cc cd cf cg ch ci ck cl cm cn co cr cu cv cw cx cy cz de dj dk dm do dz ec ee eg er es et eu fi fj fk fm fo fr ga gd ge gf gg gh gi gl gm gn gp gq gr gs gt gu gw gy hk hm hn hr ht hu id ie il im in io iq ir is it je jm jo jp ke kg kh ki km kn kp kr kw ky kz la lb lc li lk lr ls lt lu lv ly ma mc md me mg mh mk ml mm mn mo mp mq mr ms mt mu mv mw mx my mz na nc ne nf ng ni nl no np nr nu nz om pa pe pf pg ph pk pl pm pn pr ps pt pw py qa re ro rs ru rw sa sb sc sd se sg sh si sk sl sm sn so sr ss st su sv sx sy sz tc td tf tg th tj tk tl tm tn to tr tt tv tw tz ua ug uk us uy uz va vc ve vg vi vn vu wf ws ye yt za zm zw'.split(' '));
-function isTopLevelDomain(label) {
-    const value = label.toLowerCase();
-    return countryCodeTopLevelDomains.has(value) || topLevelDomains.has(value);
-}
-const extensionTopLevelDomains = new Set([
-    'ai',
-    'app',
-    'as',
-    'cc',
-    'md',
-    'ml',
-    'mm',
-    'pl',
-    'py',
-    'rs',
-    'sh',
-    'so',
-]);
-function hasEmailAddress(value, repositoryPath) {
-    if (!repositoryPath)
-        return emailPattern.test(value);
-    for (const match of value.matchAll(emailMatchPattern)) {
-        const labels = (match[1] ?? '').split('.');
-        const last = (labels[labels.length - 1] ?? '').toLowerCase();
-        if (isTopLevelDomain(last) && !extensionTopLevelDomains.has(last))
-            return true;
-        const beforeExtension = (labels[labels.length - 2] ?? '').toLowerCase();
-        if (labels.length >= 3 &&
-            topLevelDomains.has(beforeExtension) &&
-            !extensionTopLevelDomains.has(beforeExtension))
-            return true;
-    }
-    return false;
-}
-const knownCredentialPattern = /\b(?:A(?:KI|SI)A[0-9A-Z]{16}|gh[pousr]_[A-Z0-9]{30,}|github_pat_[A-Z0-9_]{30,}|sk_live_[A-Z0-9]{16,}|rk_live_[A-Z0-9]{16,}|xox[baprs]-[A-Z0-9-]{12,}|AIza[A-Z0-9_-]{35}|sk-(?:proj-)?[A-Z0-9_-]{20,})\b/i;
-const bearerPattern = /\bbearer\s+[a-z0-9._~-]+/i;
-const jwtPattern = /\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\b/;
-const privateKeyPattern = /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/;
-const credentialUrlPattern = /\b[a-z][a-z0-9+.-]*:\/\/[^\s/@:]+:[^\s/@]+@/i;
-const ipv4Pattern = /\b(?:\d{1,3}\.){3}\d{1,3}\b/;
-const personalHomePathPattern = /(?:[A-Z]:[\\/](?:Users|Documents and Settings)[\\/][^\\/\s"'<>:|?*]+|(?:^|[\s"'(<=])\/(?:Users|home)\/[^/\s"'<>*]+)(?:[\\/]|$)/i;
+import { restrictedFieldKind, restrictedNumberKind, restrictedValueKind, } from './privacy-detector.js';
+const controlFields = new Set(['claimtoken', 'cookie', 'setcookie']);
+const personalCacheFieldPattern = /(email|phone|mobile|user.?id|user.?name)/i;
 export class OfflineOutbox {
     root;
     queues = new Map();
@@ -282,33 +151,39 @@ export class OfflineOutbox {
         }
     }
 }
-export function assertSafeToPersist(value, key = '') {
-    if (key && sensitiveKeyPattern.test(key)) {
-        throw new Error(`Sensitive field cannot be persisted: ${key}`);
-    }
-    if (typeof value === 'string') {
-        const repositoryPath = repositoryPathFields.has(key);
-        if (hasEmailAddress(value, repositoryPath) ||
-            phonePattern.test(value) ||
-            bearerPattern.test(value) ||
-            jwtPattern.test(value) ||
-            privateKeyPattern.test(value) ||
-            knownCredentialPattern.test(value) ||
-            credentialUrlPattern.test(value) ||
-            personalHomePathPattern.test(value) ||
-            (!repositoryPath && ipv4Pattern.test(value))) {
-            throw new Error(`PII or credentials cannot be persisted: ${key}`);
+export function assertSafeToPersist(value, key = '', path = key) {
+    if (typeof value === 'string' || typeof value === 'number') {
+        const detector = typeof value === 'string'
+            ? restrictedValueKind(value, key)
+            : restrictedNumberKind(value, key);
+        if (detector) {
+            throw new Error(`PII or credentials cannot be persisted: ${path || '(value)'} (${detector})`);
         }
         return;
     }
     if (Array.isArray(value)) {
-        value.forEach((item) => assertSafeToPersist(item, key));
+        value.forEach((item, index) => assertSafeToPersist(item, key, `${path}[${index}]`));
         return;
     }
     if (value !== null && typeof value === 'object') {
-        Object.entries(value).forEach(([childKey, item]) => assertSafeToPersist(item, childKey));
+        Object.entries(value).forEach(([childKey, item]) => {
+            const field = sensitiveFieldKind(childKey, item);
+            if (field) {
+                throw new Error(`Sensitive field cannot be persisted: ${childPath(path, childKey)} (${field})`);
+            }
+            assertSafeToPersist(item, childKey, childPath(path, childKey));
+        });
     }
 }
+function sensitiveFieldKind(key, value) {
+    if (controlFields.has(key.replace(/[^a-z0-9]/gi, '').toLowerCase()))
+        return 'bridge control field';
+    return (restrictedFieldKind(key, value) ??
+        (typeof value === 'number' ? restrictedNumberKind(value, key) : null));
+}
+function childPath(path, key) {
+    return path ? `${path}.${key}` : key;
+}
 export function normalizeRepositoryPaths(value, repoRoot) {
     const root = trimTrailingSeparators(repoRoot);
     if (!root) {
@@ -333,21 +208,21 @@ export function redactForCache(value) {
         return null;
     }
 }
-function redactValue(value, key = '') {
+function redactValue(value, key = '', path = key) {
     if (typeof value === 'string') {
-        assertSafeToPersist(value, key);
+        assertSafeToPersist(value, key, path);
         return value;
     }
     if (Array.isArray(value)) {
-        return value.map((entry) => redactValue(entry, key));
+        return value.map((entry, index) => redactValue(entry, key, `${path}[${index}]`));
     }
     if (value !== null && typeof value === 'object') {
         const result = {};
-        for (const [key, entry] of Object.entries(value)) {
-            if (sensitiveKeyPattern.test(key)) {
+        for (const [childKey, entry] of Object.entries(value)) {
+            if (personalCacheFieldPattern.test(childKey) || sensitiveFieldKind(childKey, entry)) {
                 continue;
             }
-            result[key] = redactValue(entry, key);
+            result[childKey] = redactValue(entry, childKey, childPath(path, childKey));
         }
         return result;
     }

package/runtime/dist/src/runtime/phase-timer.js

@@ -0,0 +1,26 @@
+const enabled = Boolean(process.env.ENGINEERING_MEMORY_BRIDGE_DEBUG);
+function write(line) {
+    process.stderr.write(`${line}\n`);
+}
+export function startPhaseTimer(operation) {
+    if (!enabled) {
+        return { mark: () => { }, finish: () => { } };
+    }
+    const start = process.hrtime.bigint();
+    let last = start;
+    const phases = [];
+    return {
+        mark(phase) {
+            const now = process.hrtime.bigint();
+            phases.push({ phase, ms: Number(now - last) / 1e6 });
+            last = now;
+        },
+        finish() {
+            const now = process.hrtime.bigint();
+            const totalMs = Number(now - start) / 1e6;
+            const breakdown = phases.map((entry) => `${entry.phase}=${entry.ms.toFixed(1)}ms`).join(' ');
+            write(`[engineering-memory bridge] ${operation} total=${totalMs.toFixed(1)}ms ${breakdown}`);
+        },
+    };
+}
+//# sourceMappingURL=phase-timer.js.map

package/runtime/dist/src/runtime/privacy-detector.js

@@ -0,0 +1,263 @@
+const emailSource = '(?:^|[^\\p{L}\\p{N}._%+-])[\\p{L}\\p{N}._%+-]{1,64}@([\\p{L}\\p{N}.-]+\\.[\\p{L}]{2,63})(?=$|[^\\p{L}\\p{N}.-])';
+const emailPattern = new RegExp(emailSource, 'iu');
+const emailMatchPattern = new RegExp(emailSource, 'giu');
+const repositoryPathFields = new Set([
+    'path',
+    'paths',
+    'originalPath',
+    'changedPaths',
+    'pathPatterns',
+    'evidencePaths',
+    'screenPathPatterns',
+    'componentPathPatterns',
+    'pathChanges',
+    'templatePath',
+]);
+const topLevelDomains = new Set([
+    'com',
+    'net',
+    'org',
+    'edu',
+    'gov',
+    'mil',
+    'int',
+    'info',
+    'biz',
+    'name',
+    'pro',
+    'mobi',
+    'tel',
+    'asia',
+    'jobs',
+    'coop',
+    'aero',
+    'museum',
+    'travel',
+    'app',
+    'dev',
+    'cloud',
+    'online',
+    'site',
+    'store',
+    'tech',
+    'space',
+    'website',
+    'digital',
+    'agency',
+    'systems',
+    'solutions',
+    'email',
+    'xyz',
+    'top',
+    'club',
+    'live',
+    'life',
+    'world',
+    'today',
+    'network',
+    'group',
+    'company',
+    'center',
+    'media',
+    'studio',
+    'design',
+    'software',
+    'tools',
+    'works',
+    'team',
+    'zone',
+    'one',
+    'ltd',
+    'inc',
+    'llc',
+    'gmbh',
+    'shop',
+    'page',
+    'blog',
+    'wiki',
+    'news',
+    'art',
+    'link',
+    'host',
+    'press',
+    'fyi',
+    'test',
+    'example',
+    'invalid',
+    'local',
+    'localhost',
+    'internal',
+]);
+const countryCodeTopLevelDomains = new Set('ac ad ae af ag ai al am ao aq ar as at au aw ax az ba bb bd be bf bg bh bi bj bm bn bo br bs bt bw by bz ca cc cd cf cg ch ci ck cl cm cn co cr cu cv cw cx cy cz de dj dk dm do dz ec ee eg er es et eu fi fj fk fm fo fr ga gd ge gf gg gh gi gl gm gn gp gq gr gs gt gu gw gy hk hm hn hr ht hu id ie il im in io iq ir is it je jm jo jp ke kg kh ki km kn kp kr kw ky kz la lb lc li lk lr ls lt lu lv ly ma mc md me mg mh mk ml mm mn mo mp mq mr ms mt mu mv mw mx my mz na nc ne nf ng ni nl no np nr nu nz om pa pe pf pg ph pk pl pm pn pr ps pt pw py qa re ro rs ru rw sa sb sc sd se sg sh si sk sl sm sn so sr ss st su sv sx sy sz tc td tf tg th tj tk tl tm tn to tr tt tv tw tz ua ug uk us uy uz va vc ve vg vi vn vu wf ws ye yt za zm zw'.split(' '));
+function isTopLevelDomain(label) {
+    const value = label.toLowerCase();
+    return countryCodeTopLevelDomains.has(value) || topLevelDomains.has(value);
+}
+const extensionTopLevelDomains = new Set([
+    'ai',
+    'app',
+    'as',
+    'cc',
+    'md',
+    'ml',
+    'mm',
+    'pl',
+    'py',
+    'rs',
+    'sh',
+    'so',
+]);
+function hasEmailAddress(value, repositoryPath) {
+    if (!repositoryPath)
+        return emailPattern.test(value);
+    for (const match of value.matchAll(emailMatchPattern)) {
+        const labels = (match[1] ?? '').split('.');
+        const last = (labels[labels.length - 1] ?? '').toLowerCase();
+        if (isTopLevelDomain(last) && !extensionTopLevelDomains.has(last))
+            return true;
+        const beforeExtension = (labels[labels.length - 2] ?? '').toLowerCase();
+        if (labels.length >= 3 &&
+            topLevelDomains.has(beforeExtension) &&
+            !extensionTopLevelDomains.has(beforeExtension))
+            return true;
+    }
+    return false;
+}
+const bearerPattern = /\bbearer\s+([A-Z0-9._~+/=-]{12,})/i;
+const jwtPattern = /(?:^|[^A-Z0-9_-])eyJ[A-Z0-9_-]{5,}\.[A-Z0-9_-]{8,}\.[A-Z0-9_-]{8,}(?=$|[^A-Z0-9_-])/i;
+const privateKeyPattern = /-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/i;
+const knownCredentialPattern = /\b(?:A(?:KI|SI)A[0-9A-Z]{16}|gh[pousr]_[A-Z0-9]{30,}|github_pat_[A-Z0-9_]{30,}|sk_live_[A-Z0-9]{16,}|rk_live_[A-Z0-9]{16,}|xox[baprs]-[A-Z0-9-]{12,}|AIza[A-Z0-9_-]{35}|sk-(?:proj-)?[A-Z0-9_-]{20,})\b/i;
+const credentialUrlPattern = /\b[A-Z][A-Z0-9+.-]{1,15}:\/\/[^/\s:@]+:[^/\s@]{4,}@/i;
+const secretAssignmentPattern = /(?:^|[\s{[(,;])["']?(password|passwd|pwd|secret|credential|client[_-]?secret|api[_-]?key|access[_-]?key|private[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|authorization)["']?\s*(=>|:=|=|:)\s*(?:"([^"\r\n]*)"|'([^'\r\n]*)'|([^\s,;}\]\r\n]+))/im;
+const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const secretFieldPattern = /^(?:password|passwd|pwd|secret|credential|clientsecret|apikey|accesskey|privatekey|token|accesstoken|refreshtoken|idtoken|authorization)$/;
+const phoneFieldPattern = /^(?:phone|tel|telephone|mobile|gsm|msisdn|cell|fax|whatsapp)$/;
+const placeholderPattern = /^(?:null|undefined|true|false|string|number|boolean|required|optional|redacted|masked|example|sample|placeholder|password|secret|token|none)$/i;
+const templatePattern = /^(?:<[^>]+>|\{\{[^}]+\}\}|\$\{[^}]+\}|\*+)$/;
+const referencePattern = /^(?:process\.env|deno\.env|env\.|config\.|settings\.|request\.|dto\.|input\.|controller\.|model\.|user\.|secretref|vaultref)/i;
+const bearerPlaceholderPattern = /^(?:your[_-]|example[_-]|sample[_-]|placeholder|redacted|masked|synthetic|x{8,}|\*+)/i;
+const personalHomePathPattern = /(?:[A-Z]:[\\/](?:Users|Documents and Settings)[\\/][^\\/\s"'<>:|?*]+|(?:^|[\s"'(<=])\/(?:Users|home)\/[^/\s"'<>*]+)(?:[\\/]|$)/i;
+const mountedHomePathPattern = /(?:\/mnt\/[a-z]\/Users\/|\/cygdrive\/[a-z]\/Users\/|(?:^|[\s"'(])\/[a-z]\/Users\/|\/System\/Volumes\/Data\/(?:Users|home)\/|\/Volumes\/[^/\s]+\/(?:Users|home)\/|\\\\[^\\\s]+\\Users\\|\/export\/home\/)[^/\\\s"'<>*]+/i;
+function hasPersonalHomePath(value) {
+    return personalHomePathPattern.test(value) || mountedHomePathPattern.test(value);
+}
+export function restrictedFieldKind(key, value) {
+    return isSecretField(key) && isSecretValue(value) ? 'secret field with a credential value' : null;
+}
+export function restrictedNumberKind(value, key) {
+    return phoneFieldPattern.test(normalizedKey(key)) &&
+        Number.isSafeInteger(value) &&
+        hasPhoneNumber(String(value))
+        ? 'phone number'
+        : null;
+}
+export function restrictedValueKind(value, key) {
+    const repositoryPath = repositoryPathFields.has(key);
+    if (hasEmailAddress(value, repositoryPath))
+        return 'email address';
+    if (privateKeyPattern.test(value))
+        return 'private key';
+    if (jwtPattern.test(value))
+        return 'signed token';
+    if (knownCredentialPattern.test(value))
+        return 'credential';
+    if (credentialUrlPattern.test(value))
+        return 'credential in a URL';
+    if (hasPersonalHomePath(value))
+        return 'personal home directory';
+    if (hasBearerCredential(value))
+        return 'bearer credential';
+    if (hasPhoneNumber(value))
+        return 'phone number';
+    if (!repositoryPath && hasIpv4Address(value))
+        return 'network address';
+    if (hasSecretAssignment(value))
+        return 'assigned secret';
+    return null;
+}
+function hasSecretAssignment(value) {
+    const matches = value.matchAll(new RegExp(secretAssignmentPattern.source, 'gim'));
+    for (const match of matches) {
+        const key = match[1] ?? '';
+        const delimiter = match[2] ?? '';
+        const doubleQuoted = match[3];
+        const singleQuoted = match[4];
+        const raw = match[5];
+        const candidate = doubleQuoted ?? singleQuoted ?? raw ?? '';
+        if (isSecretString(candidate, doubleQuoted !== undefined || singleQuoted !== undefined, delimiter, key)) {
+            return true;
+        }
+    }
+    return false;
+}
+function hasBearerCredential(value) {
+    const candidate = bearerPattern.exec(value)?.[1];
+    return Boolean(candidate && !bearerPlaceholderPattern.test(candidate));
+}
+function normalizedKey(key) {
+    return key.replace(/[^a-z0-9]/gi, '').toLowerCase();
+}
+function isSecretField(key) {
+    return secretFieldPattern.test(normalizedKey(key));
+}
+function isSecretValue(value) {
+    if (typeof value === 'string')
+        return isSecretString(value, true, ':', '');
+    if (typeof value === 'number')
+        return String(Math.abs(value)).length >= 8;
+    return false;
+}
+function isSecretString(value, quoted, delimiter, key) {
+    const candidate = value.trim();
+    if (candidate.length < 8)
+        return false;
+    if (placeholderPattern.test(candidate) ||
+        templatePattern.test(candidate) ||
+        referencePattern.test(candidate) ||
+        candidate.startsWith('${') ||
+        candidate.startsWith('{') ||
+        candidate.startsWith('[')) {
+        return false;
+    }
+    if (!quoted && /^[A-Z_$][A-Z0-9_$?.<>[\].]*$/i.test(candidate)) {
+        return delimiter === '=' && key === key.toUpperCase();
+    }
+    return true;
+}
+function hasIpv4Address(value) {
+    const matches = value.matchAll(/(?:^|[^\d.])((?:\d{1,3}\.){3}\d{1,3})(?![\d.])/g);
+    for (const match of matches) {
+        const candidate = match[1];
+        if (candidate && candidate.split('.').every((part) => Number(part) <= 255)) {
+            return true;
+        }
+    }
+    return false;
+}
+function hasPhoneNumber(value) {
+    const trimmed = value.trim();
+    if (uuidPattern.test(trimmed))
+        return false;
+    if (/^\d{10,11}$/.test(trimmed))
+        return true;
+    if (/(?:^|[^A-Z0-9])0?5\d{2}[\s.-]?\d{3}[\s.-]?\d{2}[\s.-]?\d{2}(?![A-Z0-9])/i.test(value)) {
+        return true;
+    }
+    if (/\b(?:call|mobile|phone|tel|telephone|telefon|ara)\b[^\d+]{0,20}\d{10,11}\b/i.test(value)) {
+        return true;
+    }
+    if (/(?:^|\D)\(\d{3}\)\s*\d{3}[-\s]\d{4}(?:$|\D)/.test(value)) {
+        return true;
+    }
+    if (/(?:^|\D)\d{3}[-.\s]\d{3}[-.\s]\d{4}(?:$|\D)/.test(value)) {
+        return true;
+    }
+    const internationalMatches = value.matchAll(/(?:^|[^A-Z0-9])\+(\d(?:[\d ().-]{7,}\d))(?![A-Z0-9])/gi);
+    for (const match of internationalMatches) {
+        const digitCount = (match[1] ?? '').replace(/\D/g, '').length;
+        if (digitCount >= 9 && digitCount <= 14)
+            return true;
+    }
+    return false;
+}
+//# sourceMappingURL=privacy-detector.js.map