@reproapp/node-sdk 0.0.45 → 0.0.47

package/src/ingest/client.ts

@@ -140,6 +140,9 @@ const selectSendBatch = (
     if (first.priority === 'normal') {
         return { batch: first, queueItems: 1 };
     }
+    if (first.attempts > 0) {
+        return { batch: first, queueItems: 1 };
+    }
 
     const entries = first.entries.slice();
     let bytes = serializedBytes(entries);
@@ -376,6 +379,16 @@ const drainIngestQueue = async (): Promise<void> => {
                 backoffMs = 0;
                 continue;
             }
+            if (
+                batch.priority === 'high' &&
+                selected.queueItems === 1 &&
+                ingestQueue.length > 1
+            ) {
+                const [failedItem] = ingestQueue.splice(0, 1);
+                if (failedItem) {
+                    ingestQueue.push(failedItem);
+                }
+            }
 
             backoffMs = Math.min(
                 DEFAULT_QUEUE_BACKOFF_MAX_MS,

package/src/privacy.ts

@@ -79,6 +79,25 @@ type NormalizedStatement = {
     descendantRawTextMode?: RuntimePrivacyRawTextMode;
 };
 
+type FunctionStatementTrieNode = {
+    children: Map<string, FunctionStatementTrieNode>;
+    terminal: NormalizedStatement[];
+    subtree: NormalizedStatement[];
+};
+
+type RuntimePrivacySurfaceIndex = {
+    contextualStatements: NormalizedStatement[];
+    fieldStatementsBySuffix: Map<string, NormalizedStatement[]>;
+    fieldWildcardTargetStatements: NormalizedStatement[];
+};
+
+type RuntimePrivacyStatementIndex = {
+    bySurface: Map<RuntimePrivacySurface, RuntimePrivacySurfaceIndex>;
+    functionStatements: FunctionStatementTrieNode;
+    candidateCache: Map<string, NormalizedStatement[]>;
+    hasNumericTargetSegments: boolean;
+};
+
 type TransformFallback = {
     action: RuntimePrivacyAction;
     rawTextMode?: RuntimePrivacyRawTextMode;
@@ -120,13 +139,17 @@ export type NormalizedRuntimePrivacyPolicy = {
     strength?: 'weak' | 'medium' | 'strict';
     statements: NormalizedStatement[];
     rawTextHints: NormalizedRawTextHint[];
+    fingerprint: string;
 };
 
 const EMPTY_RUNTIME_PRIVACY_POLICY: NormalizedRuntimePrivacyPolicy = {
     statements: [],
     rawTextHints: [],
+    fingerprint: 'empty',
 };
 
+const runtimePrivacyStatementIndexes = new WeakMap<object, RuntimePrivacyStatementIndex>();
+const RUNTIME_PRIVACY_CANDIDATE_CACHE_MAX_ENTRIES = 2048;
 const RUNTIME_PRIVACY_PROVENANCE_MAX_ENTRIES = 1000;
 const RUNTIME_PRIVACY_PROVENANCE_MAX_SERIALIZED_LENGTH = 200_000;
 let objectRuntimePrivacyProvenance = new WeakMap<object, RuntimePrivacyProvenance>();
@@ -303,16 +326,16 @@ export function normalizeRuntimePrivacyPolicy(raw: unknown): NormalizedRuntimePr
     const policy = raw as Record<string, any>;
     const flattened = flattenStatements(policy);
     const rawTextHints = normalizeRawTextHints(policy.rawTextHints);
-    const statements: NormalizedStatement[] = flattened
+    const statements = deduplicateNormalizedStatements(flattened
         .map((entry, index) => normalizeStatement(entry, index))
         .filter((entry): entry is NormalizedStatement => entry !== null)
-        .sort((left, right) => left.priority - right.priority);
+        .sort((left, right) => left.priority - right.priority));
 
     if (!statements.length) {
         return null;
     }
 
-    return {
+    const normalizedPolicy: NormalizedRuntimePrivacyPolicy = {
         environment: typeof policy.environment === 'string' ? policy.environment : undefined,
         strength:
             policy.strength === 'weak' || policy.strength === 'medium' || policy.strength === 'strict'
@@ -320,7 +343,15 @@ export function normalizeRuntimePrivacyPolicy(raw: unknown): NormalizedRuntimePr
                 : undefined,
         statements,
         rawTextHints,
+        fingerprint: runtimePrivacyPolicyFingerprint(
+            typeof policy.environment === 'string' ? policy.environment : undefined,
+            typeof policy.strength === 'string' ? policy.strength : undefined,
+            statements,
+            rawTextHints,
+        ),
     };
+    runtimePrivacyStatementIndexes.set(normalizedPolicy, buildRuntimePrivacyStatementIndex(statements));
+    return normalizedPolicy;
 }
 
 export function applyRuntimePrivacyPolicy(
@@ -425,6 +456,365 @@ function normalizeStatement(input: any, index: number): NormalizedStatement | nu
     };
 }
 
+function deduplicateNormalizedStatements(statements: NormalizedStatement[]): NormalizedStatement[] {
+    const seen = new Set<string>();
+    const deduplicated: NormalizedStatement[] = [];
+    for (const statement of statements) {
+        const key = normalizedStatementSemanticKey(statement);
+        if (seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        deduplicated.push(statement);
+    }
+    return deduplicated;
+}
+
+function normalizedStatementSemanticKey(statement: NormalizedStatement): string {
+    return JSON.stringify({
+        scope: statement.scope,
+        selectorTokens: statement.selectorTokens,
+        selectorLeaves: statement.selectorLeaves,
+        action: statement.action,
+        pattern: statement.pattern
+            ? {
+                source: statement.pattern.source,
+                flags: statement.pattern.flags,
+            }
+            : null,
+        targetMatchers: statement.targetMatchers.map((matcher) => ({
+            surfaces: matcher.surfaces,
+            path: matcher.path,
+        })),
+        textPolicy: statement.textPolicy ?? null,
+        rawTextMode: statement.rawTextMode ?? null,
+        rawTextHintNames: statement.rawTextHintNames ?? null,
+        descendantRawTextMode: statement.descendantRawTextMode ?? null,
+    });
+}
+
+function runtimePrivacyPolicyFingerprint(
+    environment: string | undefined,
+    strength: string | undefined,
+    statements: NormalizedStatement[],
+    rawTextHints: NormalizedRawTextHint[],
+): string {
+    const serialized = JSON.stringify({
+        environment: environment ?? null,
+        strength: strength ?? null,
+        statements: statements.map(normalizedStatementSemanticKey),
+        rawTextHints: rawTextHints.map((hint) => ({
+            name: hint.name,
+            action: hint.action,
+            pattern: hint.pattern.source,
+            flags: hint.pattern.flags,
+            tokenLabel: hint.tokenLabel,
+        })),
+    });
+    return createHash('sha256').update(serialized).digest('hex');
+}
+
+function createFunctionStatementTrieNode(): FunctionStatementTrieNode {
+    return {
+        children: new Map<string, FunctionStatementTrieNode>(),
+        terminal: [],
+        subtree: [],
+    };
+}
+
+function appendUniqueStatement(target: NormalizedStatement[], statement: NormalizedStatement): void {
+    if (!target.includes(statement)) {
+        target.push(statement);
+    }
+}
+
+function functionSelectorSegments(value: string): string[] {
+    return normalizeFunctionSelector(value)
+        .split('.')
+        .map((segment) => segment.trim())
+        .filter(Boolean)
+        .reverse();
+}
+
+function addFunctionStatementToTrie(
+    root: FunctionStatementTrieNode,
+    selector: string,
+    statement: NormalizedStatement,
+): void {
+    const segments = functionSelectorSegments(selector);
+    if (!segments.length) {
+        return;
+    }
+
+    let node = root;
+    for (const segment of segments) {
+        let child = node.children.get(segment);
+        if (!child) {
+            child = createFunctionStatementTrieNode();
+            node.children.set(segment, child);
+        }
+        node = child;
+        appendUniqueStatement(node.subtree, statement);
+    }
+    appendUniqueStatement(node.terminal, statement);
+}
+
+function createRuntimePrivacySurfaceIndex(): RuntimePrivacySurfaceIndex {
+    return {
+        contextualStatements: [],
+        fieldStatementsBySuffix: new Map<string, NormalizedStatement[]>(),
+        fieldWildcardTargetStatements: [],
+    };
+}
+
+function appendStatementToMap(
+    target: Map<string, NormalizedStatement[]>,
+    key: string,
+    statement: NormalizedStatement,
+): void {
+    if (!key) {
+        return;
+    }
+    const current = target.get(key);
+    if (current) {
+        appendUniqueStatement(current, statement);
+        return;
+    }
+    target.set(key, [statement]);
+}
+
+function explicitTargetSurfaces(statement: NormalizedStatement): Set<RuntimePrivacySurface> | null {
+    if (!statement.targetMatchers.length) {
+        return null;
+    }
+    return new Set(statement.targetMatchers.flatMap((matcher) => matcher.surfaces));
+}
+
+function scopeCanMatchSurface(
+    statement: NormalizedStatement,
+    surface: RuntimePrivacySurface,
+): boolean {
+    if (statement.scope === 'FUNCTION') {
+        return surface.startsWith('trace.');
+    }
+    if (statement.scope === 'ROUTE') {
+        return surface.startsWith('request.') || surface === 'response.body';
+    }
+    if (statement.scope === 'SCHEMA' || statement.scope === 'QUERY') {
+        return surface.startsWith('db.');
+    }
+    return true;
+}
+
+function addFieldStatementToSurfaceIndex(
+    index: RuntimePrivacySurfaceIndex,
+    statement: NormalizedStatement,
+): void {
+    for (const selector of statement.selectorLeaves) {
+        appendStatementToMap(
+            index.fieldStatementsBySuffix,
+            normalizeComparableToken(selector),
+            statement,
+        );
+    }
+    for (const matcher of statement.targetMatchers) {
+        const terminal = matcher.path?.[matcher.path.length - 1];
+        if (!terminal) {
+            continue;
+        }
+        if (terminal === '*') {
+            appendUniqueStatement(index.fieldWildcardTargetStatements, statement);
+            continue;
+        }
+        appendStatementToMap(
+            index.fieldStatementsBySuffix,
+            normalizeComparableToken(terminal),
+            statement,
+        );
+    }
+}
+
+function buildRuntimePrivacyStatementIndex(
+    statements: NormalizedStatement[],
+): RuntimePrivacyStatementIndex {
+    const bySurface = new Map<RuntimePrivacySurface, RuntimePrivacySurfaceIndex>(
+        ALL_SURFACES.map((surface) => [surface, createRuntimePrivacySurfaceIndex()]),
+    );
+    const functionStatements = createFunctionStatementTrieNode();
+    let hasNumericTargetSegments = false;
+
+    for (const statement of statements) {
+        for (const matcher of statement.targetMatchers) {
+            if (matcher.path?.some((segment) => /^\d+$/.test(segment))) {
+                hasNumericTargetSegments = true;
+            }
+        }
+
+        if (statement.scope === 'FUNCTION') {
+            for (const selector of statement.selectorTokens) {
+                addFunctionStatementToTrie(functionStatements, selector, statement);
+            }
+            continue;
+        }
+
+        const targetSurfaces = explicitTargetSurfaces(statement);
+        for (const surface of ALL_SURFACES) {
+            if (!scopeCanMatchSurface(statement, surface)) {
+                continue;
+            }
+            // FIELD protections deliberately have cross-surface fallback semantics.
+            if (
+                statement.scope !== 'FIELD' &&
+                targetSurfaces &&
+                !targetSurfaces.has(surface)
+            ) {
+                continue;
+            }
+            const surfaceIndex = bySurface.get(surface)!;
+            if (statement.scope === 'FIELD') {
+                addFieldStatementToSurfaceIndex(surfaceIndex, statement);
+            } else {
+                appendUniqueStatement(surfaceIndex.contextualStatements, statement);
+            }
+        }
+    }
+
+    return {
+        bySurface,
+        functionStatements,
+        candidateCache: new Map<string, NormalizedStatement[]>(),
+        hasNumericTargetSegments,
+    };
+}
+
+function getRuntimePrivacyStatementIndex(
+    policy: NormalizedRuntimePrivacyPolicy,
+): RuntimePrivacyStatementIndex {
+    const existing = runtimePrivacyStatementIndexes.get(policy);
+    if (existing) {
+        return existing;
+    }
+    const created = buildRuntimePrivacyStatementIndex(policy.statements);
+    runtimePrivacyStatementIndexes.set(policy, created);
+    return created;
+}
+
+function matchingFunctionStatements(
+    root: FunctionStatementTrieNode,
+    functionName: string,
+): NormalizedStatement[] {
+    const segments = functionSelectorSegments(functionName);
+    if (!segments.length) {
+        return [];
+    }
+
+    const matches = new Set<NormalizedStatement>();
+    let node: FunctionStatementTrieNode | undefined = root;
+    for (let index = 0; index < segments.length; index += 1) {
+        node = node.children.get(segments[index]);
+        if (!node) {
+            break;
+        }
+        for (const statement of node.terminal) {
+            matches.add(statement);
+        }
+        if (index === segments.length - 1) {
+            for (const statement of node.subtree) {
+                matches.add(statement);
+            }
+        }
+    }
+    return Array.from(matches);
+}
+
+function normalizedPathSuffixes(path: string[], preserveNumericSegments: boolean): string[] {
+    const joined = path
+        .filter((segment) =>
+            segment !== '*' &&
+            (preserveNumericSegments || !/^\d+$/.test(segment)),
+        )
+        .map(normalizeComparableToken)
+        .filter(Boolean)
+        .join('.');
+    if (!joined) {
+        return [];
+    }
+    const segments = joined.split('.').filter(Boolean);
+    const suffixes: string[] = [];
+    for (let index = 0; index < segments.length; index += 1) {
+        suffixes.push(segments.slice(index).join('.'));
+    }
+    return suffixes;
+}
+
+function runtimePrivacyCandidateCacheKey(
+    index: RuntimePrivacyStatementIndex,
+    surface: RuntimePrivacySurface,
+    context: RuntimePrivacyApplyContext,
+    path: string[],
+): string {
+    const pathKey = path
+        .map((segment) =>
+            !index.hasNumericTargetSegments && /^\d+$/.test(segment)
+                ? '*'
+                : segment,
+        )
+        .join('.');
+    return [
+        surface,
+        normalizeFunctionSelector(context.trace?.fn?.trim() ?? ''),
+        context.routeKey?.trim() ?? '',
+        String(context.db?.type ?? ''),
+        String(context.db?.collection ?? ''),
+        String(context.db?.op ?? ''),
+        pathKey,
+    ].join('\u0000');
+}
+
+function candidateStatementsForMatch(
+    policy: NormalizedRuntimePrivacyPolicy,
+    surface: RuntimePrivacySurface,
+    context: RuntimePrivacyApplyContext,
+    path: string[],
+): NormalizedStatement[] {
+    const index = getRuntimePrivacyStatementIndex(policy);
+    const cacheKey = runtimePrivacyCandidateCacheKey(index, surface, context, path);
+    const cached = index.candidateCache.get(cacheKey);
+    if (cached) {
+        return cached;
+    }
+
+    const surfaceIndex = index.bySurface.get(surface);
+    const candidateSet = new Set<NormalizedStatement>(
+        surfaceIndex?.contextualStatements ?? [],
+    );
+    if (surfaceIndex && path.length) {
+        for (const suffix of normalizedPathSuffixes(path, index.hasNumericTargetSegments)) {
+            for (const statement of surfaceIndex.fieldStatementsBySuffix.get(suffix) ?? []) {
+                candidateSet.add(statement);
+            }
+        }
+        for (const statement of surfaceIndex.fieldWildcardTargetStatements) {
+            candidateSet.add(statement);
+        }
+    }
+    if (surface.startsWith('trace.')) {
+        const functionName = context.trace?.fn?.trim() ?? '';
+        if (functionName) {
+            for (const statement of matchingFunctionStatements(index.functionStatements, functionName)) {
+                candidateSet.add(statement);
+            }
+        }
+    }
+    const candidates = Array.from(candidateSet)
+        .sort((left, right) => left.priority - right.priority);
+    if (index.candidateCache.size >= RUNTIME_PRIVACY_CANDIDATE_CACHE_MAX_ENTRIES) {
+        index.candidateCache.clear();
+    }
+    index.candidateCache.set(cacheKey, candidates);
+    return candidates;
+}
+
 function resolveEffectiveAction(
     baseAction: RuntimePrivacyAction | null | undefined,
     value: any,
@@ -1299,7 +1689,7 @@ function matchStatement(
     path: string[],
 ): NormalizedStatement | null {
     let candidates: Array<{ statement: NormalizedStatement; targetScore: number; score: number }> = [];
-    for (const statement of policy.statements) {
+    for (const statement of candidateStatementsForMatch(policy, surface, context, path)) {
         if (shouldIgnoreBroadContainerRuleOnDescendant(statement, surface, path)) {
             continue;
         }
@@ -1705,12 +2095,24 @@ function querySelectorMatches(
 
 function patternMatches(pattern: RegExp | undefined, leaf: string, joinedPath: string, value: any): boolean {
     if (!pattern) return false;
-    if (leaf && pattern.test(leaf)) return true;
-    if (joinedPath && pattern.test(joinedPath)) return true;
-    if (typeof value === 'string' && pattern.test(value)) return true;
+    if (leaf && testRuntimePrivacyPattern(pattern, leaf)) return true;
+    if (joinedPath && testRuntimePrivacyPattern(pattern, joinedPath)) return true;
+    if (typeof value === 'string' && testRuntimePrivacyPattern(pattern, value)) return true;
     return false;
 }
 
+function testRuntimePrivacyPattern(pattern: RegExp, value: string): boolean {
+    // Generated policies may contain global/sticky expressions. RegExp.test mutates
+    // lastIndex for those flags, which otherwise makes identical captures alternate
+    // between matched and unmatched.
+    pattern.lastIndex = 0;
+    try {
+        return pattern.test(value);
+    } finally {
+        pattern.lastIndex = 0;
+    }
+}
+
 function functionSelectorMatches(selector: string, fn: string): boolean {
     const normalizedSelector = normalizeFunctionSelector(selector);
     const normalizedFn = normalizeFunctionSelector(fn);

package/test/ingest-high-priority-fairness.test.js

@@ -0,0 +1,76 @@
+const assert = require('node:assert/strict');
+
+const originalFetch = global.fetch;
+const attempts = [];
+
+global.fetch = async (_url, init = {}) => {
+    const body = JSON.parse(String(init.body || '{}'));
+    const requestIds = (body.events || []).map((event) => event?.payload?.requestRid);
+    attempts.push(requestIds);
+    const hasPoisonedItem = requestIds.includes('R_poisoned');
+    return {
+        ok: !hasPoisonedItem,
+        status: hasPoisonedItem ? 503 : 202,
+        json: async () => ({ ok: !hasPoisonedItem }),
+        text: async () => JSON.stringify({ ok: !hasPoisonedItem }),
+    };
+};
+
+const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
+
+function manifest(requestRid) {
+    return {
+        actionId: 'A_fairness',
+        requestRid,
+        requestCaptureManifest: {
+            scopeId: `req:${requestRid}`,
+            requestRid,
+            transport: 'http',
+            expected: {
+                backendRequest: 1,
+                traceBatch: 0,
+                dbChange: 0,
+                emailEvent: 0,
+            },
+            closedAt: Date.now(),
+        },
+        t: Date.now(),
+    };
+}
+
+async function main() {
+    const cfg = {
+        tenantId: 'TENANT_test',
+        appId: 'APP_test',
+        appSecret: 'secret',
+        serviceName: 'ingest-fairness-test',
+        ingestBase: 'http://127.0.0.1:65535',
+    };
+
+    enqueueIngestEntries(cfg, 'S_fairness', [manifest('R_poisoned')], { priority: 'high' });
+    enqueueIngestEntries(cfg, 'S_fairness', [manifest('R_valid')], { priority: 'high' });
+
+    await flushIngestQueue();
+    await flushIngestQueue();
+    await flushIngestQueue();
+
+    assert(
+        attempts.some((requestIds) => requestIds.length === 1 && requestIds[0] === 'R_valid'),
+        `valid completion evidence remained blocked: ${JSON.stringify(attempts)}`,
+    );
+    assert(
+        attempts.filter((requestIds) => requestIds.includes('R_poisoned')).length >= 2,
+        `failed high-priority evidence was dropped instead of retained: ${JSON.stringify(attempts)}`,
+    );
+
+    console.log('ingest high priority fairness OK');
+}
+
+main()
+    .catch((error) => {
+        console.error(error);
+        process.exitCode = 1;
+    })
+    .finally(() => {
+        global.fetch = originalFetch;
+    });