@reproapp/node-sdk 0.0.26 → 0.0.27

package/src/ingest/client.ts

@@ -56,6 +56,25 @@ export const kickIngestQueueDrain = (): void => {
     }
 };
 
+const isCriticalIngestEntry = (entry: LegacyEntry): boolean => (
+    entry.requestCaptureManifest !== undefined ||
+    entry.sessionCaptureManifest !== undefined ||
+    entry.request !== undefined ||
+    entry.db !== undefined ||
+    entry.trace !== undefined ||
+    entry.email !== undefined
+);
+
+const resolveBatchPriority = (
+    entries: LegacyEntry[],
+    options: EnqueueIngestOptions,
+): QueuedIngestBatch['priority'] => {
+    if (options.priority === 'high') {
+        return 'high';
+    }
+    return entries.some(isCriticalIngestEntry) ? 'high' : 'normal';
+};
+
 const resolveIngestUrl = (config: IngestClientConfig): string => {
     const envBase = typeof process !== 'undefined' ? (process as any)?.env?.REPRO_INGEST_BASE : undefined;
     const ingestBase = String(envBase || config.ingestBase || DEFAULT_INGEST_BASE).replace(/\/+$/, '');
@@ -185,7 +204,7 @@ const drainIngestQueue = async (): Promise<void> => {
             }
 
             batch.attempts += 1;
-            if (batch.attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS) {
+            if (batch.attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS && batch.priority !== 'high') {
                 ingestQueue.shift();
                 backoffMs = 0;
                 continue;
@@ -215,7 +234,7 @@ export const enqueueIngestEntries = (
     if (!sessionId || !Array.isArray(entries) || entries.length === 0) {
         return false;
     }
-    const priority = options.priority === 'high' ? 'high' : 'normal';
+    const priority = resolveBatchPriority(entries, options);
     const batch: QueuedIngestBatch = {
         config: { ...config },
         sessionId,
@@ -235,13 +254,16 @@ export const enqueueIngestEntries = (
     }
     if (DEFAULT_QUEUE_MAX_ITEMS > 0 && ingestQueue.length > DEFAULT_QUEUE_MAX_ITEMS) {
         while (ingestQueue.length > DEFAULT_QUEUE_MAX_ITEMS) {
-            let removeIndex = ingestQueue.length - 1;
+            let removeIndex = -1;
             for (let index = ingestQueue.length - 1; index >= 0; index -= 1) {
                 if (ingestQueue[index]?.priority !== 'high') {
                     removeIndex = index;
                     break;
                 }
             }
+            if (removeIndex === -1) {
+                break;
+            }
             ingestQueue.splice(removeIndex, 1);
         }
     }

package/test/ingest-critical-retention.test.js

@@ -0,0 +1,147 @@
+process.env.REPRO_INGEST_QUEUE_MAX_ITEMS = '3';
+
+const assert = require('assert');
+
+const originalFetch = global.fetch;
+const capturedBodies = [];
+
+global.fetch = async (_url, init = {}) => {
+    capturedBodies.push(JSON.parse(String(init.body || '{}')));
+    return {
+        ok: true,
+        status: 200,
+        json: async () => ({ ok: true }),
+        text: async () => '{"ok":true}',
+    };
+};
+
+const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
+
+async function main() {
+    const cfg = {
+        tenantId: 'TENANT_test',
+        appId: 'APP_test',
+        appSecret: 'secret',
+        serviceName: 'critical-retention-test',
+        ingestBase: 'http://127.0.0.1:65535',
+    };
+    const sessionId = 'S_critical_retention';
+
+    for (let index = 0; index < 5; index += 1) {
+        enqueueIngestEntries(cfg, sessionId, [{
+            actionId: 'A_optional',
+            message: `optional-${index}`,
+            t: Date.now(),
+        }]);
+    }
+
+    const criticalEntries = [
+        {
+            actionId: 'A_critical',
+            request: {
+                rid: 'R_request',
+                method: 'GET',
+                url: '/critical-request',
+                status: 200,
+            },
+            t: Date.now(),
+        },
+        {
+            actionId: 'A_critical',
+            requestRid: 'R_trace',
+            trace: [{
+                t: Date.now(),
+                type: 'enter',
+                fn: 'criticalTrace',
+            }],
+            traceBatch: {
+                rid: 'R_trace',
+                index: 0,
+                total: 1,
+            },
+            t: Date.now(),
+        },
+        {
+            actionId: 'A_critical',
+            requestRid: 'R_db',
+            db: [{
+                collection: 'critical_collection',
+                op: 'findOne',
+            }],
+            t: Date.now(),
+        },
+        {
+            actionId: 'A_critical',
+            requestRid: 'R_email',
+            email: {
+                provider: 'critical-email',
+                subject: 'critical',
+            },
+            t: Date.now(),
+        },
+        {
+            actionId: 'A_critical',
+            requestRid: 'R_manifest',
+            requestCaptureManifest: {
+                scopeId: 'req:R_manifest',
+                requestRid: 'R_manifest',
+                transport: 'http',
+                expected: {
+                    backendRequest: 1,
+                    traceBatch: 0,
+                    dbChange: 0,
+                    emailEvent: 0,
+                },
+                closedAt: Date.now(),
+            },
+            t: Date.now(),
+        },
+        {
+            actionId: 'A_critical',
+            sessionCaptureManifest: {
+                mode: 'staging',
+                expected: {
+                    frontendActionDoc: 0,
+                    rrwebChunk: 0,
+                    frontendSessionEnd: 1,
+                },
+                closedAt: Date.now(),
+            },
+            t: Date.now(),
+        },
+    ];
+
+    for (const entry of criticalEntries) {
+        const enqueued = enqueueIngestEntries(cfg, sessionId, [entry]);
+        assert.strictEqual(enqueued, true, `critical event was dropped: ${JSON.stringify(Object.keys(entry))}`);
+    }
+
+    await flushIngestQueue();
+
+    const eventTypes = capturedBodies
+        .flatMap((body) => Array.isArray(body?.events) ? body.events : [])
+        .map((event) => event?.event_type)
+        .sort();
+
+    assert.deepStrictEqual(eventTypes, [
+        'backend_request',
+        'db_change',
+        'email_event',
+        'request_capture_manifest',
+        'session_capture_manifest',
+        'trace_batch',
+    ]);
+
+    // eslint-disable-next-line no-console
+    console.log('ingest critical retention OK');
+}
+
+main()
+    .catch((error) => {
+        // eslint-disable-next-line no-console
+        console.error(error);
+        process.exitCode = 1;
+    })
+    .finally(() => {
+        global.fetch = originalFetch;
+    });

package/test/ingest-critical-retry-retention.test.js

@@ -0,0 +1,57 @@
+const assert = require('assert');
+
+const originalFetch = global.fetch;
+let attempts = 0;
+
+global.fetch = async () => {
+    attempts += 1;
+    return {
+        ok: attempts > 6,
+        status: attempts > 6 ? 200 : 503,
+        json: async () => ({ ok: attempts > 6 }),
+        text: async () => attempts > 6 ? '{"ok":true}' : '{"ok":false}',
+    };
+};
+
+const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
+
+async function main() {
+    const cfg = {
+        tenantId: 'TENANT_test',
+        appId: 'APP_test',
+        appSecret: 'secret',
+        serviceName: 'critical-retry-retention-test',
+        ingestBase: 'http://127.0.0.1:65535',
+    };
+
+    const enqueued = enqueueIngestEntries(cfg, 'S_critical_retry_retention', [{
+        actionId: 'A_critical_retry',
+        request: {
+            rid: 'R_critical_retry',
+            method: 'GET',
+            url: '/critical-retry',
+            status: 200,
+        },
+        t: Date.now(),
+    }]);
+    assert.strictEqual(enqueued, true);
+
+    for (let index = 0; index < 7; index += 1) {
+        await flushIngestQueue();
+    }
+
+    assert.strictEqual(attempts, 7, `critical batch was dropped before eventual success; attempts=${attempts}`);
+
+    // eslint-disable-next-line no-console
+    console.log('ingest critical retry retention OK');
+}
+
+main()
+    .catch((error) => {
+        // eslint-disable-next-line no-console
+        console.error(error);
+        process.exitCode = 1;
+    })
+    .finally(() => {
+        global.fetch = originalFetch;
+    });

package/test/ingest-high-priority-retention.test.js

@@ -0,0 +1,75 @@
+process.env.REPRO_INGEST_QUEUE_MAX_ITEMS = '3';
+
+const assert = require('assert');
+
+const originalFetch = global.fetch;
+const capturedBodies = [];
+
+global.fetch = async (_url, init = {}) => {
+    capturedBodies.push(JSON.parse(String(init.body || '{}')));
+    return {
+        ok: true,
+        status: 200,
+        json: async () => ({ ok: true }),
+        text: async () => '{"ok":true}',
+    };
+};
+
+const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
+
+async function main() {
+    const cfg = {
+        tenantId: 'TENANT_test',
+        appId: 'APP_test',
+        appSecret: 'secret',
+        serviceName: 'high-priority-retention-test',
+        ingestBase: 'http://127.0.0.1:65535',
+    };
+    const sessionId = 'S_high_priority_retention';
+
+    for (let index = 0; index < 5; index += 1) {
+        const enqueued = enqueueIngestEntries(cfg, sessionId, [{
+            actionId: 'A_high_priority_retention',
+            requestRid: `R_high_priority_${index}`,
+            requestCaptureManifest: {
+                scopeId: `req:R_high_priority_${index}`,
+                requestRid: `R_high_priority_${index}`,
+                transport: 'http',
+                expected: {
+                    backendRequest: 1,
+                    traceBatch: 0,
+                    dbChange: 0,
+                    emailEvent: 0,
+                },
+                closedAt: Date.now(),
+            },
+            t: Date.now(),
+        }], { priority: 'high' });
+        assert.strictEqual(enqueued, true, `high-priority batch ${index} was dropped`);
+    }
+
+    await flushIngestQueue();
+
+    const manifests = capturedBodies
+        .flatMap((body) => Array.isArray(body?.events) ? body.events : [])
+        .filter((event) => event?.event_type === 'request_capture_manifest');
+
+    assert.strictEqual(manifests.length, 5, JSON.stringify({
+        got: manifests.length,
+        expected: 5,
+        requestIds: manifests.map((event) => event?.request_id),
+    }));
+
+    // eslint-disable-next-line no-console
+    console.log('ingest high priority retention OK');
+}
+
+main()
+    .catch((error) => {
+        // eslint-disable-next-line no-console
+        console.error(error);
+        process.exitCode = 1;
+    })
+    .finally(() => {
+        global.fetch = originalFetch;
+    });

package/test/mongodb-integration.test.js

@@ -0,0 +1,332 @@
+process.env.REPRO_INGEST_REQUEST_TIMEOUT_MS = '50';
+process.env.REPRO_SDK_BACKGROUND_MAX_DEFER_MS = '25';
+process.env.REPRO_SDK_BACKGROUND_IDLE_DELAY_MS = '0';
+process.env.REPRO_SDK_BACKGROUND_BUSY_IDLE_DELAY_MS = '0';
+process.env.TRACE_IDLE_FLUSH_MS = '10';
+process.env.TRACE_LINGER_AFTER_FINISH_MS = '0';
+
+const assert = require('assert');
+const { EventEmitter } = require('events');
+const { flushIngestQueue } = require('../dist/ingest/client');
+const { reproMiddleware, __reproTestHooks } = require('../dist');
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+async function waitFor(predicate, timeoutMs = 2000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (predicate()) return;
+        await sleep(20);
+    }
+    assert(predicate(), 'timed out waiting for expected ingest payload');
+}
+
+function flattenEvents(posts) {
+    return posts.flatMap((body) => Array.isArray(body?.events) ? body.events : []);
+}
+
+async function drainSdk() {
+    await __reproTestHooks.drainSdkBackgroundQueueForTest();
+    await flushIngestQueue();
+}
+
+async function withCapturedFetch(fn) {
+    const capturedBodies = [];
+    const originalFetch = global.fetch;
+    global.fetch = async (_url, init) => {
+        capturedBodies.push(JSON.parse(String(init?.body ?? '{}')));
+        return {
+            ok: true,
+            status: 202,
+            json: async () => ({ ok: true }),
+            text: async () => '{"ok":true}',
+        };
+    };
+
+    try {
+        await fn(capturedBodies);
+    } finally {
+        await drainSdk();
+        global.fetch = originalFetch;
+    }
+}
+
+function makeConfig(sessionId, actionId) {
+    return {
+        tenantId: 'TENANT_test',
+        appId: 'APP_test',
+        appSecret: 'secret',
+        captureHeaders: false,
+        privacy: { environment: 'dev' },
+        ingestBase: 'http://127.0.0.1:65535',
+        resolveContext: () => ({ sid: sessionId, aid: actionId }),
+    };
+}
+
+function makeReqRes(sessionId, actionId) {
+    const req = new EventEmitter();
+    req.method = 'GET';
+    req.url = '/native-mongodb';
+    req.originalUrl = '/native-mongodb';
+    req.headers = {
+        'x-bug-session-id': sessionId,
+        'x-bug-action-id': actionId,
+    };
+    req.query = {};
+    req.params = {};
+    req.body = undefined;
+
+    const res = new EventEmitter();
+    res.statusCode = 200;
+    res.getHeader = () => undefined;
+    res.getHeaders = () => ({});
+    res.json = function json(body) {
+        this.emit('finish');
+        return body;
+    };
+    res.send = function send(body) {
+        this.emit('finish');
+        return body;
+    };
+    res.write = function write() {
+        return true;
+    };
+    res.end = function end() {
+        this.emit('finish');
+        return this;
+    };
+
+    return { req, res };
+}
+
+async function runCapturedRequest(cfg, sessionId, actionId, fn) {
+    const middleware = reproMiddleware(cfg);
+    const { req, res } = makeReqRes(sessionId, actionId);
+    await new Promise((resolve, reject) => {
+        middleware(req, res, (err) => {
+            if (err) {
+                reject(err);
+                return;
+            }
+            Promise.resolve()
+                .then(fn)
+                .then(resolve, reject);
+        });
+    });
+    res.end();
+}
+
+function dbChanges(events) {
+    return events
+        .filter((event) => event.event_type === 'db_change')
+        .flatMap((event) => Array.isArray(event.payload?.db) ? event.payload.db : []);
+}
+
+async function testNativeInsertOneIsCaptured() {
+    await withCapturedFetch(async (capturedBodies) => {
+        __reproTestHooks.resetSdkBackgroundQueuesForTest();
+        const sessionId = 'S_mongo_insert';
+        const actionId = 'A_mongo_insert';
+        const cfg = makeConfig(sessionId, actionId);
+
+        class FakeCollection {
+            constructor() {
+                this.collectionName = 'subjects';
+                this.dbName = 'clinical';
+            }
+
+            async insertOne(doc, options) {
+                assert.strictEqual(options.session.id, 'native-session-1');
+                doc._id = doc._id ?? 'subject-1';
+                return { acknowledged: true, insertedId: doc._id };
+            }
+        }
+
+        __reproTestHooks.patchMongoCollectionPrototypeForTest(FakeCollection.prototype, cfg);
+        const collection = new FakeCollection();
+
+        await runCapturedRequest(cfg, sessionId, actionId, async () => {
+            const result = await collection.insertOne(
+                { subjectNumber: '10013038', status: 'screened' },
+                {
+                    session: {
+                        id: 'native-session-1',
+                        hasEnded: false,
+                        inTransaction: () => true,
+                    },
+                },
+            );
+            assert.deepStrictEqual(result, { acknowledged: true, insertedId: 'subject-1' });
+        });
+
+        await waitFor(() => dbChanges(flattenEvents(capturedBodies)).length === 1);
+        await drainSdk();
+
+        const [change] = dbChanges(flattenEvents(capturedBodies));
+        assert.strictEqual(change.type, 'mongo');
+        assert.strictEqual(change.collection, 'subjects');
+        assert.strictEqual(change.op, 'insertOne');
+        assert.strictEqual(change.pk._id, 'subject-1');
+        assert.strictEqual(change.query.driver, 'mongodb');
+        assert.strictEqual(change.query.db, 'clinical');
+        assert.strictEqual(change.query.doc.subjectNumber, '10013038');
+        assert.strictEqual(change.query.options.session.__kind, 'mongodb_client_session');
+        assert.strictEqual(change.query.options.session.id, 'native-session-1');
+        assert.strictEqual(change.query.options.session.inTransaction, true);
+        assert.strictEqual(change.after.subjectNumber, '10013038');
+        assert.strictEqual(change.resultMeta.insertedId, 'subject-1');
+        assert.strictEqual(change.resultMeta.acknowledged, true);
+    });
+}
+
+async function testNativeUpdateOneUpsertIsCaptured() {
+    await withCapturedFetch(async (capturedBodies) => {
+        __reproTestHooks.resetSdkBackgroundQueuesForTest();
+        const sessionId = 'S_mongo_update';
+        const actionId = 'A_mongo_update';
+        const cfg = makeConfig(sessionId, actionId);
+
+        class FakeCollection {
+            constructor() {
+                this.collectionName = 'subjects';
+                this.dbName = 'clinical';
+            }
+
+            async updateOne() {
+                return {
+                    acknowledged: true,
+                    matchedCount: 0,
+                    modifiedCount: 0,
+                    upsertedId: 'subject-2',
+                };
+            }
+        }
+
+        __reproTestHooks.patchMongoCollectionPrototypeForTest(FakeCollection.prototype, cfg);
+        const collection = new FakeCollection();
+
+        await runCapturedRequest(cfg, sessionId, actionId, async () => {
+            await collection.updateOne(
+                { _id: 'subject-2' },
+                { $set: { status: 'randomized' } },
+                { upsert: true },
+            );
+        });
+
+        await waitFor(() => dbChanges(flattenEvents(capturedBodies)).length === 1);
+        await drainSdk();
+
+        const [change] = dbChanges(flattenEvents(capturedBodies));
+        assert.strictEqual(change.collection, 'subjects');
+        assert.strictEqual(change.op, 'updateOne');
+        assert.strictEqual(change.pk._id, 'subject-2');
+        assert.strictEqual(change.query.filter._id, 'subject-2');
+        assert.strictEqual(change.query.update.$set.status, 'randomized');
+        assert.strictEqual(change.query.options.upsert, true);
+        assert.strictEqual(change.resultMeta.matched, 0);
+        assert.strictEqual(change.resultMeta.modified, 0);
+        assert.strictEqual(change.resultMeta.upsertedId, 'subject-2');
+    });
+}
+
+async function testNativeFindCursorToArrayIsCaptured() {
+    await withCapturedFetch(async (capturedBodies) => {
+        __reproTestHooks.resetSdkBackgroundQueuesForTest();
+        const sessionId = 'S_mongo_find';
+        const actionId = 'A_mongo_find';
+        const cfg = makeConfig(sessionId, actionId);
+
+        class FakeCollection {
+            constructor() {
+                this.collectionName = 'subjects';
+                this.dbName = 'clinical';
+            }
+
+            find() {
+                return {
+                    async toArray() {
+                        return [
+                            { _id: 'subject-3', status: 'screened' },
+                            { _id: 'subject-4', status: 'randomized' },
+                        ];
+                    },
+                };
+            }
+        }
+
+        __reproTestHooks.patchMongoCollectionPrototypeForTest(FakeCollection.prototype, cfg);
+        const collection = new FakeCollection();
+
+        await runCapturedRequest(cfg, sessionId, actionId, async () => {
+            const docs = await collection.find({ siteId: 'site-1001' }, { projection: { status: 1 } }).toArray();
+            assert.strictEqual(docs.length, 2);
+        });
+
+        await waitFor(() => dbChanges(flattenEvents(capturedBodies)).length === 1);
+        await drainSdk();
+
+        const [change] = dbChanges(flattenEvents(capturedBodies));
+        assert.strictEqual(change.collection, 'subjects');
+        assert.strictEqual(change.op, 'find');
+        assert.strictEqual(change.query.filter.siteId, 'site-1001');
+        assert.deepStrictEqual(change.query.options.projection, { status: 1 });
+        assert.strictEqual(change.query.cursorMethod, 'toArray');
+        assert.strictEqual(change.resultMeta.docsCount, 2);
+        assert.strictEqual(change.resultMeta.result[0]._id, 'subject-3');
+    });
+}
+
+async function testNativeOperationFailureIsCapturedAndRethrown() {
+    await withCapturedFetch(async (capturedBodies) => {
+        __reproTestHooks.resetSdkBackgroundQueuesForTest();
+        const sessionId = 'S_mongo_error';
+        const actionId = 'A_mongo_error';
+        const cfg = makeConfig(sessionId, actionId);
+
+        class FakeCollection {
+            constructor() {
+                this.collectionName = 'subjects';
+                this.dbName = 'clinical';
+            }
+
+            async deleteOne() {
+                const err = new Error('delete failed');
+                err.code = 12345;
+                err.codeName = 'DeleteFailed';
+                throw err;
+            }
+        }
+
+        __reproTestHooks.patchMongoCollectionPrototypeForTest(FakeCollection.prototype, cfg);
+        const collection = new FakeCollection();
+
+        await assert.rejects(
+            () => runCapturedRequest(cfg, sessionId, actionId, async () => {
+                await collection.deleteOne({ _id: 'subject-5' });
+            }),
+            /delete failed/,
+        );
+
+        await waitFor(() => dbChanges(flattenEvents(capturedBodies)).length === 1);
+        await drainSdk();
+
+        const [change] = dbChanges(flattenEvents(capturedBodies));
+        assert.strictEqual(change.collection, 'subjects');
+        assert.strictEqual(change.op, 'deleteOne');
+        assert.strictEqual(change.query.filter._id, 'subject-5');
+        assert.strictEqual(change.error.message, 'delete failed');
+        assert.strictEqual(change.error.code, 12345);
+        assert.strictEqual(change.error.codeName, 'DeleteFailed');
+    });
+}
+
+(async () => {
+    await testNativeInsertOneIsCaptured();
+    await testNativeUpdateOneUpsertIsCaptured();
+    await testNativeFindCursorToArrayIsCaptured();
+    await testNativeOperationFailureIsCapturedAndRethrown();
+    console.log('mongodb integration tests passed');
+})().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});