@proofhound/core 0.1.13 → 0.1.15

package/dist/worker/runners/dataset-raw-import-runner.js

@@ -0,0 +1,238 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDatasetRawImportRunner = createDatasetRawImportRunner;
+const node_buffer_1 = require("node:buffer");
+const node_crypto_1 = require("node:crypto");
+const dataset_field_schema_util_1 = require("../../server/modules/dataset/dataset-field-schema.util");
+const dataset_import_raw_parser_1 = require("../../server/modules/dataset/dataset-import-raw-parser");
+const dataset_import_repository_1 = require("../../server/modules/dataset/dataset-import.repository");
+const usage_metering_hook_1 = require("../../server/common/contracts/usage-metering.hook");
+const TYPE_INFERENCE_SAMPLE_LIMIT = 500;
+const IMAGE_ROLES = new Set(['image', 'image_url', 'image_base64']);
+const RAW_IMPORT_BATCH_MAX_ROWS = 1000;
+const RAW_IMPORT_BATCH_MAX_BYTES = 8 * 1024 * 1024;
+const RAW_IMPORT_MAX_SAMPLE_BYTES = 8 * 1024 * 1024;
+const RAW_IMPORT_MAX_BUFFERED_BYTES = 64 * 1024 * 1024;
+class DatasetImportAbortedError extends Error {
+    constructor() {
+        super('dataset_import_aborted');
+        this.name = 'DatasetImportAbortedError';
+    }
+}
+function createDatasetRawImportRunner(deps) {
+    return async function runDatasetRawImportJob(input, jobContext) {
+        const session = await deps.repo.findImportById(input.projectId, input.importId);
+        if (!session)
+            return { importId: input.importId, datasetId: null, sampleCount: 0, status: 'missing' };
+        if (session.status === 'completed' || session.status === 'failed' || session.status === 'aborted') {
+            return {
+                importId: session.id,
+                datasetId: session.datasetId,
+                sampleCount: session.receivedRows,
+                status: session.status,
+            };
+        }
+        if (session.importMode !== 'raw_object') {
+            await deps.repo.markFailed(input.projectId, input.importId, 'dataset_import_mode_invalid', 'raw import job requires raw_object mode');
+            return { importId: input.importId, datasetId: null, sampleCount: 0, status: 'failed' };
+        }
+        if (!session.rawObjectRef) {
+            await deps.repo.markFailed(input.projectId, input.importId, 'dataset_raw_object_missing', 'uploaded raw object is missing');
+            return { importId: input.importId, datasetId: null, sampleCount: 0, status: 'failed' };
+        }
+        try {
+            const parsing = (await deps.repo.markParsing(input.projectId, input.importId)) ?? session;
+            await ingestRawObjectIntoStaging(deps, input, parsing, session.rawObjectRef);
+            const result = await promoteStagedImport(deps, input, parsing);
+            await cleanupRawObjectRef(deps, session.rawObjectRef, input.importId);
+            await recordJobEvent(deps, input, jobContext, 'job.completed', {
+                status: 'completed',
+                datasetId: result.datasetId,
+                sampleCount: result.sampleCount,
+            });
+            deps.logger.info({ importId: input.importId, datasetId: result.datasetId, sampleCount: result.sampleCount }, 'dataset_raw_import_completed');
+            return {
+                importId: input.importId,
+                datasetId: result.datasetId,
+                sampleCount: result.sampleCount,
+                status: 'completed',
+            };
+        }
+        catch (error) {
+            const latest = await deps.repo.findImportById(input.projectId, input.importId);
+            if (latest?.status === 'aborted' || error instanceof DatasetImportAbortedError) {
+                await cleanupRawObjectRef(deps, session.rawObjectRef, input.importId);
+                return { importId: input.importId, datasetId: null, sampleCount: latest?.receivedRows ?? 0, status: 'aborted' };
+            }
+            const errorCode = errorToCode(error);
+            await deps.repo.markFailed(input.projectId, input.importId, errorCode, error instanceof Error ? error.message : String(error));
+            await cleanupRawObjectRef(deps, session.rawObjectRef, input.importId);
+            await recordJobEvent(deps, input, jobContext, 'job.failed', {
+                status: 'failed',
+                errorKind: errorCode,
+            });
+            deps.logger.warn({ importId: input.importId, error: error instanceof Error ? error.message : String(error) }, 'dataset_raw_import_failed');
+            return { importId: input.importId, datasetId: null, sampleCount: 0, status: 'failed' };
+        }
+    };
+}
+async function ingestRawObjectIntoStaging(deps, input, session, rawObjectRef) {
+    const stream = await deps.storage.getObjectStream(rawObjectRef);
+    const fieldMappings = toFieldMappings(session);
+    const externalIdField = externalIdFieldName(fieldMappings);
+    const project = { projectId: input.projectId, source: 'local' };
+    const actor = toJobActor(input);
+    let batch = [];
+    let batchBytes = jsonArrayBytesForEmptyBatch();
+    let rowIndex = 0;
+    const flush = async () => {
+        if (batch.length === 0)
+            return;
+        await deps.quotaPolicy.assertCanStore({
+            actor,
+            bytes: batchBytes,
+            project,
+            source: 'dataset_raw_import_batch',
+        });
+        await deps.repo.appendBatch(session.id, batch, rowIndex);
+        await assertNotAborted(deps.repo, input.projectId, input.importId);
+        batch = [];
+        batchBytes = jsonArrayBytesForEmptyBatch();
+    };
+    for await (const rawRow of (0, dataset_import_raw_parser_1.parseRawDatasetRows)(stream, session.sourceFormat, {
+        maxBufferedBytes: RAW_IMPORT_MAX_BUFFERED_BYTES,
+    })) {
+        if (batch.length >= RAW_IMPORT_BATCH_MAX_ROWS)
+            await flush();
+        const data = projectSample(rawRow, fieldMappings);
+        const sampleBytes = utf8JsonBytes(data);
+        if (sampleBytes > RAW_IMPORT_MAX_SAMPLE_BYTES)
+            throw new Error('dataset_import_sample_too_large');
+        let candidateBytes = jsonArrayBytesAfterAppend(batchBytes, batch.length, sampleBytes);
+        if (candidateBytes > RAW_IMPORT_BATCH_MAX_BYTES) {
+            if (batch.length === 0)
+                throw new Error('dataset_import_sample_too_large');
+            await flush();
+            candidateBytes = jsonArrayBytesAfterAppend(batchBytes, batch.length, sampleBytes);
+            if (candidateBytes > RAW_IMPORT_BATCH_MAX_BYTES)
+                throw new Error('dataset_import_sample_too_large');
+        }
+        batch.push({ rowIndex, data, externalId: getExternalId(data, externalIdField) });
+        batchBytes = candidateBytes;
+        rowIndex += 1;
+    }
+    await flush();
+}
+async function promoteStagedImport(deps, input, session) {
+    const sampleRows = await deps.repo.getSampleDataForInference(session.id, TYPE_INFERENCE_SAMPLE_LIMIT);
+    const fieldSchema = (0, dataset_field_schema_util_1.buildDatasetFieldSchema)(toFieldMappings(session), sampleRows);
+    const hasImages = fieldSchema.some((field) => IMAGE_ROLES.has(field.role));
+    const datasetId = (0, node_crypto_1.randomUUID)();
+    const { sampleCount } = await deps.repo.promote({
+        importId: session.id,
+        projectId: input.projectId,
+        actorUserId: input.actorId ?? session.createdBy,
+        datasetId,
+        name: session.name,
+        description: session.description,
+        fieldSchema,
+        hasImages,
+    });
+    await recordDatasetImportCompleted(deps, input.projectId, input.actorId ?? session.createdBy, session.id, datasetId, sampleCount);
+    return { datasetId, sampleCount };
+}
+async function assertNotAborted(repo, projectId, importId) {
+    const latest = await repo.findImportById(projectId, importId);
+    if (latest?.status === 'aborted')
+        throw new DatasetImportAbortedError();
+}
+function toFieldMappings(session) {
+    return Array.isArray(session.fieldMappings) ? session.fieldMappings : [];
+}
+function externalIdFieldName(fieldMappings) {
+    return fieldMappings.find((field) => field.role === 'id')?.name ?? null;
+}
+function getExternalId(sample, fieldName) {
+    if (!fieldName)
+        return null;
+    const value = sample[fieldName];
+    if (value === undefined || value === null)
+        return null;
+    return String(value);
+}
+function projectSample(sample, fieldMappings) {
+    return Object.fromEntries(fieldMappings.map((field) => [
+        field.name,
+        Object.prototype.hasOwnProperty.call(sample, field.name) ? sample[field.name] : null,
+    ]));
+}
+async function cleanupRawObjectRef(deps, ref, importId) {
+    await deps.storage.deleteObjects([ref]).catch((error) => {
+        deps.logger.warn({ importId, key: ref.key, error: error.message }, 'dataset_raw_object_delete_failed');
+    });
+}
+async function recordDatasetImportCompleted(deps, projectId, actorId, importId, datasetId, sampleCount) {
+    const occurredAt = new Date();
+    for (const eventType of ['dataset_import.completed', 'storage.dirty']) {
+        await (0, usage_metering_hook_1.safeRecordUsageEvent)(deps.usageMetering, {
+            idempotencyKey: `storage:${eventType}:${datasetId}:${importId}`,
+            dimension: 'storage',
+            eventType,
+            projectId,
+            actorId,
+            occurredAt,
+            source: 'worker',
+            payload: {
+                importId,
+                datasetId,
+                sampleCount,
+                reason: eventType === 'storage.dirty' ? 'dataset_import.completed' : undefined,
+            },
+        }, deps.logger);
+    }
+}
+async function recordJobEvent(deps, input, jobContext, eventType, payload) {
+    await (0, usage_metering_hook_1.safeRecordUsageEvent)(deps.usageMetering, {
+        idempotencyKey: `job:${jobContext.bullmqQueue}:${jobContext.bullmqJobId}:${jobContext.attempt}:${eventType}`,
+        dimension: 'job',
+        eventType,
+        projectId: input.projectId,
+        actorId: input.actorId ?? null,
+        occurredAt: new Date(),
+        source: 'worker',
+        payload: {
+            queue: jobContext.bullmqQueue,
+            jobId: jobContext.bullmqJobId,
+            attempt: jobContext.attempt,
+            importId: input.importId,
+            source: 'dataset_import',
+            ...payload,
+        },
+    }, deps.logger);
+}
+function toJobActor(input) {
+    return {
+        actorKind: 'local_user',
+        actorId: input.actorId ?? 'dataset_import_worker',
+        projectId: input.projectId,
+    };
+}
+function errorToCode(error) {
+    if (error instanceof dataset_import_repository_1.DatasetImportEmptyError)
+        return 'dataset_import_empty';
+    if (error instanceof dataset_import_repository_1.DatasetNameTakenError)
+        return 'dataset_name_taken';
+    if (error instanceof Error && error.message)
+        return error.message.slice(0, 120).replace(/[^a-z0-9_.:-]+/giu, '_');
+    return 'dataset_import_failed';
+}
+function utf8JsonBytes(value) {
+    return node_buffer_1.Buffer.byteLength(JSON.stringify(value ?? null), 'utf8');
+}
+function jsonArrayBytesForEmptyBatch() {
+    return 2; // []
+}
+function jsonArrayBytesAfterAppend(currentArrayBytes, currentLength, nextItemBytes) {
+    return currentLength === 0 ? jsonArrayBytesForEmptyBatch() + nextItemBytes : currentArrayBytes + 1 + nextItemBytes;
+}
+//# sourceMappingURL=dataset-raw-import-runner.js.map

package/dist/worker/runners/dataset-raw-import-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-raw-import-runner.js","sourceRoot":"","sources":["../../../src/worker/runners/dataset-raw-import-runner.ts"],"names":[],"mappings":";;AA+CA,oEAiFC;AAhID,6CAAqC;AACrC,6CAAyC;AAIzC,sGAAiG;AACjG,sGAA6F;AAC7F,sGAMgE;AAGhE,2FAAiH;AAGjH,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AACpE,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACnD,MAAM,2BAA2B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACpD,MAAM,6BAA6B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAgBvD,MAAM,yBAA0B,SAAQ,KAAK;IAC3C;QACE,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED,SAAgB,4BAA4B,CAAC,IAAwC;IACnF,OAAO,KAAK,UAAU,sBAAsB,CAC1C,KAAiC,EACjC,UAAsC;QAEtC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAChF,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACtG,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAClG,OAAO;gBACL,QAAQ,EAAE,OAAO,CAAC,EAAE;gBACpB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,WAAW,EAAE,OAAO,CAAC,YAAY;gBACjC,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CACxB,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,QAAQ,EACd,6BAA6B,EAC7B,yCAAyC,CAC1C,CAAC;YACF,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CACxB,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,QAAQ,EACd,4BAA4B,EAC5B,gCAAgC,CACjC,CAAC;YACF,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACzF,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,OAAO,CAAC;YAC1F,MAAM,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;YAC7E,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC/D,MAAM,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtE,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE;gBAC7D,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,EAC1F,8BAA8B,CAC/B,CAAC;YACF,OAAO;gBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,MAAM,EAAE,WAAW;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC/E,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,IAAI,KAAK,YAAY,yBAAyB,EAAE,CAAC;gBAC/E,MAAM,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACtE,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;YAClH,CAAC;YAED,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CACxB,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,QAAQ,EACd,SAAS,EACT,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;YACF,MAAM,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtE,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE;gBAC1D,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC3F,2BAA2B,CAC5B,CAAC;YACF,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACzF,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,IAAwC,EACxC,KAAiC,EACjC,OAAyB,EACzB,YAA6B;IAE7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;IAChE,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAgB,EAAE,CAAC;IACzE,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,KAAK,GAAqB,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,2BAA2B,EAAE,CAAC;IAC/C,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;QACvB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC/B,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YACpC,KAAK;YACL,KAAK,EAAE,UAAU;YACjB,OAAO;YACP,MAAM,EAAE,0BAA0B;SACnC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnE,KAAK,GAAG,EAAE,CAAC;QACX,UAAU,GAAG,2BAA2B,EAAE,CAAC;IAC7C,CAAC,CAAC;IAEF,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAA,+CAAmB,EAAC,MAAM,EAAE,OAAO,CAAC,YAAyC,EAAE;QACxG,gBAAgB,EAAE,6BAA6B;KAChD,CAAC,EAAE,CAAC;QACH,IAAI,KAAK,CAAC,MAAM,IAAI,yBAAyB;YAAE,MAAM,KAAK,EAAE,CAAC;QAE7D,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAClD,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,WAAW,GAAG,2BAA2B;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAElG,IAAI,cAAc,GAAG,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACtF,IAAI,cAAc,GAAG,0BAA0B,EAAE,CAAC;YAChD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAC3E,MAAM,KAAK,EAAE,CAAC;YACd,cAAc,GAAG,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAClF,IAAI,cAAc,GAAG,0BAA0B;gBAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACtG,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,CAAC,CAAC;QACjF,UAAU,GAAG,cAAc,CAAC;QAC5B,QAAQ,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,EAAE,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,IAAwC,EACxC,KAAiC,EACjC,OAAyB;IAEzB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,EAAE,EAAE,2BAA2B,CAAC,CAAC;IACtG,MAAM,WAAW,GAAG,IAAA,mDAAuB,EAAC,eAAe,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,CAAC;IAClF,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,IAAA,wBAAU,GAAE,CAAC;IAC/B,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAC9C,QAAQ,EAAE,OAAO,CAAC,EAAE;QACpB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,WAAW,EAAE,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS;QAC/C,SAAS;QACT,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,WAAW;QACX,SAAS;KACV,CAAC,CAAC;IACH,MAAM,4BAA4B,CAChC,IAAI,EACJ,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAClC,OAAO,CAAC,EAAE,EACV,SAAS,EACT,WAAW,CACZ,CAAC;IACF,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAA6B,EAAE,SAAiB,EAAE,QAAgB;IAChG,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC9D,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS;QAAE,MAAM,IAAI,yBAAyB,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IAChD,OAAO,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAE,OAAO,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;AACzG,CAAC;AAED,SAAS,mBAAmB,CAAC,aAAuC;IAClE,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;AAC1E,CAAC;AAED,SAAS,aAAa,CAAC,MAA+B,EAAE,SAAwB;IAC9E,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,aAAa,CACpB,MAA+B,EAC/B,aAAuC;IAEvC,OAAO,MAAM,CAAC,WAAW,CACvB,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI;QACV,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KACrF,CAAC,CACH,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,IAAwC,EACxC,GAAoB,EACpB,QAAgB;IAEhB,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,kCAAkC,CAAC,CAAC;IACpH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,IAAwC,EACxC,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAU,EAAE,CAAC;QAC/E,MAAM,IAAA,0CAAoB,EACxB,IAAI,CAAC,aAAa,EAClB;YACE,cAAc,EAAE,WAAW,SAAS,IAAI,SAAS,IAAI,QAAQ,EAAE;YAC/D,SAAS,EAAE,SAAS;YACpB,SAAS;YACT,SAAS;YACT,OAAO;YACP,UAAU;YACV,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE;gBACP,QAAQ;gBACR,SAAS;gBACT,WAAW;gBACX,MAAM,EAAE,SAAS,KAAK,eAAe,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,SAAS;aAC/E;SACF,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,IAAwC,EACxC,KAAiC,EACjC,UAAsC,EACtC,SAAiB,EACjB,OAAgC;IAEhC,MAAM,IAAA,0CAAoB,EACxB,IAAI,CAAC,aAAa,EAClB;QACE,cAAc,EAAE,OAAO,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,OAAO,IAAI,SAAS,EAAE;QAC5G,SAAS,EAAE,KAAK;QAChB,SAAS;QACT,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;QAC9B,UAAU,EAAE,IAAI,IAAI,EAAE;QACtB,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE;YACP,KAAK,EAAE,UAAU,CAAC,WAAW;YAC7B,KAAK,EAAE,UAAU,CAAC,WAAW;YAC7B,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,gBAAgB;YACxB,GAAG,OAAO;SACX;KACF,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAiC;IACnD,OAAO;QACL,SAAS,EAAE,YAAY;QACvB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,uBAAuB;QACjD,SAAS,EAAE,KAAK,CAAC,SAAS;KAC3B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,YAAY,mDAAuB;QAAE,OAAO,sBAAsB,CAAC;IAC5E,IAAI,KAAK,YAAY,iDAAqB;QAAE,OAAO,oBAAoB,CAAC;IACxE,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IAClH,OAAO,uBAAuB,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,oBAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,2BAA2B;IAClC,OAAO,CAAC,CAAC,CAAC,KAAK;AACjB,CAAC;AAED,SAAS,yBAAyB,CAAC,iBAAyB,EAAE,aAAqB,EAAE,aAAqB;IACxG,OAAO,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,2BAA2B,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,GAAG,aAAa,CAAC;AACrH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proofhound/core",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "ProofHound reusable backend runtime: server / webhook / worker NestJS modules, contracts, local defaults, Services, Repositories",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -29,6 +29,7 @@
     "jsonrepair": "^3.14.0",
     "kafkajs": "^2.2.4",
     "nestjs-pino": "^4.6.1",
+    "papaparse": "^5.5.4",
     "pino": "^10.3.1",
     "pino-http": "^11.0.0",
     "postgres": "^3.4.9",
@@ -36,23 +37,24 @@
     "rxjs": "^7.8.2",
     "sharp": "^0.34.5",
     "zod": "^4.4.3",
-    "@proofhound/connector-client": "0.1.13",
-    "@proofhound/crypto": "0.1.13",
-    "@proofhound/judgment": "0.1.13",
-    "@proofhound/db": "0.1.13",
-    "@proofhound/limiter": "0.1.13",
-    "@proofhound/llm-client": "0.1.13",
-    "@proofhound/logger": "0.1.13",
-    "@proofhound/metrics": "0.1.13",
-    "@proofhound/shared": "0.1.13",
-    "@proofhound/optimization-strategy": "0.1.13",
-    "@proofhound/orchestration-shared": "0.1.13"
+    "@proofhound/connector-client": "0.1.15",
+    "@proofhound/crypto": "0.1.15",
+    "@proofhound/judgment": "0.1.15",
+    "@proofhound/limiter": "0.1.15",
+    "@proofhound/llm-client": "0.1.15",
+    "@proofhound/db": "0.1.15",
+    "@proofhound/logger": "0.1.15",
+    "@proofhound/optimization-strategy": "0.1.15",
+    "@proofhound/orchestration-shared": "0.1.15",
+    "@proofhound/metrics": "0.1.15",
+    "@proofhound/shared": "0.1.15"
   },
   "devDependencies": {
     "@nestjs/testing": "^11.1.21",
     "@swc/core": "^1.13.20",
     "@types/express": "^5.0.6",
     "@types/node": "^24.12.4",
+    "@types/papaparse": "^5.5.2",
     "@types/supertest": "^7.2.0",
     "@vitest/coverage-v8": "^4.1.6",
     "supertest": "^7.2.2",