@proofhound/core 0.1.12 → 0.1.14

package/dist/server/modules/dataset/dataset-import.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dataset-import.service.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-import.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EACV,gCAAgC,EAChC,sBAAsB,EAEtB,qBAAqB,EACrB,6BAA6B,EAC7B,oBAAoB,EAGrB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,+CAA+C,CAAC;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,0CAA0C,CAAC;AAC3E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AAEzF,OAAO,EAEL,uBAAuB,EAIxB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAQnD,qBACa,oBAAqB,YAAW,YAAY,EAAE,eAAe;IAMtE,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAR9B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiE;IACxF,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,QAAQ,CAAS;gBAGN,IAAI,EAAE,uBAAuB,EAC7B,cAAc,EAAE,cAAc,EAC9B,aAAa,EAAE,oBAAoB,EACnC,WAAW,EAAE,eAAe;IAG/C,YAAY,IAAI,IAAI;IAKpB,eAAe,IAAI,IAAI;IAOjB,YAAY,CAChB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,sBAAsB,EAC3B,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,oBAAoB,CAAC;IAkB1B,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,qBAAqB,EAC1B,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,6BAA6B,CAAC;IA2BnC,QAAQ,CACZ,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,gCAAgC,CAAC;IAsCtC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpF,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAQxG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;YAgB1B,oBAAoB;YAMpB,kBAAkB;YAKlB,gBAAgB;IAO9B,OAAO,CAAC,wBAAwB;IAOhC,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,mBAAmB;IAI3B,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,YAAY;IAkBpB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,iBAAiB;CAI1B"}
+{"version":3,"file":"dataset-import.service.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-import.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAML,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EACV,gCAAgC,EAChC,sBAAsB,EACtB,yBAAyB,EACzB,iCAAiC,EAEjC,qBAAqB,EACrB,6BAA6B,EAC7B,oBAAoB,EACpB,+BAA+B,EAGhC,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,+CAA+C,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAwB,MAAM,gDAAgD,CAAC;AAC7G,OAAO,EAAE,eAAe,EAAE,MAAM,0CAA0C,CAAC;AAC3E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AAGzF,OAAO,EAEL,uBAAuB,EAIxB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAanD,qBACa,oBAAqB,YAAW,YAAY,EAAE,eAAe;IAMtE,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAT1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiE;IACxF,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,QAAQ,CAAS;gBAGN,IAAI,EAAE,uBAAuB,EAC7B,cAAc,EAAE,cAAc,EAC9B,aAAa,EAAE,oBAAoB,EACnC,WAAW,EAAE,eAAe,EAC5B,OAAO,EAAE,qBAAqB;IAGjD,YAAY,IAAI,IAAI;IAKpB,eAAe,IAAI,IAAI;IAOjB,YAAY,CAChB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,sBAAsB,EAC3B,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,oBAAoB,CAAC;IAmB1B,wBAAwB,CAC5B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,+BAA+B,CAAC;IAQrC,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,yBAAyB,EAC9B,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,iCAAiC,CAAC;IAoDvC,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,qBAAqB,EAC1B,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,6BAA6B,CAAC;IA8BnC,QAAQ,CACZ,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,gCAAgC,CAAC;IA+CtC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAOpF,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAQxG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;YAsB1B,oBAAoB;YAMpB,kBAAkB;YAKlB,gBAAgB;IAO9B,OAAO,CAAC,wBAAwB;IAOhC,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,mBAAmB;IAI3B,OAAO,CAAC,aAAa;YAOP,0BAA0B;YAoD1B,kBAAkB;IAqBhC,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,0BAA0B;IAIlC,OAAO,CAAC,2BAA2B;YAMrB,yBAAyB;YAWzB,mBAAmB;IAMjC,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,YAAY;IAmBpB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,iBAAiB;IAKzB,OAAO,CAAC,oBAAoB;CAI7B"}

package/dist/server/modules/dataset/dataset-import.service.js

@@ -16,8 +16,10 @@ const common_1 = require("@nestjs/common");
 const logger_1 = require("@proofhound/logger");
 const access_control_1 = require("../../common/access-control");
 const access_control_service_1 = require("../../common/contracts/access-control.service");
+const object_storage_provider_1 = require("../../common/contracts/object-storage.provider");
 const quota_policy_hook_1 = require("../../common/contracts/quota-policy.hook");
 const dataset_field_schema_util_1 = require("./dataset-field-schema.util");
+const dataset_import_raw_parser_1 = require("./dataset-import-raw-parser");
 const dataset_import_repository_1 = require("./dataset-import.repository");
 const dataset_service_1 = require("./dataset.service");
 const TYPE_INFERENCE_SAMPLE_LIMIT = 500;
@@ -25,12 +27,18 @@ const DEFAULT_SWEEP_INTERVAL_MS = 60_000;
 const DEFAULT_STALE_TIMEOUT_MS = 120_000;
 const MIN_TICK_MS = 1_000;
 const IMAGE_ROLES = new Set(['image', 'image_url', 'image_base64']);
+const DEFAULT_DATASET_RAW_UPLOAD_MAX_BYTES = 2 * 1024 * 1024 * 1024;
+const DEFAULT_RAW_UPLOAD_EXPIRES_IN_SECONDS = 60 * 60;
+const RAW_IMPORT_BATCH_MAX_ROWS = 1000;
+const RAW_IMPORT_BATCH_MAX_BYTES = 8 * 1024 * 1024;
+const RAW_IMPORT_MAX_SAMPLE_BYTES = 8 * 1024 * 1024;
 let DatasetImportService = class DatasetImportService {
-    constructor(repo, datasetService, accessControl, quotaPolicy) {
+    constructor(repo, datasetService, accessControl, quotaPolicy, storage) {
         this.repo = repo;
         this.datasetService = datasetService;
         this.accessControl = accessControl;
         this.quotaPolicy = quotaPolicy;
+        this.storage = storage;
         this.logger = (0, logger_1.createLogger)('dataset-import.service', { service: 'server' });
         this.sweepTimer = null;
         this.sweeping = false;
@@ -48,6 +56,7 @@ let DatasetImportService = class DatasetImportService {
     async createImport(projectId, dto, actor) {
         await this.getWritableProject(projectId, actor);
         this.assertConsistentMappings(dto);
+        this.assertUploadSizeWithinLimit(dto.sourceFile.fileSizeBytes);
         if (await this.repo.isDatasetNameTaken(projectId, dto.name)) {
             throw new common_1.ConflictException('dataset_name_taken');
         }
@@ -60,9 +69,65 @@ let DatasetImportService = class DatasetImportService {
         const row = await this.repo.createImport({ projectId, actorUserId: actor.sub, dto });
         return this.toImportItem(row);
     }
+    async getRawImportCapabilities(projectId, actor) {
+        await this.getAccessibleProject(projectId, actor);
+        return {
+            supported: this.storage.isEnabled() && this.storage.supportsClientUploadSessions(),
+            maxBytes: this.getRawUploadMaxBytes(),
+        };
+    }
+    async createRawImport(projectId, dto, actor) {
+        await this.getWritableProject(projectId, actor);
+        this.assertConsistentMappings(dto);
+        this.assertUploadSizeWithinLimit(dto.sourceFile.fileSizeBytes);
+        if (await this.repo.isDatasetNameTaken(projectId, dto.name)) {
+            throw new common_1.ConflictException('dataset_name_taken');
+        }
+        if (!this.storage.isEnabled() || !this.storage.supportsClientUploadSessions()) {
+            throw new common_1.ServiceUnavailableException('dataset_raw_upload_unavailable');
+        }
+        await this.quotaPolicy.assertCanStore({
+            actor: (0, access_control_1.toActorContext)(actor),
+            bytes: nonnegativeInteger(dto.sourceFile.fileSizeBytes),
+            project: { projectId, source: 'local' },
+            source: 'dataset_raw_import',
+        });
+        const importId = (0, node_crypto_1.randomUUID)();
+        const uploadSession = await this.storage.createUploadSession({
+            project: { projectId, source: 'local' },
+            resourceType: 'dataset_raw',
+            resourceId: importId,
+            name: this.getRawObjectName(dto),
+        }, {
+            contentType: dto.sourceFile.contentType,
+            maxBytes: Math.min(dto.sourceFile.fileSizeBytes, this.getRawUploadMaxBytes()),
+            expiresInSeconds: DEFAULT_RAW_UPLOAD_EXPIRES_IN_SECONDS,
+        });
+        if (!uploadSession) {
+            throw new common_1.ServiceUnavailableException('dataset_raw_upload_unavailable');
+        }
+        try {
+            const row = await this.repo.createImport({
+                importId,
+                projectId,
+                actorUserId: actor.sub,
+                dto,
+                importMode: 'raw_object',
+                rawUploadSession: { sessionId: uploadSession.sessionId, expiresAt: uploadSession.expiresAt },
+            });
+            return { import: this.toImportItem(row), uploadSession, maxBytes: this.getRawUploadMaxBytes() };
+        }
+        catch (error) {
+            await this.storage.abortUpload(uploadSession.sessionId).catch(() => undefined);
+            throw error;
+        }
+    }
     async appendBatch(projectId, importId, dto, actor) {
         await this.getWritableProject(projectId, actor);
         const session = await this.requireImporting(projectId, importId);
+        if (session.importMode !== 'batch') {
+            throw new common_1.ConflictException('dataset_import_batch_not_allowed');
+        }
         // Reject gaps so the committed staging rows always form a contiguous [0, receivedRows) prefix.
         if (dto.batchStartIndex > session.receivedRows) {
             throw new common_1.BadRequestException({ message: 'dataset_import_batch_gap', receivedRows: session.receivedRows });
@@ -85,11 +150,16 @@ let DatasetImportService = class DatasetImportService {
     async complete(projectId, importId, actor) {
         await this.getWritableProject(projectId, actor);
         const session = await this.requireImporting(projectId, importId);
-        const sampleRows = await this.repo.getSampleDataForInference(importId, TYPE_INFERENCE_SAMPLE_LIMIT);
-        const fieldSchema = (0, dataset_field_schema_util_1.buildDatasetFieldSchema)(this.toFieldMappings(session), sampleRows);
-        const hasImages = fieldSchema.some((field) => IMAGE_ROLES.has(field.role));
-        const datasetId = (0, node_crypto_1.randomUUID)();
+        let rawObjectRef = null;
         try {
+            if (session.importMode === 'raw_object') {
+                rawObjectRef = await this.ensureRawObjectRef(projectId, session, actor);
+                await this.ingestRawObjectIntoStaging(projectId, session, actor, rawObjectRef);
+            }
+            const sampleRows = await this.repo.getSampleDataForInference(importId, TYPE_INFERENCE_SAMPLE_LIMIT);
+            const fieldSchema = (0, dataset_field_schema_util_1.buildDatasetFieldSchema)(this.toFieldMappings(session), sampleRows);
+            const hasImages = fieldSchema.some((field) => IMAGE_ROLES.has(field.role));
+            const datasetId = (0, node_crypto_1.randomUUID)();
             const { sampleCount } = await this.repo.promote({
                 importId,
                 projectId,
@@ -108,10 +178,15 @@ let DatasetImportService = class DatasetImportService {
                 actorId: actor.sub,
                 sampleCount,
             });
+            if (rawObjectRef)
+                await this.cleanupRawObjectRef(rawObjectRef, importId);
             this.logger.info({ importId, datasetId, sampleCount }, 'dataset_import_completed');
             return { dataset, sampleCount };
         }
         catch (error) {
+            if (session.importMode === 'raw_object') {
+                await this.cleanupRawImportResources({ ...session, rawObjectRef: rawObjectRef ?? session.rawObjectRef });
+            }
             if (error instanceof dataset_import_repository_1.DatasetImportEmptyError)
                 throw new common_1.BadRequestException('dataset_import_empty');
             if (error instanceof dataset_import_repository_1.DatasetNameTakenError)
@@ -122,7 +197,10 @@ let DatasetImportService = class DatasetImportService {
     // Cancel an in-progress import: delete the session (staging rows cascade). Idempotent — missing session is a no-op.
     async abort(projectId, importId, actor) {
         await this.getWritableProject(projectId, actor);
+        const row = await this.repo.findImportById(projectId, importId);
         await this.repo.deleteImport(projectId, importId);
+        if (row)
+            await this.cleanupRawImportResources(row);
     }
     async getImport(projectId, importId, actor) {
         await this.getAccessibleProject(projectId, actor);
@@ -138,11 +216,16 @@ let DatasetImportService = class DatasetImportService {
         this.sweeping = true;
         try {
             const threshold = new Date(Date.now() - this.getStaleTimeoutMs());
-            const ids = await this.repo.findStaleImportIds(threshold);
-            if (ids.length === 0)
+            const staleImports = await this.repo.findStaleImports(threshold);
+            const ids = staleImports.map((row) => row.id);
+            if (ids.length === 0) {
+                await this.storage.sweepPendingUploads(threshold.toISOString()).catch(() => undefined);
                 return;
+            }
             const deleted = await this.repo.deleteImportsByIds(ids);
-            this.logger.info({ deleted }, 'dataset_import_sweep_reaped');
+            await Promise.all(staleImports.map((row) => this.cleanupRawImportResources(row)));
+            const pendingUploads = await this.storage.sweepPendingUploads(threshold.toISOString()).catch(() => 0);
+            this.logger.info({ deleted, pendingUploads }, 'dataset_import_sweep_reaped');
         }
         catch (error) {
             this.logger.error({ error: error.message }, 'dataset_import_sweep_failed');
@@ -191,11 +274,104 @@ let DatasetImportService = class DatasetImportService {
             return null;
         return String(value);
     }
+    async ingestRawObjectIntoStaging(projectId, session, actor, rawObjectRef) {
+        const stream = await this.storage.getObjectStream(rawObjectRef);
+        const fieldMappings = this.toFieldMappings(session);
+        const externalIdField = this.externalIdFieldName(session);
+        let batch = [];
+        let batchBytes = jsonArrayBytesForEmptyBatch();
+        let rowIndex = 0;
+        const flush = async () => {
+            if (batch.length === 0)
+                return;
+            await this.quotaPolicy.assertCanStore({
+                actor: (0, access_control_1.toActorContext)(actor),
+                bytes: batchBytes,
+                project: { projectId, source: 'local' },
+                source: 'dataset_raw_import_batch',
+            });
+            await this.repo.appendBatch(session.id, batch, rowIndex);
+            batch = [];
+            batchBytes = jsonArrayBytesForEmptyBatch();
+        };
+        for await (const rawRow of (0, dataset_import_raw_parser_1.parseRawDatasetRows)(stream, session.sourceFormat)) {
+            if (batch.length >= RAW_IMPORT_BATCH_MAX_ROWS)
+                await flush();
+            const data = this.projectSample(rawRow, fieldMappings);
+            const sampleBytes = utf8JsonBytes(data);
+            if (sampleBytes > RAW_IMPORT_MAX_SAMPLE_BYTES)
+                throw new common_1.BadRequestException('dataset_import_sample_too_large');
+            let candidateBytes = jsonArrayBytesAfterAppend(batchBytes, batch.length, sampleBytes);
+            if (candidateBytes > RAW_IMPORT_BATCH_MAX_BYTES) {
+                if (batch.length === 0)
+                    throw new common_1.BadRequestException('dataset_import_sample_too_large');
+                await flush();
+                candidateBytes = jsonArrayBytesAfterAppend(batchBytes, batch.length, sampleBytes);
+                if (candidateBytes > RAW_IMPORT_BATCH_MAX_BYTES) {
+                    throw new common_1.BadRequestException('dataset_import_sample_too_large');
+                }
+            }
+            batch.push({ rowIndex, data, externalId: this.getExternalId(data, externalIdField) });
+            batchBytes = candidateBytes;
+            rowIndex += 1;
+        }
+        await flush();
+        return rawObjectRef;
+    }
+    async ensureRawObjectRef(projectId, session, actor) {
+        if (session.rawObjectRef) {
+            this.assertRawObjectWithinLimit(session.rawObjectRef);
+            return session.rawObjectRef;
+        }
+        if (!session.rawUploadSessionId)
+            throw new common_1.BadRequestException('dataset_raw_upload_missing_session');
+        const ref = await this.storage.completeUpload({
+            sessionId: session.rawUploadSessionId,
+            actor: (0, access_control_1.toActorContext)(actor),
+            project: { projectId, source: 'local' },
+        });
+        this.assertRawObjectWithinLimit(ref);
+        await this.repo.markRawObjectRef(projectId, session.id, ref);
+        return ref;
+    }
+    projectSample(sample, fieldMappings) {
+        return Object.fromEntries(fieldMappings.map((field) => [
+            field.name,
+            Object.prototype.hasOwnProperty.call(sample, field.name) ? sample[field.name] : null,
+        ]));
+    }
+    assertRawObjectWithinLimit(ref) {
+        this.assertUploadSizeWithinLimit(ref.bytes);
+    }
+    assertUploadSizeWithinLimit(bytes) {
+        if (nonnegativeInteger(bytes) > this.getRawUploadMaxBytes()) {
+            throw new common_1.BadRequestException('dataset_raw_upload_too_large');
+        }
+    }
+    async cleanupRawImportResources(row) {
+        if (row.rawUploadSessionId) {
+            await this.storage.abortUpload(row.rawUploadSessionId).catch((error) => {
+                this.logger.warn({ importId: row.id, error: error.message }, 'dataset_raw_upload_abort_failed');
+            });
+        }
+        if (row.rawObjectRef) {
+            await this.cleanupRawObjectRef(row.rawObjectRef, row.id);
+        }
+    }
+    async cleanupRawObjectRef(ref, importId) {
+        await this.storage.deleteObjects([ref]).catch((error) => {
+            this.logger.warn({ importId, key: ref.key, error: error.message }, 'dataset_raw_object_delete_failed');
+        });
+    }
+    getRawObjectName(dto) {
+        return `input.${dto.sourceFormat}`;
+    }
     toImportItem(row) {
         return {
             id: row.id,
             projectId: row.projectId,
             datasetId: row.datasetId,
+            importMode: row.importMode === 'raw_object' ? 'raw_object' : 'batch',
             name: row.name,
             description: row.description,
             fileName: row.fileName,
@@ -216,6 +392,10 @@ let DatasetImportService = class DatasetImportService {
         const raw = Number(process.env['DATASET_IMPORT_STALE_TIMEOUT_MS']);
         return Number.isFinite(raw) && raw >= MIN_TICK_MS ? Math.floor(raw) : DEFAULT_STALE_TIMEOUT_MS;
     }
+    getRawUploadMaxBytes() {
+        const raw = Number(process.env['DATASET_RAW_UPLOAD_MAX_BYTES']);
+        return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : DEFAULT_DATASET_RAW_UPLOAD_MAX_BYTES;
+    }
 };
 exports.DatasetImportService = DatasetImportService;
 exports.DatasetImportService = DatasetImportService = __decorate([
@@ -223,11 +403,18 @@ exports.DatasetImportService = DatasetImportService = __decorate([
     __metadata("design:paramtypes", [dataset_import_repository_1.DatasetImportRepository,
         dataset_service_1.DatasetService,
         access_control_service_1.AccessControlService,
-        quota_policy_hook_1.QuotaPolicyHook])
+        quota_policy_hook_1.QuotaPolicyHook,
+        object_storage_provider_1.ObjectStorageProvider])
 ], DatasetImportService);
 function utf8JsonBytes(value) {
     return node_buffer_1.Buffer.byteLength(JSON.stringify(value ?? null), 'utf8');
 }
+function jsonArrayBytesForEmptyBatch() {
+    return 2; // []
+}
+function jsonArrayBytesAfterAppend(currentArrayBytes, currentLength, nextItemBytes) {
+    return currentLength === 0 ? jsonArrayBytesForEmptyBatch() + nextItemBytes : currentArrayBytes + 1 + nextItemBytes;
+}
 function nonnegativeInteger(value) {
     return Number.isFinite(value) ? Math.max(0, Math.trunc(value ?? 0)) : 0;
 }

package/dist/server/modules/dataset/dataset-import.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"dataset-import.service.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-import.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,6CAAqC;AACrC,6CAAyC;AACzC,2CAOwB;AACxB,+CAAkD;AAWlD,gEAA6D;AAC7D,0FAAqF;AACrF,gFAA2E;AAE3E,2EAAsE;AACtE,2EAMqC;AACrC,uDAAmD;AAEnD,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AACzC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AACzC,MAAM,WAAW,GAAG,KAAK,CAAC;AAC1B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AAG7D,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;IAK/B,YACmB,IAA6B,EAC7B,cAA8B,EAC9B,aAAmC,EACnC,WAA4B;QAH5B,SAAI,GAAJ,IAAI,CAAyB;QAC7B,mBAAc,GAAd,cAAc,CAAgB;QAC9B,kBAAa,GAAb,aAAa,CAAsB;QACnC,gBAAW,GAAX,WAAW,CAAiB;QAR9B,WAAM,GAAG,IAAA,qBAAY,EAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChF,eAAU,GAA0B,IAAI,CAAC;QACzC,aAAQ,GAAG,KAAK,CAAC;IAOtB,CAAC;IAEJ,YAAY;QACV,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;IAC5B,CAAC;IAED,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,SAAiB,EACjB,GAA2B,EAC3B,KAAyB;QAEzB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,0BAAiB,CAAC,oBAAoB,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YACpC,KAAK,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC;YAC5B,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YACvD,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;YACvC,MAAM,EAAE,gBAAgB;SACzB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACrF,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,WAAW,CACf,SAAiB,EACjB,QAAgB,EAChB,GAA0B,EAC1B,KAAyB;QAEzB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEjE,+FAA+F;QAC/F,IAAI,GAAG,CAAC,eAAe,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YAC/C,MAAM,IAAI,4BAAmB,CAAC,EAAE,OAAO,EAAE,0BAA0B,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QAC7G,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YACpC,KAAK,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC;YAC5B,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC;YACjC,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;YACvC,MAAM,EAAE,sBAAsB;SAC/B,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAClE,QAAQ,EAAE,GAAG,CAAC,eAAe,GAAG,MAAM;YACtC,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,eAAe,CAAC;SACxD,CAAC,CAAC,CAAC;QAEJ,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3G,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,SAAiB,EACjB,QAAgB,EAChB,KAAyB;QAEzB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEjE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,2BAA2B,CAAC,CAAC;QACpG,MAAM,WAAW,GAAG,IAAA,mDAAuB,EAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,CAAC;QACvF,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3E,MAAM,SAAS,GAAG,IAAA,wBAAU,GAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;gBAC9C,QAAQ;gBACR,SAAS;gBACT,WAAW,EAAE,KAAK,CAAC,GAAG;gBACtB,SAAS;gBACT,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW;gBACX,SAAS;aACV,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,cAAc,CAAC,4BAA4B,CAAC;gBACrD,SAAS;gBACT,SAAS;gBACT,QAAQ;gBACR,OAAO,EAAE,KAAK,CAAC,GAAG;gBAClB,WAAW;aACZ,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,0BAA0B,CAAC,CAAC;YACnF,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;QAClC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,mDAAuB;gBAAE,MAAM,IAAI,4BAAmB,CAAC,sBAAsB,CAAC,CAAC;YACpG,IAAI,KAAK,YAAY,iDAAqB;gBAAE,MAAM,IAAI,0BAAiB,CAAC,oBAAoB,CAAC,CAAC;YAC9F,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,oHAAoH;IACpH,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,QAAgB,EAAE,KAAyB;QACxE,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAiB,EAAE,QAAgB,EAAE,KAAyB;QAC5E,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,0BAAiB,CAAC,kBAAkB,QAAQ,YAAY,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,sGAAsG;IACtG,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAClE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC1D,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC7B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACxF,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,SAAiB,EAAE,KAAyB;QAC7E,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAA,+BAAc,EAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,cAAc,CAAC,CAAC;QAC1G,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,0BAAiB,CAAC,WAAW,SAAS,YAAY,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,SAAiB,EAAE,KAAyB;QAC3E,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAA,+BAAc,EAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;QAC3G,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,SAAiB,EAAE,QAAgB;QAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,0BAAiB,CAAC,kBAAkB,QAAQ,YAAY,CAAC,CAAC;QAC9E,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,MAAM,IAAI,0BAAiB,CAAC,8BAA8B,CAAC,CAAC;QAC5F,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,wBAAwB,CAAC,GAA2B;QAC1D,MAAM,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;QAC5F,IAAI,aAAa,GAAG,CAAC;YAAE,MAAM,IAAI,0BAAiB,CAAC,+BAA+B,CAAC,CAAC;QACpF,MAAM,KAAK,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,0BAAiB,CAAC,iCAAiC,CAAC,CAAC;IAC3G,CAAC;IAEO,eAAe,CAAC,OAAyB;QAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAE,OAAO,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IACzG,CAAC;IAEO,mBAAmB,CAAC,OAAyB;QACnD,OAAO,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;IAC1F,CAAC;IAEO,aAAa,CAAC,MAA+B,EAAE,SAAwB;QAC7E,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACvD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAEO,YAAY,CAAC,GAAqB;QACxC,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC;YACxC,YAAY,EAAE,GAAG,CAAC,YAAyC;YAC3D,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,MAAM,EAAE,GAAG,CAAC,MAA6B;YACzC,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE;YACtC,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IAEO,kBAAkB;QACxB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC;IAClG,CAAC;IAEO,iBAAiB;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;QACnE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC;IACjG,CAAC;CACF,CAAA;AAxNY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;qCAOc,mDAAuB;QACb,gCAAc;QACf,6CAAoB;QACtB,mCAAe;GATpC,oBAAoB,CAwNhC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,oBAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACnD,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1E,CAAC"}
+{"version":3,"file":"dataset-import.service.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-import.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,6CAAqC;AACrC,6CAAyC;AACzC,2CAQwB;AACxB,+CAAkD;AAclD,gEAA6D;AAC7D,0FAAqF;AACrF,4FAA6G;AAC7G,gFAA2E;AAE3E,2EAAsE;AACtE,2EAAkE;AAClE,2EAMqC;AACrC,uDAAmD;AAEnD,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AACzC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AACzC,MAAM,WAAW,GAAG,KAAK,CAAC;AAC1B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AACpE,MAAM,oCAAoC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AACpE,MAAM,qCAAqC,GAAG,EAAE,GAAG,EAAE,CAAC;AACtD,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACnD,MAAM,2BAA2B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAG7C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;IAK/B,YACmB,IAA6B,EAC7B,cAA8B,EAC9B,aAAmC,EACnC,WAA4B,EAC5B,OAA8B;QAJ9B,SAAI,GAAJ,IAAI,CAAyB;QAC7B,mBAAc,GAAd,cAAc,CAAgB;QAC9B,kBAAa,GAAb,aAAa,CAAsB;QACnC,gBAAW,GAAX,WAAW,CAAiB;QAC5B,YAAO,GAAP,OAAO,CAAuB;QAThC,WAAM,GAAG,IAAA,qBAAY,EAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChF,eAAU,GAA0B,IAAI,CAAC;QACzC,aAAQ,GAAG,KAAK,CAAC;IAQtB,CAAC;IAEJ,YAAY;QACV,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;IAC5B,CAAC;IAED,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,SAAiB,EACjB,GAA2B,EAC3B,KAAyB;QAEzB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,0BAAiB,CAAC,oBAAoB,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YACpC,KAAK,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC;YAC5B,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YACvD,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;YACvC,MAAM,EAAE,gBAAgB;SACzB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACrF,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,SAAiB,EACjB,KAAyB;QAEzB,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAClD,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,4BAA4B,EAAE;YAClF,QAAQ,EAAE,IAAI,CAAC,oBAAoB,EAAE;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,SAAiB,EACjB,GAA8B,EAC9B,KAAyB;QAEzB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,0BAAiB,CAAC,oBAAoB,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;YAC9E,MAAM,IAAI,oCAA2B,CAAC,gCAAgC,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YACpC,KAAK,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC;YAC5B,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YACvD,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;YACvC,MAAM,EAAE,oBAAoB;SAC7B,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAA,wBAAU,GAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAC1D;YACE,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;YACvC,YAAY,EAAE,aAAa;YAC3B,UAAU,EAAE,QAAQ;YACpB,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC;SACjC,EACD;YACE,WAAW,EAAE,GAAG,CAAC,UAAU,CAAC,WAAW;YACvC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC7E,gBAAgB,EAAE,qCAAqC;SACxD,CACF,CAAC;QACF,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,oCAA2B,CAAC,gCAAgC,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;gBACvC,QAAQ;gBACR,SAAS;gBACT,WAAW,EAAE,KAAK,CAAC,GAAG;gBACtB,GAAG;gBACH,UAAU,EAAE,YAAY;gBACxB,gBAAgB,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,SAAS,EAAE;aAC7F,CAAC,CAAC;YACH,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAClG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC/E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,SAAiB,EACjB,QAAgB,EAChB,GAA0B,EAC1B,KAAyB;QAEzB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACjE,IAAI,OAAO,CAAC,UAAU,KAAK,OAAO,EAAE,CAAC;YACnC,MAAM,IAAI,0BAAiB,CAAC,kCAAkC,CAAC,CAAC;QAClE,CAAC;QAED,+FAA+F;QAC/F,IAAI,GAAG,CAAC,eAAe,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YAC/C,MAAM,IAAI,4BAAmB,CAAC,EAAE,OAAO,EAAE,0BAA0B,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QAC7G,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YACpC,KAAK,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC;YAC5B,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC;YACjC,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;YACvC,MAAM,EAAE,sBAAsB;SAC/B,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAClE,QAAQ,EAAE,GAAG,CAAC,eAAe,GAAG,MAAM;YACtC,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,eAAe,CAAC;SACxD,CAAC,CAAC,CAAC;QAEJ,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3G,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,SAAiB,EACjB,QAAgB,EAChB,KAAyB;QAEzB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACjE,IAAI,YAAY,GAA2B,IAAI,CAAC;QAEhD,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;gBACxC,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxE,MAAM,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,2BAA2B,CAAC,CAAC;YACpG,MAAM,WAAW,GAAG,IAAA,mDAAuB,EAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,CAAC;YACvF,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3E,MAAM,SAAS,GAAG,IAAA,wBAAU,GAAE,CAAC;YAC/B,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;gBAC9C,QAAQ;gBACR,SAAS;gBACT,WAAW,EAAE,KAAK,CAAC,GAAG;gBACtB,SAAS;gBACT,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW;gBACX,SAAS;aACV,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,cAAc,CAAC,4BAA4B,CAAC;gBACrD,SAAS;gBACT,SAAS;gBACT,QAAQ;gBACR,OAAO,EAAE,KAAK,CAAC,GAAG;gBAClB,WAAW;aACZ,CAAC,CAAC;YACH,IAAI,YAAY;gBAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;YACzE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,0BAA0B,CAAC,CAAC;YACnF,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;QAClC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,OAAO,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,yBAAyB,CAAC,EAAE,GAAG,OAAO,EAAE,YAAY,EAAE,YAAY,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YAC3G,CAAC;YACD,IAAI,KAAK,YAAY,mDAAuB;gBAAE,MAAM,IAAI,4BAAmB,CAAC,sBAAsB,CAAC,CAAC;YACpG,IAAI,KAAK,YAAY,iDAAqB;gBAAE,MAAM,IAAI,0BAAiB,CAAC,oBAAoB,CAAC,CAAC;YAC9F,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,oHAAoH;IACpH,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,QAAgB,EAAE,KAAyB;QACxE,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChE,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAClD,IAAI,GAAG;YAAE,MAAM,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAiB,EAAE,QAAgB,EAAE,KAAyB;QAC5E,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,0BAAiB,CAAC,kBAAkB,QAAQ,YAAY,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,sGAAsG;IACtG,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAClE,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACjE,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBACvF,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACxD,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAClF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACtG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,6BAA6B,CAAC,CAAC;QAC/E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACxF,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,SAAiB,EAAE,KAAyB;QAC7E,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAA,+BAAc,EAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,cAAc,CAAC,CAAC;QAC1G,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,0BAAiB,CAAC,WAAW,SAAS,YAAY,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,SAAiB,EAAE,KAAyB;QAC3E,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAA,+BAAc,EAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;QAC3G,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,SAAiB,EAAE,QAAgB;QAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,0BAAiB,CAAC,kBAAkB,QAAQ,YAAY,CAAC,CAAC;QAC9E,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,MAAM,IAAI,0BAAiB,CAAC,8BAA8B,CAAC,CAAC;QAC5F,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,wBAAwB,CAAC,GAA2B;QAC1D,MAAM,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;QAC5F,IAAI,aAAa,GAAG,CAAC;YAAE,MAAM,IAAI,0BAAiB,CAAC,+BAA+B,CAAC,CAAC;QACpF,MAAM,KAAK,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,0BAAiB,CAAC,iCAAiC,CAAC,CAAC;IAC3G,CAAC;IAEO,eAAe,CAAC,OAAyB;QAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAE,OAAO,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IACzG,CAAC;IAEO,mBAAmB,CAAC,OAAyB;QACnD,OAAO,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;IAC1F,CAAC;IAEO,aAAa,CAAC,MAA+B,EAAE,SAAwB;QAC7E,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACvD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,SAAiB,EACjB,OAAyB,EACzB,KAAyB,EACzB,YAA6B;QAE7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACpD,MAAM,eAAe,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1D,IAAI,KAAK,GAAqB,EAAE,CAAC;QACjC,IAAI,UAAU,GAAG,2BAA2B,EAAE,CAAC;QAC/C,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC/B,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;gBACpC,KAAK,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC;gBAC5B,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;gBACvC,MAAM,EAAE,0BAA0B;aACnC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzD,KAAK,GAAG,EAAE,CAAC;YACX,UAAU,GAAG,2BAA2B,EAAE,CAAC;QAC7C,CAAC,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAA,+CAAmB,EAAC,MAAM,EAAE,OAAO,CAAC,YAAyC,CAAC,EAAE,CAAC;YAC1G,IAAI,KAAK,CAAC,MAAM,IAAI,yBAAyB;gBAAE,MAAM,KAAK,EAAE,CAAC;YAE7D,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACvD,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,WAAW,GAAG,2BAA2B;gBAAE,MAAM,IAAI,4BAAmB,CAAC,iCAAiC,CAAC,CAAC;YAEhH,IAAI,cAAc,GAAG,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YACtF,IAAI,cAAc,GAAG,0BAA0B,EAAE,CAAC;gBAChD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM,IAAI,4BAAmB,CAAC,iCAAiC,CAAC,CAAC;gBACzF,MAAM,KAAK,EAAE,CAAC;gBACd,cAAc,GAAG,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAClF,IAAI,cAAc,GAAG,0BAA0B,EAAE,CAAC;oBAChD,MAAM,IAAI,4BAAmB,CAAC,iCAAiC,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,CAAC,CAAC;YACtF,UAAU,GAAG,cAAc,CAAC;YAC5B,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;QAED,MAAM,KAAK,EAAE,CAAC;QACd,OAAO,YAAY,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAAiB,EACjB,OAAyB,EACzB,KAAyB;QAEzB,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACtD,OAAO,OAAO,CAAC,YAAY,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,kBAAkB;YAAE,MAAM,IAAI,4BAAmB,CAAC,oCAAoC,CAAC,CAAC;QAErG,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YAC5C,SAAS,EAAE,OAAO,CAAC,kBAAkB;YACrC,KAAK,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC;YAC5B,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;SACxC,CAAC,CAAC;QACH,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,aAAa,CACnB,MAA+B,EAC/B,aAAuC;QAEvC,OAAO,MAAM,CAAC,WAAW,CACvB,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI;YACV,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;SACrF,CAAC,CACH,CAAC;IACJ,CAAC;IAEO,0BAA0B,CAAC,GAAoB;QACrD,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAEO,2BAA2B,CAAC,KAAyB;QAC3D,IAAI,kBAAkB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;YAC5D,MAAM,IAAI,4BAAmB,CAAC,8BAA8B,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,GAAqB;QAC3D,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,iCAAiC,CAAC,CAAC;YAC7G,CAAC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,GAAoB,EAAE,QAAgB;QACtE,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,kCAAkC,CAAC,CAAC;QACpH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,gBAAgB,CAAC,GAA2B;QAClD,OAAO,SAAS,GAAG,CAAC,YAAY,EAAE,CAAC;IACrC,CAAC;IAEO,YAAY,CAAC,GAAqB;QACxC,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,UAAU,EAAE,GAAG,CAAC,UAAU,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO;YACpE,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC;YACxC,YAAY,EAAE,GAAG,CAAC,YAAyC;YAC3D,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,MAAM,EAAE,GAAG,CAAC,MAA6B;YACzC,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE;YACtC,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IAEO,kBAAkB;QACxB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC;IAClG,CAAC;IAEO,iBAAiB;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;QACnE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC;IACjG,CAAC;IAEO,oBAAoB;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;QAChE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,oCAAoC,CAAC;IAClG,CAAC;CACF,CAAA;AA3aY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;qCAOc,mDAAuB;QACb,gCAAc;QACf,6CAAoB;QACtB,mCAAe;QACnB,+CAAqB;GAVtC,oBAAoB,CA2ahC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,oBAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,2BAA2B;IAClC,OAAO,CAAC,CAAC,CAAC,KAAK;AACjB,CAAC;AAED,SAAS,yBAAyB,CAAC,iBAAyB,EAAE,aAAqB,EAAE,aAAqB;IACxG,OAAO,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,2BAA2B,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,GAAG,aAAa,CAAC;AACrH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACnD,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1E,CAAC"}

package/dist/server/modules/dataset/dataset-sample-offload.d.ts

@@ -0,0 +1,36 @@
+import { Buffer } from 'node:buffer';
+import type { DatasetFieldSchemaDto } from '@proofhound/shared';
+import type { ObjectCodec, StoredObjectRef } from '../../common/contracts/object-storage.provider';
+export interface StagingSample {
+    data: unknown;
+    externalId: string | null;
+}
+export interface DatasetSampleOffloadRow {
+    datasetId: string;
+    data: null;
+    externalId: string | null;
+    payloadRef: {
+        shard: StoredObjectRef;
+        rowIndex: number;
+    };
+    searchPreview: string | null;
+    expectedOutputScalar: string | null;
+    labelScalar: string | null;
+    categoryScalar: string | null;
+    indexValues: Record<string, string> | null;
+}
+export interface OffloadStagingOptions {
+    datasetId: string;
+    sampleCount: number;
+    batchSize: number;
+    fieldSchema: DatasetFieldSchemaDto[];
+    codec?: ObjectCodec;
+    readBatch: (offset: number, limit: number) => Promise<StagingSample[]>;
+    putShard: (name: string, body: Buffer) => Promise<StoredObjectRef>;
+    insertRows: (rows: DatasetSampleOffloadRow[]) => Promise<void>;
+}
+export declare function offloadStagingToShards(opts: OffloadStagingOptions): Promise<{
+    shards: number;
+    storagePrefix: string | null;
+}>;
+//# sourceMappingURL=dataset-sample-offload.d.ts.map

package/dist/server/modules/dataset/dataset-sample-offload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-sample-offload.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-offload.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,gDAAgD,CAAC;AAInG,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE;QAAE,KAAK,EAAE,eAAe,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,qBAAqB,EAAE,CAAC;IACrC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACvE,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;IACnE,UAAU,EAAE,CAAC,IAAI,EAAE,uBAAuB,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAChE;AAED,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CA2C3D"}

package/dist/server/modules/dataset/dataset-sample-offload.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.offloadStagingToShards = offloadStagingToShards;
+const run_result_payload_1 = require("../run-result/run-result-payload");
+const dataset_sample_projection_1 = require("./dataset-sample-projection");
+async function offloadStagingToShards(opts) {
+    const codec = opts.codec ?? 'gzip';
+    let shardSeq = 0;
+    let offset = 0;
+    let firstShardKey = null;
+    while (offset < opts.sampleCount) {
+        const batch = await opts.readBatch(offset, opts.batchSize);
+        if (batch.length === 0)
+            break;
+        const body = await (0, run_result_payload_1.encodeShard)(batch.map((row) => row.data), codec);
+        const shardRef = await opts.putShard(`shard-${String(shardSeq).padStart(5, '0')}.jsonl.gz`, body);
+        firstShardKey ??= shardRef.key;
+        await opts.insertRows(batch.map((row, rowIndex) => {
+            const data = (row.data ?? null);
+            const projection = (0, dataset_sample_projection_1.projectDatasetSample)(data, opts.fieldSchema);
+            return {
+                datasetId: opts.datasetId,
+                data: null,
+                externalId: row.externalId,
+                payloadRef: { shard: shardRef, rowIndex },
+                searchPreview: projection.searchPreview,
+                expectedOutputScalar: projection.expectedOutputScalar,
+                labelScalar: projection.labelScalar,
+                categoryScalar: projection.categoryScalar,
+                indexValues: projection.indexValues,
+            };
+        }));
+        offset += batch.length;
+        shardSeq += 1;
+    }
+    return {
+        shards: shardSeq,
+        storagePrefix: firstShardKey ? firstShardKey.slice(0, firstShardKey.lastIndexOf('/') + 1) : null,
+    };
+}
+//# sourceMappingURL=dataset-sample-offload.js.map

package/dist/server/modules/dataset/dataset-sample-offload.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-sample-offload.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-offload.ts"],"names":[],"mappings":";;AAwCA,wDA6CC;AA5ED,yEAA+D;AAC/D,2EAAmE;AA8B5D,KAAK,UAAU,sBAAsB,CAC1C,IAA2B;IAE3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC;IACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,aAAa,GAAkB,IAAI,CAAC;IAExC,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC3D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAE9B,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAW,EAC5B,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAC5B,KAAK,CACN,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAClG,aAAa,KAAK,QAAQ,CAAC,GAAG,CAAC;QAE/B,MAAM,IAAI,CAAC,UAAU,CACnB,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAmC,CAAC;YAClE,MAAM,UAAU,GAAG,IAAA,gDAAoB,EAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAChE,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,IAAI;gBACV,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,UAAU,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE;gBACzC,aAAa,EAAE,UAAU,CAAC,aAAa;gBACvC,oBAAoB,EAAE,UAAU,CAAC,oBAAoB;gBACrD,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,cAAc,EAAE,UAAU,CAAC,cAAc;gBACzC,WAAW,EAAE,UAAU,CAAC,WAAW;aACpC,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,QAAQ,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,OAAO;QACL,MAAM,EAAE,QAAQ;QAChB,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;KACjG,CAAC;AACJ,CAAC"}

package/dist/server/modules/dataset/dataset-sample-payload.d.ts

@@ -0,0 +1,21 @@
+import { ObjectStorageProvider, type StoredObjectRef } from '../../common/contracts/object-storage.provider';
+/** Stored in `dataset_samples.payload_ref`: the shard holding this sample's data + its line. */
+export interface DatasetSamplePayloadRef {
+    shard: StoredObjectRef;
+    rowIndex: number;
+}
+export interface DatasetSamplePayloadRow {
+    data: unknown;
+    payloadRef: DatasetSamplePayloadRef | null;
+}
+export declare class DatasetSamplePayloadReader {
+    private readonly storage;
+    constructor(storage: ObjectStorageProvider);
+    /** Resolve one sample's data: inline when present, else from its shard. */
+    hydrate(row: DatasetSamplePayloadRow): Promise<unknown>;
+    /** Batch: groups rows by shard so a 500-sample batch is one GET per shard, not one per sample. */
+    hydrateMany(rows: DatasetSamplePayloadRow[]): Promise<unknown[]>;
+    /** The ref to read from, or null when the inline value should be used (cache / no offload / disabled). */
+    private shardRef;
+}
+//# sourceMappingURL=dataset-sample-payload.d.ts.map

package/dist/server/modules/dataset/dataset-sample-payload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-sample-payload.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-payload.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,qBAAqB,EAAE,KAAK,eAAe,EAAE,MAAM,gDAAgD,CAAC;AAG7G,gGAAgG;AAChG,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,uBAAuB,GAAG,IAAI,CAAC;CAC5C;AAED,qBACa,0BAA0B;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,qBAAqB;IAE3D,2EAA2E;IACrE,OAAO,CAAC,GAAG,EAAE,uBAAuB,GAAG,OAAO,CAAC,OAAO,CAAC;IAO7D,kGAAkG;IAC5F,WAAW,CAAC,IAAI,EAAE,uBAAuB,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAqBtE,0GAA0G;IAC1G,OAAO,CAAC,QAAQ;CAIjB"}

package/dist/server/modules/dataset/dataset-sample-payload.js

@@ -0,0 +1,72 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatasetSamplePayloadReader = void 0;
+// DatasetSamplePayloadReader — the read seam for dataset-sample content (SPEC 22 §7.3).
+//
+// The worker hot paths (experiment rendering, optimization rounds) load a sample's full `data`. Once
+// promote tiers it out, `data` is null and the authoritative content lives in an object-storage shard.
+// This seam returns the inline value when present (a small-sample cache, or no offload), else reads
+// the shard. Pure pass-through when storage is disabled / the row was never offloaded. It reuses the
+// generic JSONL shard codec from the run-result seam (one data object per line).
+const common_1 = require("@nestjs/common");
+const object_storage_provider_1 = require("../../common/contracts/object-storage.provider");
+const run_result_payload_1 = require("../run-result/run-result-payload");
+let DatasetSamplePayloadReader = class DatasetSamplePayloadReader {
+    constructor(storage) {
+        this.storage = storage;
+    }
+    /** Resolve one sample's data: inline when present, else from its shard. */
+    async hydrate(row) {
+        const ref = this.shardRef(row);
+        if (!ref)
+            return row.data ?? null;
+        const lines = await (0, run_result_payload_1.decodeShard)(await this.storage.getObject(ref.shard), ref.shard.codec);
+        return lines[ref.rowIndex] ?? null;
+    }
+    /** Batch: groups rows by shard so a 500-sample batch is one GET per shard, not one per sample. */
+    async hydrateMany(rows) {
+        const out = rows.map((r) => r.data ?? null);
+        const byShard = new Map();
+        rows.forEach((row, index) => {
+            const ref = this.shardRef(row);
+            if (!ref)
+                return;
+            const key = shardKey(ref.shard);
+            const group = byShard.get(key);
+            if (group)
+                group.entries.push({ index, rowIndex: ref.rowIndex });
+            else
+                byShard.set(key, { shard: ref.shard, entries: [{ index, rowIndex: ref.rowIndex }] });
+        });
+        await Promise.all([...byShard.values()].map(async ({ shard, entries }) => {
+            const lines = await (0, run_result_payload_1.decodeShard)(await this.storage.getObject(shard), shard.codec);
+            for (const { index, rowIndex } of entries)
+                out[index] = lines[rowIndex] ?? null;
+        }));
+        return out;
+    }
+    /** The ref to read from, or null when the inline value should be used (cache / no offload / disabled). */
+    shardRef(row) {
+        if (row.data != null || row.payloadRef == null || !this.storage.isEnabled())
+            return null;
+        return row.payloadRef;
+    }
+};
+exports.DatasetSamplePayloadReader = DatasetSamplePayloadReader;
+exports.DatasetSamplePayloadReader = DatasetSamplePayloadReader = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [object_storage_provider_1.ObjectStorageProvider])
+], DatasetSamplePayloadReader);
+function shardKey(ref) {
+    return `${ref.provider}:${ref.bucket ?? ''}:${ref.key}`;
+}
+//# sourceMappingURL=dataset-sample-payload.js.map

package/dist/server/modules/dataset/dataset-sample-payload.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-sample-payload.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-payload.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wFAAwF;AACxF,EAAE;AACF,qGAAqG;AACrG,uGAAuG;AACvG,oGAAoG;AACpG,qGAAqG;AACrG,iFAAiF;AACjF,2CAA4C;AAC5C,4FAA6G;AAC7G,yEAA+D;AAcxD,IAAM,0BAA0B,GAAhC,MAAM,0BAA0B;IACrC,YAA6B,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;IAAG,CAAC;IAE/D,2EAA2E;IAC3E,KAAK,CAAC,OAAO,CAAC,GAA4B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG;YAAE,OAAO,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,IAAA,gCAAW,EAA0B,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnH,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IAED,kGAAkG;IAClG,KAAK,CAAC,WAAW,CAAC,IAA+B;QAC/C,MAAM,GAAG,GAAc,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2F,CAAC;QACnH,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,OAAO;YACjB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,KAAK;gBAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;;gBAC5D,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE;YACrD,MAAM,KAAK,GAAG,MAAM,IAAA,gCAAW,EAA0B,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3G,KAAK,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,OAAO;gBAAE,GAAG,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QAClF,CAAC,CAAC,CACH,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,0GAA0G;IAClG,QAAQ,CAAC,GAA4B;QAC3C,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,IAAI,GAAG,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAAE,OAAO,IAAI,CAAC;QACzF,OAAO,GAAG,CAAC,UAAU,CAAC;IACxB,CAAC;CACF,CAAA;AAtCY,gEAA0B;qCAA1B,0BAA0B;IADtC,IAAA,mBAAU,GAAE;qCAE2B,+CAAqB;GADhD,0BAA0B,CAsCtC;AAED,SAAS,QAAQ,CAAC,GAAoB;IACpC,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;AAC1D,CAAC"}

package/dist/server/modules/dataset/dataset-sample-projection.d.ts

@@ -0,0 +1,11 @@
+import type { DatasetFieldSchemaDto } from '@proofhound/shared';
+export interface DatasetSampleProjection {
+    searchPreview: string | null;
+    expectedOutputScalar: string | null;
+    labelScalar: string | null;
+    categoryScalar: string | null;
+    indexValues: Record<string, string> | null;
+}
+/** Build the DB-side projection for one sample from its data + the dataset's field schema. */
+export declare function projectDatasetSample(data: Record<string, unknown> | null, fieldSchema: DatasetFieldSchemaDto[]): DatasetSampleProjection;
+//# sourceMappingURL=dataset-sample-projection.d.ts.map

package/dist/server/modules/dataset/dataset-sample-projection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-sample-projection.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-projection.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEhE,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CAC5C;AAoBD,8FAA8F;AAC9F,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,EACpC,WAAW,EAAE,qBAAqB,EAAE,GACnC,uBAAuB,CAwBzB"}

package/dist/server/modules/dataset/dataset-sample-projection.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.projectDatasetSample = projectDatasetSample;
+const PREVIEW_MAX = 1000;
+const SCALAR_MAX = 200;
+const IMAGE_ROLES = new Set(['image', 'image_url', 'image_base64']);
+function asScalar(value) {
+    if (value === null || value === undefined)
+        return null;
+    if (typeof value === 'string')
+        return value.length > SCALAR_MAX ? null : value;
+    if (typeof value === 'number' || typeof value === 'boolean')
+        return String(value);
+    return null;
+}
+function previewOf(data) {
+    if (data === null || data === undefined)
+        return null;
+    const text = typeof data === 'string' ? data : JSON.stringify(data);
+    if (text.length === 0)
+        return null;
+    return text.length > PREVIEW_MAX ? text.slice(0, PREVIEW_MAX) : text;
+}
+/** Build the DB-side projection for one sample from its data + the dataset's field schema. */
+function projectDatasetSample(data, fieldSchema) {
+    if (data === null) {
+        return { searchPreview: null, expectedOutputScalar: null, labelScalar: null, categoryScalar: null, indexValues: null };
+    }
+    const expectedField = fieldSchema.find((f) => f.role === 'expected_output')?.name;
+    const expectedOutputScalar = expectedField ? asScalar(data[expectedField]) : null;
+    // index_values: short scalar values of the non-image fields, so distribution / filter on any
+    // configurable field works off `index_values->>field` once `data` is offloaded.
+    const indexValues = {};
+    for (const field of fieldSchema) {
+        if (IMAGE_ROLES.has(field.role))
+            continue;
+        const scalar = asScalar(data[field.name]);
+        if (scalar !== null)
+            indexValues[field.name] = scalar;
+    }
+    return {
+        searchPreview: previewOf(data),
+        expectedOutputScalar,
+        labelScalar: null,
+        categoryScalar: null,
+        indexValues: Object.keys(indexValues).length > 0 ? indexValues : null,
+    };
+}
+//# sourceMappingURL=dataset-sample-projection.js.map