npm - @proofhound/core - Versions diffs - 0.1.12 → 0.1.13 - Mend

@proofhound/core 0.1.12 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (82) hide show

package/dist/server/modules/dataset/dataset-import.repository.js CHANGED Viewed

@@ -17,7 +17,12 @@ const common_1 = require("@nestjs/common");
 const drizzle_orm_1 = require("drizzle-orm");
 const db_1 = require("@proofhound/db");
 const database_constants_1 = require("../../../shared/database/database.constants");
-const { datasetImports, datasetImportSamples, datasets, projects } = db_1.schema;
+const object_storage_provider_1 = require("../../common/contracts/object-storage.provider");
+const dataset_sample_offload_1 = require("./dataset-sample-offload");
+const { datasetImports, datasetImportSamples, datasetSamples, datasets, projects } = db_1.schema;
+// Per-shard batch for offload-at-promote. Bounded so a batch's data stays in memory only briefly
+// (large image/base64 samples make per-row size unpredictable); each batch becomes one R2 shard.
+const PROMOTE_SHARD_BATCH = 200;
 // Thrown inside the promote transaction so the caller can map to the right HTTP status while the tx rolls back.
 class DatasetImportEmptyError extends Error {
 }
@@ -26,8 +31,9 @@ class DatasetNameTakenError extends Error {
 }
 exports.DatasetNameTakenError = DatasetNameTakenError;
 let DatasetImportRepository = class DatasetImportRepository {
-    constructor(db) {
+    constructor(db, storage) {
         this.db = db;
+        this.storage = storage;
     }
     async findProjectAccess(projectId) {
         const rows = await this.db
@@ -129,12 +135,41 @@ let DatasetImportRepository = class DatasetImportRepository {
                 hasImages: args.hasImages,
                 createdBy: args.actorUserId,
             });
-            await tx.execute((0, drizzle_orm_1.sql) `
-        INSERT INTO ph_assets.dataset_samples (dataset_id, data, external_id)
-        SELECT ${args.datasetId}::uuid, data, external_id
-        FROM ph_assets.dataset_import_samples
-        WHERE import_id = ${args.importId}::uuid
-      `);
+            if (this.storage.isEnabled()) {
+                // Offload-at-promote (SPEC 22 §7.2): stream staging into shards + projected rows. The pure
+                // orchestration lives in dataset-sample-offload.ts; here we just bind the tx / storage I/O.
+                const project = { projectId: args.projectId, source: 'local' };
+                const { storagePrefix } = await (0, dataset_sample_offload_1.offloadStagingToShards)({
+                    datasetId: args.datasetId,
+                    sampleCount,
+                    batchSize: PROMOTE_SHARD_BATCH,
+                    fieldSchema: args.fieldSchema,
+                    readBatch: (offset, limit) => tx
+                        .select({ data: datasetImportSamples.data, externalId: datasetImportSamples.externalId })
+                        .from(datasetImportSamples)
+                        .where((0, drizzle_orm_1.eq)(datasetImportSamples.importId, args.importId))
+                        .orderBy((0, drizzle_orm_1.asc)(datasetImportSamples.rowIndex))
+                        .limit(limit)
+                        .offset(offset),
+                    putShard: (name, body) => this.storage.putObject({ project, resourceType: 'dataset_normalized', resourceId: args.datasetId, name }, body, {
+                        codec: 'gzip',
+                    }),
+                    insertRows: async (rows) => {
+                        await tx.insert(datasetSamples).values(rows);
+                    },
+                });
+                if (storagePrefix) {
+                    await tx.update(datasets).set({ storagePrefix }).where((0, drizzle_orm_1.eq)(datasets.id, args.datasetId));
+                }
+            }
+            else {
+                await tx.execute((0, drizzle_orm_1.sql) `
+          INSERT INTO ph_assets.dataset_samples (dataset_id, data, external_id)
+          SELECT ${args.datasetId}::uuid, data, external_id
+          FROM ph_assets.dataset_import_samples
+          WHERE import_id = ${args.importId}::uuid
+        `);
+            }
             await tx
                 .update(datasetImports)
                 .set({ status: 'ready', datasetId: args.datasetId, updatedAt: new Date() })
@@ -171,6 +206,6 @@ exports.DatasetImportRepository = DatasetImportRepository;
 exports.DatasetImportRepository = DatasetImportRepository = __decorate([
     (0, common_1.Injectable)(),
     __param(0, (0, common_1.Inject)(database_constants_1.DATABASE_CLIENT)),
-    __metadata("design:paramtypes", [Object])
+    __metadata("design:paramtypes", [Object, object_storage_provider_1.ObjectStorageProvider])
 ], DatasetImportRepository);
 //# sourceMappingURL=dataset-import.repository.js.map

package/dist/server/modules/dataset/dataset-import.repository.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"dataset-import.repository.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-import.repository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAAoD;AACpD,6CAAqE;AAErE,uCAAwC;AAExC,oFAA8E;~~AAE9E~~,MAAM,EAAE,cAAc,EAAE,oBAAoB,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,WAAM,CAAC;~~AA4C5E~~,gHAAgH;AAChH,MAAa,uBAAwB,SAAQ,KAAK;CAAG;AAArD,0DAAqD;AACrD,MAAa,qBAAsB,SAAQ,KAAK;CAAG;AAAnD,sDAAmD;AAG5C,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAClC,~~YAAsD~~,EAAY;~~QAAZ~~,OAAE,GAAF,EAAE,CAAU;~~IAAG~~,CAAC;~~IAEtE~~,KAAK,CAAC,iBAAiB,CAAC,SAAiB;QACvC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;aAC3B,IAAI,CAAC,QAAQ,CAAC;aACd,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,IAAA,oBAAM,EAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;aAClE,KAAK,CAAC,CAAC,CAAC,CAAC;QACZ,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,SAAiB,EAAE,IAAY;QACtD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;aAC3B,IAAI,CAAC,QAAQ,CAAC;aACd,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAA,oBAAM,EAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;aAClG,KAAK,CAAC,CAAC,CAAC,CAAC;QACZ,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAA6B;QAC9C,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE;aACxB,MAAM,CAAC,cAAc,CAAC;aACtB,MAAM,CAAC;YACN,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;YACnB,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI;YACjD,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa;YACrC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ;YACtC,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa;YAChD,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,WAAW,IAAI,IAAI;YACpD,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY;YACnC,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACrD,SAAS,EAAE,IAAI,CAAC,WAAW;SAC5B,CAAC;aACD,SAAS,EAAE,CAAC;QACf,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACnE,OAAO,GAAuB,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,QAAgB;QACtD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,EAAE;aACR,IAAI,CAAC,cAAc,CAAC;aACpB,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;aACpF,KAAK,CAAC,CAAC,CAAC,CAAC;QACZ,OAAQ,IAAI,CAAC,CAAC,CAAkC,IAAI,IAAI,CAAC;IAC3D,CAAC;IAED,uGAAuG;IACvG,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,IAAsB,EAAE,gBAAwB;QAClF,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACtC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,EAAE;qBACL,MAAM,CAAC,oBAAoB,CAAC;qBAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;qBAC7G,mBAAmB,EAAE,CAAC;YAC3B,CAAC;YACD,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,EAAE;iBACvB,MAAM,CAAC,cAAc,CAAC;iBACtB,GAAG,CAAC;gBACH,YAAY,EAAE,IAAA,iBAAG,EAAA,YAAY,cAAc,CAAC,YAAY,KAAK,gBAAgB,GAAG;gBAChF,SAAS,EAAE,IAAI,IAAI,EAAE;aACtB,CAAC;iBACD,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;iBACtC,SAAS,CAAC,EAAE,YAAY,EAAE,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;YAC5D,OAAO,OAAO,EAAE,YAAY,IAAI,gBAAgB,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,QAAgB,EAAE,KAAa;QAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,IAAI,EAAE,oBAAoB,CAAC,IAAI,EAAE,CAAC;aAC3C,IAAI,CAAC,oBAAoB,CAAC;aAC1B,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;aAClD,OAAO,CAAC,IAAA,iBAAG,EAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;aAC3C,KAAK,CAAC,KAAK,CAAC,CAAC;QAChB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACtB,GAAG,CAAC,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,IAAgC,CAAC,CAAC,CAAC,EAAE,CAClH,CAAC;IACJ,CAAC;IAED,6HAA6H;IAC7H,KAAK,CAAC,OAAO,CAAC,IAA8B;QAC1C,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACtC,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE;iBACxB,MAAM,CAAC,EAAE,KAAK,EAAE,IAAA,iBAAG,EAAQ,eAAe,EAAE,CAAC;iBAC7C,IAAI,CAAC,oBAAoB,CAAC;iBAC1B,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YACjD,IAAI,WAAW,KAAK,CAAC;gBAAE,MAAM,IAAI,uBAAuB,EAAE,CAAC;YAE3D,MAAM,KAAK,GAAG,MAAM,EAAE;iBACnB,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;iBAC3B,IAAI,CAAC,QAAQ,CAAC;iBACd,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAA,oBAAM,EAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;iBAC5G,KAAK,CAAC,CAAC,CAAC,CAAC;YACZ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,IAAI,qBAAqB,EAAE,CAAC;YAExD,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;gBAC/B,EAAE,EAAE,IAAI,CAAC,SAAS;gBAClB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW;gBACX,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,WAAW;aAC5B,CAAC,CAAC;YAEH,MAAM,EAAE,CAAC,OAAO,CAAC,IAAA,iBAAG,EAAA;;~~iBAET~~,IAAI,CAAC,SAAS;;~~4BAEH~~,IAAI,CAAC,QAAQ;~~OAClC~~,CAAC,CAAC;~~YAEH~~,MAAM,EAAE;iBACL,MAAM,CAAC,cAAc,CAAC;iBACtB,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;iBAC1E,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAE/C,MAAM,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAE9F,OAAO,EAAE,WAAW,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,QAAgB;QACpD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE;aAC1B,MAAM,CAAC,cAAc,CAAC;aACtB,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;aACpF,SAAS,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,SAAe;QACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC;aACjC,IAAI,CAAC,cAAc,CAAC;aACpB,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,IAAA,gBAAE,EAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,GAAa;QACpC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE;aAC1B,MAAM,CAAC,cAAc,CAAC;aACtB,KAAK,CAAC,IAAA,qBAAO,EAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;aACtC,SAAS,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;CACF,CAAA;~~AAzJY~~,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;~~IAEE~~,WAAA,IAAA,eAAM,EAAC,oCAAe,CAAC,CAAA~~;;GADzB~~,uBAAuB,~~CAyJnC~~"}
1	+ {"version":3,"file":"dataset-import.repository.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-import.repository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAAoD;AACpD,6CAAqE;AAErE,uCAAwC;AAExC,oFAA8E;AAC9E,4FAAuF;AACvF,qEAAkE;AAElE,MAAM,EAAE,cAAc,EAAE,oBAAoB,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,WAAM,CAAC;AAE5F,iGAAiG;AACjG,iGAAiG;AACjG,MAAM,mBAAmB,GAAG,GAAG,CAAC;AA4ChC,gHAAgH;AAChH,MAAa,uBAAwB,SAAQ,KAAK;CAAG;AAArD,0DAAqD;AACrD,MAAa,qBAAsB,SAAQ,KAAK;CAAG;AAAnD,sDAAmD;AAG5C,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAClC,YAC4C,EAAY,EACrC,OAA8B;QADL,OAAE,GAAF,EAAE,CAAU;QACrC,YAAO,GAAP,OAAO,CAAuB;IAC9C,CAAC;IAEJ,KAAK,CAAC,iBAAiB,CAAC,SAAiB;QACvC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;aAC3B,IAAI,CAAC,QAAQ,CAAC;aACd,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,IAAA,oBAAM,EAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;aAClE,KAAK,CAAC,CAAC,CAAC,CAAC;QACZ,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,SAAiB,EAAE,IAAY;QACtD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;aAC3B,IAAI,CAAC,QAAQ,CAAC;aACd,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAA,oBAAM,EAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;aAClG,KAAK,CAAC,CAAC,CAAC,CAAC;QACZ,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAA6B;QAC9C,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE;aACxB,MAAM,CAAC,cAAc,CAAC;aACtB,MAAM,CAAC;YACN,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;YACnB,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI;YACjD,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa;YACrC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ;YACtC,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa;YAChD,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,WAAW,IAAI,IAAI;YACpD,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY;YACnC,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACrD,SAAS,EAAE,IAAI,CAAC,WAAW;SAC5B,CAAC;aACD,SAAS,EAAE,CAAC;QACf,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACnE,OAAO,GAAuB,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,QAAgB;QACtD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,EAAE;aACR,IAAI,CAAC,cAAc,CAAC;aACpB,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;aACpF,KAAK,CAAC,CAAC,CAAC,CAAC;QACZ,OAAQ,IAAI,CAAC,CAAC,CAAkC,IAAI,IAAI,CAAC;IAC3D,CAAC;IAED,uGAAuG;IACvG,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,IAAsB,EAAE,gBAAwB;QAClF,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACtC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,EAAE;qBACL,MAAM,CAAC,oBAAoB,CAAC;qBAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;qBAC7G,mBAAmB,EAAE,CAAC;YAC3B,CAAC;YACD,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,EAAE;iBACvB,MAAM,CAAC,cAAc,CAAC;iBACtB,GAAG,CAAC;gBACH,YAAY,EAAE,IAAA,iBAAG,EAAA,YAAY,cAAc,CAAC,YAAY,KAAK,gBAAgB,GAAG;gBAChF,SAAS,EAAE,IAAI,IAAI,EAAE;aACtB,CAAC;iBACD,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;iBACtC,SAAS,CAAC,EAAE,YAAY,EAAE,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;YAC5D,OAAO,OAAO,EAAE,YAAY,IAAI,gBAAgB,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,QAAgB,EAAE,KAAa;QAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,IAAI,EAAE,oBAAoB,CAAC,IAAI,EAAE,CAAC;aAC3C,IAAI,CAAC,oBAAoB,CAAC;aAC1B,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;aAClD,OAAO,CAAC,IAAA,iBAAG,EAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;aAC3C,KAAK,CAAC,KAAK,CAAC,CAAC;QAChB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACtB,GAAG,CAAC,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,IAAgC,CAAC,CAAC,CAAC,EAAE,CAClH,CAAC;IACJ,CAAC;IAED,6HAA6H;IAC7H,KAAK,CAAC,OAAO,CAAC,IAA8B;QAC1C,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACtC,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE;iBACxB,MAAM,CAAC,EAAE,KAAK,EAAE,IAAA,iBAAG,EAAQ,eAAe,EAAE,CAAC;iBAC7C,IAAI,CAAC,oBAAoB,CAAC;iBAC1B,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YACjD,IAAI,WAAW,KAAK,CAAC;gBAAE,MAAM,IAAI,uBAAuB,EAAE,CAAC;YAE3D,MAAM,KAAK,GAAG,MAAM,EAAE;iBACnB,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;iBAC3B,IAAI,CAAC,QAAQ,CAAC;iBACd,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAA,oBAAM,EAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;iBAC5G,KAAK,CAAC,CAAC,CAAC,CAAC;YACZ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,IAAI,qBAAqB,EAAE,CAAC;YAExD,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;gBAC/B,EAAE,EAAE,IAAI,CAAC,SAAS;gBAClB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW;gBACX,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,WAAW;aAC5B,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;gBAC7B,2FAA2F;gBAC3F,4FAA4F;gBAC5F,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,OAAgB,EAAE,CAAC;gBACxE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAA,+CAAsB,EAAC;oBACrD,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,WAAW;oBACX,SAAS,EAAE,mBAAmB;oBAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,SAAS,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAC3B,EAAE;yBACC,MAAM,CAAC,EAAE,IAAI,EAAE,oBAAoB,CAAC,IAAI,EAAE,UAAU,EAAE,oBAAoB,CAAC,UAAU,EAAE,CAAC;yBACxF,IAAI,CAAC,oBAAoB,CAAC;yBAC1B,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;yBACvD,OAAO,CAAC,IAAA,iBAAG,EAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;yBAC3C,KAAK,CAAC,KAAK,CAAC;yBACZ,MAAM,CAAC,MAAM,CAAC;oBACnB,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CACvB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE;wBAC9G,KAAK,EAAE,MAAM;qBACd,CAAC;oBACJ,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;wBACzB,MAAM,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/C,CAAC;iBACF,CAAC,CAAC;gBACH,IAAI,aAAa,EAAE,CAAC;oBAClB,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC1F,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,EAAE,CAAC,OAAO,CAAC,IAAA,iBAAG,EAAA;;mBAET,IAAI,CAAC,SAAS;;8BAEH,IAAI,CAAC,QAAQ;SAClC,CAAC,CAAC;YACL,CAAC;YAED,MAAM,EAAE;iBACL,MAAM,CAAC,cAAc,CAAC;iBACtB,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;iBAC1E,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAE/C,MAAM,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAE9F,OAAO,EAAE,WAAW,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,QAAgB;QACpD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE;aAC1B,MAAM,CAAC,cAAc,CAAC;aACtB,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAA,gBAAE,EAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;aACpF,SAAS,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,SAAe;QACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE;aACvB,MAAM,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC;aACjC,IAAI,CAAC,cAAc,CAAC;aACpB,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,IAAA,gBAAE,EAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,GAAa;QACpC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE;aAC1B,MAAM,CAAC,cAAc,CAAC;aACtB,KAAK,CAAC,IAAA,qBAAO,EAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;aACtC,SAAS,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;CACF,CAAA;AA1LY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;IAGR,WAAA,IAAA,eAAM,EAAC,oCAAe,CAAC,CAAA;6CACE,+CAAqB;GAHtC,uBAAuB,CA0LnC"}

package/dist/server/modules/dataset/dataset-sample-offload.d.ts ADDED Viewed

@@ -0,0 +1,36 @@
+import { Buffer } from 'node:buffer';
+import type { DatasetFieldSchemaDto } from '@proofhound/shared';
+import type { ObjectCodec, StoredObjectRef } from '../../common/contracts/object-storage.provider';
+export interface StagingSample {
+    data: unknown;
+    externalId: string | null;
+}
+export interface DatasetSampleOffloadRow {
+    datasetId: string;
+    data: null;
+    externalId: string | null;
+    payloadRef: {
+        shard: StoredObjectRef;
+        rowIndex: number;
+    };
+    searchPreview: string | null;
+    expectedOutputScalar: string | null;
+    labelScalar: string | null;
+    categoryScalar: string | null;
+    indexValues: Record<string, string> | null;
+}
+export interface OffloadStagingOptions {
+    datasetId: string;
+    sampleCount: number;
+    batchSize: number;
+    fieldSchema: DatasetFieldSchemaDto[];
+    codec?: ObjectCodec;
+    readBatch: (offset: number, limit: number) => Promise<StagingSample[]>;
+    putShard: (name: string, body: Buffer) => Promise<StoredObjectRef>;
+    insertRows: (rows: DatasetSampleOffloadRow[]) => Promise<void>;
+}
+export declare function offloadStagingToShards(opts: OffloadStagingOptions): Promise<{
+    shards: number;
+    storagePrefix: string | null;
+}>;
+//# sourceMappingURL=dataset-sample-offload.d.ts.map

package/dist/server/modules/dataset/dataset-sample-offload.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"dataset-sample-offload.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-offload.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,gDAAgD,CAAC;AAInG,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE;QAAE,KAAK,EAAE,eAAe,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,qBAAqB,EAAE,CAAC;IACrC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACvE,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;IACnE,UAAU,EAAE,CAAC,IAAI,EAAE,uBAAuB,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAChE;AAED,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CA2C3D"}

package/dist/server/modules/dataset/dataset-sample-offload.js ADDED Viewed

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.offloadStagingToShards = offloadStagingToShards;
+const run_result_payload_1 = require("../run-result/run-result-payload");
+const dataset_sample_projection_1 = require("./dataset-sample-projection");
+async function offloadStagingToShards(opts) {
+    const codec = opts.codec ?? 'gzip';
+    let shardSeq = 0;
+    let offset = 0;
+    let firstShardKey = null;
+    while (offset < opts.sampleCount) {
+        const batch = await opts.readBatch(offset, opts.batchSize);
+        if (batch.length === 0)
+            break;
+        const body = await (0, run_result_payload_1.encodeShard)(batch.map((row) => row.data), codec);
+        const shardRef = await opts.putShard(`shard-${String(shardSeq).padStart(5, '0')}.jsonl.gz`, body);
+        firstShardKey ??= shardRef.key;
+        await opts.insertRows(batch.map((row, rowIndex) => {
+            const data = (row.data ?? null);
+            const projection = (0, dataset_sample_projection_1.projectDatasetSample)(data, opts.fieldSchema);
+            return {
+                datasetId: opts.datasetId,
+                data: null,
+                externalId: row.externalId,
+                payloadRef: { shard: shardRef, rowIndex },
+                searchPreview: projection.searchPreview,
+                expectedOutputScalar: projection.expectedOutputScalar,
+                labelScalar: projection.labelScalar,
+                categoryScalar: projection.categoryScalar,
+                indexValues: projection.indexValues,
+            };
+        }));
+        offset += batch.length;
+        shardSeq += 1;
+    }
+    return {
+        shards: shardSeq,
+        storagePrefix: firstShardKey ? firstShardKey.slice(0, firstShardKey.lastIndexOf('/') + 1) : null,
+    };
+}
+//# sourceMappingURL=dataset-sample-offload.js.map

package/dist/server/modules/dataset/dataset-sample-offload.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"dataset-sample-offload.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-offload.ts"],"names":[],"mappings":";;AAwCA,wDA6CC;AA5ED,yEAA+D;AAC/D,2EAAmE;AA8B5D,KAAK,UAAU,sBAAsB,CAC1C,IAA2B;IAE3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC;IACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,aAAa,GAAkB,IAAI,CAAC;IAExC,OAAO,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC3D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAE9B,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAW,EAC5B,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAC5B,KAAK,CACN,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAClG,aAAa,KAAK,QAAQ,CAAC,GAAG,CAAC;QAE/B,MAAM,IAAI,CAAC,UAAU,CACnB,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAmC,CAAC;YAClE,MAAM,UAAU,GAAG,IAAA,gDAAoB,EAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAChE,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,IAAI;gBACV,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,UAAU,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE;gBACzC,aAAa,EAAE,UAAU,CAAC,aAAa;gBACvC,oBAAoB,EAAE,UAAU,CAAC,oBAAoB;gBACrD,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,cAAc,EAAE,UAAU,CAAC,cAAc;gBACzC,WAAW,EAAE,UAAU,CAAC,WAAW;aACpC,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,QAAQ,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,OAAO;QACL,MAAM,EAAE,QAAQ;QAChB,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;KACjG,CAAC;AACJ,CAAC"}

package/dist/server/modules/dataset/dataset-sample-payload.d.ts ADDED Viewed

@@ -0,0 +1,21 @@
+import { ObjectStorageProvider, type StoredObjectRef } from '../../common/contracts/object-storage.provider';
+/** Stored in `dataset_samples.payload_ref`: the shard holding this sample's data + its line. */
+export interface DatasetSamplePayloadRef {
+    shard: StoredObjectRef;
+    rowIndex: number;
+}
+export interface DatasetSamplePayloadRow {
+    data: unknown;
+    payloadRef: DatasetSamplePayloadRef | null;
+}
+export declare class DatasetSamplePayloadReader {
+    private readonly storage;
+    constructor(storage: ObjectStorageProvider);
+    /** Resolve one sample's data: inline when present, else from its shard. */
+    hydrate(row: DatasetSamplePayloadRow): Promise<unknown>;
+    /** Batch: groups rows by shard so a 500-sample batch is one GET per shard, not one per sample. */
+    hydrateMany(rows: DatasetSamplePayloadRow[]): Promise<unknown[]>;
+    /** The ref to read from, or null when the inline value should be used (cache / no offload / disabled). */
+    private shardRef;
+}
+//# sourceMappingURL=dataset-sample-payload.d.ts.map

package/dist/server/modules/dataset/dataset-sample-payload.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"dataset-sample-payload.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-payload.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,qBAAqB,EAAE,KAAK,eAAe,EAAE,MAAM,gDAAgD,CAAC;AAG7G,gGAAgG;AAChG,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,uBAAuB,GAAG,IAAI,CAAC;CAC5C;AAED,qBACa,0BAA0B;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,qBAAqB;IAE3D,2EAA2E;IACrE,OAAO,CAAC,GAAG,EAAE,uBAAuB,GAAG,OAAO,CAAC,OAAO,CAAC;IAO7D,kGAAkG;IAC5F,WAAW,CAAC,IAAI,EAAE,uBAAuB,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAqBtE,0GAA0G;IAC1G,OAAO,CAAC,QAAQ;CAIjB"}

package/dist/server/modules/dataset/dataset-sample-payload.js ADDED Viewed

@@ -0,0 +1,72 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatasetSamplePayloadReader = void 0;
+// DatasetSamplePayloadReader — the read seam for dataset-sample content (SPEC 22 §7.3).
+//
+// The worker hot paths (experiment rendering, optimization rounds) load a sample's full `data`. Once
+// promote tiers it out, `data` is null and the authoritative content lives in an object-storage shard.
+// This seam returns the inline value when present (a small-sample cache, or no offload), else reads
+// the shard. Pure pass-through when storage is disabled / the row was never offloaded. It reuses the
+// generic JSONL shard codec from the run-result seam (one data object per line).
+const common_1 = require("@nestjs/common");
+const object_storage_provider_1 = require("../../common/contracts/object-storage.provider");
+const run_result_payload_1 = require("../run-result/run-result-payload");
+let DatasetSamplePayloadReader = class DatasetSamplePayloadReader {
+    constructor(storage) {
+        this.storage = storage;
+    }
+    /** Resolve one sample's data: inline when present, else from its shard. */
+    async hydrate(row) {
+        const ref = this.shardRef(row);
+        if (!ref)
+            return row.data ?? null;
+        const lines = await (0, run_result_payload_1.decodeShard)(await this.storage.getObject(ref.shard), ref.shard.codec);
+        return lines[ref.rowIndex] ?? null;
+    }
+    /** Batch: groups rows by shard so a 500-sample batch is one GET per shard, not one per sample. */
+    async hydrateMany(rows) {
+        const out = rows.map((r) => r.data ?? null);
+        const byShard = new Map();
+        rows.forEach((row, index) => {
+            const ref = this.shardRef(row);
+            if (!ref)
+                return;
+            const key = shardKey(ref.shard);
+            const group = byShard.get(key);
+            if (group)
+                group.entries.push({ index, rowIndex: ref.rowIndex });
+            else
+                byShard.set(key, { shard: ref.shard, entries: [{ index, rowIndex: ref.rowIndex }] });
+        });
+        await Promise.all([...byShard.values()].map(async ({ shard, entries }) => {
+            const lines = await (0, run_result_payload_1.decodeShard)(await this.storage.getObject(shard), shard.codec);
+            for (const { index, rowIndex } of entries)
+                out[index] = lines[rowIndex] ?? null;
+        }));
+        return out;
+    }
+    /** The ref to read from, or null when the inline value should be used (cache / no offload / disabled). */
+    shardRef(row) {
+        if (row.data != null || row.payloadRef == null || !this.storage.isEnabled())
+            return null;
+        return row.payloadRef;
+    }
+};
+exports.DatasetSamplePayloadReader = DatasetSamplePayloadReader;
+exports.DatasetSamplePayloadReader = DatasetSamplePayloadReader = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [object_storage_provider_1.ObjectStorageProvider])
+], DatasetSamplePayloadReader);
+function shardKey(ref) {
+    return `${ref.provider}:${ref.bucket ?? ''}:${ref.key}`;
+}
+//# sourceMappingURL=dataset-sample-payload.js.map

package/dist/server/modules/dataset/dataset-sample-payload.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"dataset-sample-payload.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-payload.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wFAAwF;AACxF,EAAE;AACF,qGAAqG;AACrG,uGAAuG;AACvG,oGAAoG;AACpG,qGAAqG;AACrG,iFAAiF;AACjF,2CAA4C;AAC5C,4FAA6G;AAC7G,yEAA+D;AAcxD,IAAM,0BAA0B,GAAhC,MAAM,0BAA0B;IACrC,YAA6B,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;IAAG,CAAC;IAE/D,2EAA2E;IAC3E,KAAK,CAAC,OAAO,CAAC,GAA4B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG;YAAE,OAAO,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,IAAA,gCAAW,EAA0B,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnH,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IAED,kGAAkG;IAClG,KAAK,CAAC,WAAW,CAAC,IAA+B;QAC/C,MAAM,GAAG,GAAc,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2F,CAAC;QACnH,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,OAAO;YACjB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,KAAK;gBAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;;gBAC5D,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE;YACrD,MAAM,KAAK,GAAG,MAAM,IAAA,gCAAW,EAA0B,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3G,KAAK,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,OAAO;gBAAE,GAAG,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QAClF,CAAC,CAAC,CACH,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,0GAA0G;IAClG,QAAQ,CAAC,GAA4B;QAC3C,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,IAAI,GAAG,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAAE,OAAO,IAAI,CAAC;QACzF,OAAO,GAAG,CAAC,UAAU,CAAC;IACxB,CAAC;CACF,CAAA;AAtCY,gEAA0B;qCAA1B,0BAA0B;IADtC,IAAA,mBAAU,GAAE;qCAE2B,+CAAqB;GADhD,0BAA0B,CAsCtC;AAED,SAAS,QAAQ,CAAC,GAAoB;IACpC,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;AAC1D,CAAC"}

package/dist/server/modules/dataset/dataset-sample-projection.d.ts ADDED Viewed

@@ -0,0 +1,11 @@
+import type { DatasetFieldSchemaDto } from '@proofhound/shared';
+export interface DatasetSampleProjection {
+    searchPreview: string | null;
+    expectedOutputScalar: string | null;
+    labelScalar: string | null;
+    categoryScalar: string | null;
+    indexValues: Record<string, string> | null;
+}
+/** Build the DB-side projection for one sample from its data + the dataset's field schema. */
+export declare function projectDatasetSample(data: Record<string, unknown> | null, fieldSchema: DatasetFieldSchemaDto[]): DatasetSampleProjection;
+//# sourceMappingURL=dataset-sample-projection.d.ts.map

package/dist/server/modules/dataset/dataset-sample-projection.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"dataset-sample-projection.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-projection.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEhE,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CAC5C;AAoBD,8FAA8F;AAC9F,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,EACpC,WAAW,EAAE,qBAAqB,EAAE,GACnC,uBAAuB,CAwBzB"}

package/dist/server/modules/dataset/dataset-sample-projection.js ADDED Viewed

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.projectDatasetSample = projectDatasetSample;
+const PREVIEW_MAX = 1000;
+const SCALAR_MAX = 200;
+const IMAGE_ROLES = new Set(['image', 'image_url', 'image_base64']);
+function asScalar(value) {
+    if (value === null || value === undefined)
+        return null;
+    if (typeof value === 'string')
+        return value.length > SCALAR_MAX ? null : value;
+    if (typeof value === 'number' || typeof value === 'boolean')
+        return String(value);
+    return null;
+}
+function previewOf(data) {
+    if (data === null || data === undefined)
+        return null;
+    const text = typeof data === 'string' ? data : JSON.stringify(data);
+    if (text.length === 0)
+        return null;
+    return text.length > PREVIEW_MAX ? text.slice(0, PREVIEW_MAX) : text;
+}
+/** Build the DB-side projection for one sample from its data + the dataset's field schema. */
+function projectDatasetSample(data, fieldSchema) {
+    if (data === null) {
+        return { searchPreview: null, expectedOutputScalar: null, labelScalar: null, categoryScalar: null, indexValues: null };
+    }
+    const expectedField = fieldSchema.find((f) => f.role === 'expected_output')?.name;
+    const expectedOutputScalar = expectedField ? asScalar(data[expectedField]) : null;
+    // index_values: short scalar values of the non-image fields, so distribution / filter on any
+    // configurable field works off `index_values->>field` once `data` is offloaded.
+    const indexValues = {};
+    for (const field of fieldSchema) {
+        if (IMAGE_ROLES.has(field.role))
+            continue;
+        const scalar = asScalar(data[field.name]);
+        if (scalar !== null)
+            indexValues[field.name] = scalar;
+    }
+    return {
+        searchPreview: previewOf(data),
+        expectedOutputScalar,
+        labelScalar: null,
+        categoryScalar: null,
+        indexValues: Object.keys(indexValues).length > 0 ? indexValues : null,
+    };
+}
+//# sourceMappingURL=dataset-sample-projection.js.map

package/dist/server/modules/dataset/dataset-sample-projection.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"dataset-sample-projection.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset-sample-projection.ts"],"names":[],"mappings":";;AAsCA,oDA2BC;AA9CD,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AAEpE,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,SAAS,CAAC,IAAa;IAC9B,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACrD,MAAM,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACpE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvE,CAAC;AAED,8FAA8F;AAC9F,SAAgB,oBAAoB,CAClC,IAAoC,EACpC,WAAoC;IAEpC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACzH,CAAC;IAED,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,iBAAiB,CAAC,EAAE,IAAI,CAAC;IAClF,MAAM,oBAAoB,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAElF,6FAA6F;IAC7F,gFAAgF;IAChF,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;QAChC,IAAI,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1C,IAAI,MAAM,KAAK,IAAI;YAAE,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;IACxD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,SAAS,CAAC,IAAI,CAAC;QAC9B,oBAAoB;QACpB,WAAW,EAAE,IAAI;QACjB,cAAc,EAAE,IAAI;QACpB,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI;KACtE,CAAC;AACJ,CAAC"}

package/dist/server/modules/dataset/dataset.module.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"dataset.module.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset.module.ts"],"names":[],"mappings":"~~AAUA~~,~~qBAYa~~,aAAa;CAAG"}
1	+ {"version":3,"file":"dataset.module.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset.module.ts"],"names":[],"mappings":"AAWA,qBAaa,aAAa;CAAG"}

package/dist/server/modules/dataset/dataset.module.js CHANGED Viewed

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatasetModule = void 0;
 const common_1 = require("@nestjs/common");
 const database_module_1 = require("../../../shared/database/database.module");
+const dataset_sample_payload_1 = require("./dataset-sample-payload");
 const dataset_deletion_hook_1 = require("./dataset-deletion.hook");
 const dataset_import_controller_1 = require("./dataset-import.controller");
 const dataset_import_repository_1 = require("./dataset-import.repository");
@@ -25,6 +26,7 @@ exports.DatasetModule = DatasetModule = __decorate([
         controllers: [dataset_controller_1.DatasetController, dataset_import_controller_1.DatasetImportController],
         providers: [
             dataset_repository_1.DatasetRepository,
+            dataset_sample_payload_1.DatasetSamplePayloadReader,
             { provide: dataset_deletion_hook_1.DatasetDeletionHook, useClass: dataset_deletion_hook_1.LocalDatasetDeletionHook },
             dataset_service_1.DatasetService,
             dataset_import_repository_1.DatasetImportRepository,

package/dist/server/modules/dataset/dataset.module.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"dataset.module.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,8EAA0E;AAC1E,mEAAwF;AACxF,2EAAsE;AACtE,2EAAsE;AACtE,qEAAgE;AAChE,6DAAyD;AACzD,6DAAyD;AACzD,uDAAmD;~~AAc5C~~,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,sCAAa;wBAAb,aAAa;~~IAZzB~~,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,gCAAc,CAAC;QACzB,WAAW,EAAE,CAAC,sCAAiB,EAAE,mDAAuB,CAAC;QACzD,SAAS,EAAE;YACT,sCAAiB;YACjB,EAAE,OAAO,EAAE,2CAAmB,EAAE,QAAQ,EAAE,gDAAwB,EAAE;YACpE,gCAAc;YACd,mDAAuB;YACvB,6CAAoB;SACrB;QACD,OAAO,EAAE,CAAC,gCAAc,EAAE,6CAAoB,CAAC;KAChD,CAAC;GACW,aAAa,CAAG"}
1	+ {"version":3,"file":"dataset.module.js","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,8EAA0E;AAC1E,qEAAsE;AACtE,mEAAwF;AACxF,2EAAsE;AACtE,2EAAsE;AACtE,qEAAgE;AAChE,6DAAyD;AACzD,6DAAyD;AACzD,uDAAmD;AAe5C,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,sCAAa;wBAAb,aAAa;IAbzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,gCAAc,CAAC;QACzB,WAAW,EAAE,CAAC,sCAAiB,EAAE,mDAAuB,CAAC;QACzD,SAAS,EAAE;YACT,sCAAiB;YACjB,mDAA0B;YAC1B,EAAE,OAAO,EAAE,2CAAmB,EAAE,QAAQ,EAAE,gDAAwB,EAAE;YACpE,gCAAc;YACd,mDAAuB;YACvB,6CAAoB;SACrB;QACD,OAAO,EAAE,CAAC,gCAAc,EAAE,6CAAoB,CAAC;KAChD,CAAC;GACW,aAAa,CAAG"}

package/dist/server/modules/dataset/dataset.repository.d.ts CHANGED Viewed

@@ -1,5 +1,7 @@
 import type { DbClient } from '@proofhound/db';
 import type { CreateDatasetDto, DatasetFieldSchemaDto } from '@proofhound/shared';
+import { ObjectStorageProvider } from '../../common/contracts/object-storage.provider';
+import { DatasetSamplePayloadReader } from './dataset-sample-payload';
 export interface DatasetProjectAccessRow {
     id: string;
 }
@@ -58,7 +60,10 @@ export interface DatasetDeletionImpactRows {
 }
 export declare class DatasetRepository {
     private readonly db;
-    constructor(db: DbClient);
+    private readonly sampleReader;
+    private readonly storage;
+    constructor(db: DbClient, sampleReader: DatasetSamplePayloadReader, storage: ObjectStorageProvider);
+    private hydrateSampleRows;
     private datasetSelectFields;
     findProjectAccess(_actorUserId: string, projectId: string, _isSuperAdmin: boolean): Promise<DatasetProjectAccessRow | null>;
     findDatasetByProjectAndName(projectId: string, name: string): Promise<DatasetRow | null>;

package/dist/server/modules/dataset/dataset.repository.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"dataset.repository.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset.repository.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;~~AAKlF~~,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,gBAAgB,CAAC;IACtB,WAAW,EAAE,qBAAqB,EAAE,CAAC;IACrC,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,wBAAwB,EAAE,CAAC;IACxC,aAAa,EAAE,wBAAwB,EAAE,CAAC;CAC3C;AAED,qBACa,iBAAiB;~~IACS~~,OAAO,CAAC,QAAQ,CAAC,EAAE;~~gBAAF~~,EAAE,EAAE,QAAQ;~~IAElE~~,OAAO,CAAC,mBAAmB,CAgBzB;IAEI,iBAAiB,CACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,OAAO,GACrB,OAAO,CAAC,uBAAuB,GAAG,IAAI,CAAC;IASpC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAUxF,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAQtD,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAUjF,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQnE,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQnE,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;~~IAUlE~~,sBAAsB,CAC1B,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,OAAO,CAAC;QAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;~~IAyBjD~~,6BAA6B,CACjC,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;~~IAkB7C~~,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuNxE,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC;IA4C5F,qBAAqB,CACzB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,yBAAyB,GAC9B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAavB,sBAAsB,CAC1B,UAAU,EAAE,MAAM,EAAE,GACnB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAuCjE,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAW1E,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5E,wBAAwB,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,UAAU,CAAC;~~IAoClF~~,OAAO,CAAC,aAAa;CAMtB"}
1	+ {"version":3,"file":"dataset.repository.d.ts","sourceRoot":"","sources":["../../../../src/server/modules/dataset/dataset.repository.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAElF,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AAEvF,OAAO,EAAgC,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAOpG,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,gBAAgB,CAAC;IACtB,WAAW,EAAE,qBAAqB,EAAE,CAAC;IACrC,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,wBAAwB,EAAE,CAAC;IACxC,aAAa,EAAE,wBAAwB,EAAE,CAAC;CAC3C;AAED,qBACa,iBAAiB;IAED,OAAO,CAAC,QAAQ,CAAC,EAAE;IAC5C,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAFkB,EAAE,EAAE,QAAQ,EACrC,YAAY,EAAE,0BAA0B,EACxC,OAAO,EAAE,qBAAqB;YAKnC,iBAAiB;IAU/B,OAAO,CAAC,mBAAmB,CAgBzB;IAEI,iBAAiB,CACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,OAAO,GACrB,OAAO,CAAC,uBAAuB,GAAG,IAAI,CAAC;IASpC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAUxF,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAQtD,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAUjF,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQnE,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQnE,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAWlE,sBAAsB,CAC1B,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,OAAO,CAAC;QAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IA6BjD,6BAA6B,CACjC,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAsB7C,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuNxE,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC;IA4C5F,qBAAqB,CACzB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,yBAAyB,GAC9B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAavB,sBAAsB,CAC1B,UAAU,EAAE,MAAM,EAAE,GACnB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAuCjE,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAW1E,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5E,wBAAwB,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,UAAU,CAAC;IAiElF,OAAO,CAAC,aAAa;CAMtB"}

package/dist/server/modules/dataset/dataset.repository.js CHANGED Viewed

@@ -17,10 +17,17 @@ const common_1 = require("@nestjs/common");
 const drizzle_orm_1 = require("drizzle-orm");
 const db_1 = require("@proofhound/db");
 const database_constants_1 = require("../../../shared/database/database.constants");
+const object_storage_provider_1 = require("../../common/contracts/object-storage.provider");
+const dataset_sample_offload_1 = require("./dataset-sample-offload");
+const dataset_sample_payload_1 = require("./dataset-sample-payload");
 const { optimizations, datasetSamples, datasets, experiments, projects, promptVersions } = db_1.schema;
+// Per-shard batch for small-file create offload (samples are already in memory; one shard per batch).
+const CREATE_SHARD_BATCH = 200;
 let DatasetRepository = class DatasetRepository {
-    constructor(db) {
+    constructor(db, sampleReader, storage) {
         this.db = db;
+        this.sampleReader = sampleReader;
+        this.storage = storage;
         this.datasetSelectFields = {
             id: datasets.id,
             projectId: datasets.projectId,
@@ -39,6 +46,15 @@ let DatasetRepository = class DatasetRepository {
             deletedAt: datasets.deletedAt,
         };
     }
+    // Resolve each row's `data` through the seam (inline when present, else from its shard) so callers
+    // that render full sample content keep working after a dataset is offloaded (SPEC 22 §7.3).
+    async hydrateSampleRows(rows) {
+        const hydrated = await this.sampleReader.hydrateMany(rows.map((r) => ({ data: r.data, payloadRef: r.payloadRef ?? null })));
+        rows.forEach((r, i) => {
+            r.data = hydrated[i] ?? null;
+        });
+        return rows;
+    }
     async findProjectAccess(_actorUserId, projectId, _isSuperAdmin) {
         const rows = await this.db
             .select({ id: projects.id })
@@ -85,18 +101,20 @@ let DatasetRepository = class DatasetRepository {
     }
     // Full scan — only for export (complete dump). Detail browsing must use listDatasetSamplesPage.
     async listDatasetSamples(datasetId) {
-        return this.db
+        const rows = await this.db
             .select()
             .from(datasetSamples)
             .where((0, drizzle_orm_1.eq)(datasetSamples.datasetId, datasetId))
             .orderBy((0, drizzle_orm_1.asc)(datasetSamples.createdAt), (0, drizzle_orm_1.asc)(datasetSamples.id));
+        return this.hydrateSampleRows(rows);
     }
     // Server-side paginated browse with optional cross-field search (data::text ILIKE), so the detail page
     // never loads an entire (potentially 100k+ sample) dataset into memory.
     async listDatasetSamplesPage(datasetId, options) {
         const searchTerm = options.search?.trim();
+        // Search matches inline data or, once a sample is offloaded, its search_preview (SPEC 22 §7.3).
         const where = searchTerm
-            ? (0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(datasetSamples.datasetId, datasetId), (0, drizzle_orm_1.sql) `${datasetSamples.data}::text ILIKE ${`%${searchTerm}%`}`)
+            ? (0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(datasetSamples.datasetId, datasetId), (0, drizzle_orm_1.sql) `(${datasetSamples.data}::text ILIKE ${`%${searchTerm}%`} OR ${datasetSamples.searchPreview} ILIKE ${`%${searchTerm}%`})`)
             : (0, drizzle_orm_1.eq)(datasetSamples.datasetId, datasetId);
         const [rows, countResult] = await Promise.all([
             this.db
@@ -111,16 +129,20 @@ let DatasetRepository = class DatasetRepository {
                 .from(datasetSamples)
                 .where(where),
         ]);
-        return { rows, total: Number(countResult[0]?.count ?? 0) };
+        return { rows: await this.hydrateSampleRows(rows), total: Number(countResult[0]?.count ?? 0) };
     }
     // SQL GROUP BY on the expected-output field so list/detail never load all sample rows into memory.
     // Mirrors DatasetService.toCategoryLabel: only scalar (string/number/boolean), non-blank, trimmed labels count.
     async aggregateCategoryDistribution(datasetId, fieldName) {
-        const label = (0, drizzle_orm_1.sql) `btrim(${datasetSamples.data} ->> ${fieldName})`;
+        // Read the field from inline data (scalar only), or from index_values once the sample is offloaded
+        // (index_values holds only short scalars by construction) (SPEC 22 §7.3).
+        const value = (0, drizzle_orm_1.sql) `COALESCE(${datasetSamples.data} ->> ${fieldName}, ${datasetSamples.indexValues} ->> ${fieldName})`;
+        const label = (0, drizzle_orm_1.sql) `btrim(${value})`;
         const rows = await this.db
             .select({ label, count: (0, drizzle_orm_1.sql) `count(*)::int` })
             .from(datasetSamples)
-            .where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(datasetSamples.datasetId, datasetId), (0, drizzle_orm_1.sql) `jsonb_typeof(${datasetSamples.data} -> ${fieldName}) IN ('string', 'number', 'boolean')`, (0, drizzle_orm_1.sql) `btrim(${datasetSamples.data} ->> ${fieldName}) <> ''`))
+            .where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(datasetSamples.datasetId, datasetId), (0, drizzle_orm_1.sql) `(jsonb_typeof(${datasetSamples.data} -> ${fieldName}) IN ('string', 'number', 'boolean')
+            OR (${datasetSamples.data} IS NULL AND ${datasetSamples.indexValues} ->> ${fieldName} IS NOT NULL))`, (0, drizzle_orm_1.sql) `btrim(${value}) <> ''`))
             // GROUP BY ordinal: the same ${fieldName} binds to different param positions in select vs group-by,
             // so Postgres won't match the expressions textually. Referencing select column 1 sidesteps that.
             .groupBy((0, drizzle_orm_1.sql) `1`);
@@ -448,11 +470,35 @@ let DatasetRepository = class DatasetRepository {
             if (!dataset) {
                 throw new Error('Dataset insert returned no row');
             }
-            await tx.insert(datasetSamples).values(args.dto.samples.map((sample) => ({
-                datasetId: dataset.id,
-                data: sample,
-                externalId: this.getExternalId(sample, args.externalIdFieldName),
-            })));
+            if (this.storage.isEnabled()) {
+                // Small-file create mirrors offload-at-promote (SPEC 22 §7.2): the samples are already in
+                // memory, so the batch reader just slices them. Object storage off → the inline insert below.
+                const samples = args.dto.samples;
+                const { storagePrefix } = await (0, dataset_sample_offload_1.offloadStagingToShards)({
+                    datasetId: dataset.id,
+                    sampleCount: samples.length,
+                    batchSize: CREATE_SHARD_BATCH,
+                    fieldSchema: args.fieldSchema,
+                    readBatch: async (offset, limit) => samples.slice(offset, offset + limit).map((sample) => ({
+                        data: sample,
+                        externalId: this.getExternalId(sample, args.externalIdFieldName),
+                    })),
+                    putShard: (name, body) => this.storage.putObject({ project: { projectId: args.projectId, source: 'local' }, resourceType: 'dataset_normalized', resourceId: dataset.id, name }, body, { codec: 'gzip' }),
+                    insertRows: async (rows) => {
+                        await tx.insert(datasetSamples).values(rows);
+                    },
+                });
+                if (storagePrefix) {
+                    await tx.update(datasets).set({ storagePrefix }).where((0, drizzle_orm_1.eq)(datasets.id, dataset.id));
+                }
+            }
+            else {
+                await tx.insert(datasetSamples).values(args.dto.samples.map((sample) => ({
+                    datasetId: dataset.id,
+                    data: sample,
+                    externalId: this.getExternalId(sample, args.externalIdFieldName),
+                })));
+            }
             return {
                 ...dataset,
                 createdByDisplayName: null,
@@ -472,6 +518,7 @@ exports.DatasetRepository = DatasetRepository;
 exports.DatasetRepository = DatasetRepository = __decorate([
     (0, common_1.Injectable)(),
     __param(0, (0, common_1.Inject)(database_constants_1.DATABASE_CLIENT)),
-    __metadata("design:paramtypes", [Object])
+    __metadata("design:paramtypes", [Object, dataset_sample_payload_1.DatasetSamplePayloadReader,
+        object_storage_provider_1.ObjectStorageProvider])
 ], DatasetRepository);
 //# sourceMappingURL=dataset.repository.js.map