deepline 0.1.226 → 0.1.227

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -2539,6 +2539,7 @@ type WorkerStepProgram = {
 
 type WorkerMapOptions = {
   description?: string;
+  mode?: 'upsert' | 'net_new';
   concurrency?: number;
   key?:
     | string
@@ -4870,9 +4871,18 @@ function createMinimalWorkerCtx(
             runId: req.runId,
             inputOffset: 0,
             attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
+            ...(opts?.mode ? { mode: opts.mode } : {}),
             ...mapWriteAttempt,
           });
     const mapWriteVersion = mapReservation?.writeVersion ?? null;
+    const netNewRowKeys =
+      opts?.mode === 'net_new'
+        ? new Set(
+            (mapReservation?.pendingRows ?? [])
+              .map((row) => resolveMapRowOutcomeKey(row))
+              .filter((key): key is string => Boolean(key)),
+          )
+        : null;
 
     let totalRowsWritten = 0;
     let datasetLifecycleRegistered = false;
@@ -4896,7 +4906,7 @@ function createMinimalWorkerCtx(
         chunkIndex,
       });
       const keyStartedAt = nowMs();
-      const chunkEntries = chunkRows.map((row, localIndex) => {
+      let chunkEntries = chunkRows.map((row, localIndex) => {
         const absoluteIndex = baseOffset + chunkStart + localIndex;
         const rowKey = resolveRowKey(
           row,
@@ -4906,6 +4916,11 @@ function createMinimalWorkerCtx(
         );
         return { row, absoluteIndex, rowKey };
       });
+      if (netNewRowKeys) {
+        chunkEntries = chunkEntries.filter(({ rowKey }) =>
+          netNewRowKeys.has(rowKey),
+        );
+      }
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.keys',
@@ -4968,7 +4983,10 @@ function createMinimalWorkerCtx(
         },
       });
       mapLatencyProfile.recordPhase('prepare_rows', nowMs() - prepareStartedAt);
-      const progressTotalRows = rowCountHint ?? chunkRows.length;
+      // net_new reserves the entire source before chunk execution, so its
+      // progress describes the admitted subset rather than the source CSV.
+      const progressTotalRows =
+        netNewRowKeys?.size ?? rowCountHint ?? chunkRows.length;
       const preparedCompletedRows = Math.min(
         progressTotalRows,
         totalRowsWritten,
@@ -5023,15 +5041,8 @@ function createMinimalWorkerCtx(
         ({ rowKey }) => !preparedKeys.has(rowKey),
       );
       if (missingPreparedRows.length > 0) {
-        const rowKeySamples = missingPreparedRows
-          .map(({ rowKey }) => rowKey)
-          .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
         throw new Error(
-          `ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
-            `Every input row must be claimed, completed, or blocked before execution starts.` +
-            (rowKeySamples.length > 0
-              ? ` Missing row keys: ${rowKeySamples.join(', ')}`
-              : ''),
+          `ctx.dataset("${name}") preparation omitted ${missingPreparedRows.length} row disposition(s).`,
         );
       }
       const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
@@ -5986,9 +5997,8 @@ function createMinimalWorkerCtx(
         // dataset (their recoverable state lives in the runtime sheet).
         if (key && executedRow) resultByKey.set(key, executedRow);
       }
-      const outEntries = chunkRows
-        .map((_row, index) => {
-          const entry = chunkEntries[index]!;
+      const outEntries = chunkEntries
+        .map((entry) => {
           const row = resultByKey.get(entry.rowKey);
           return row
             ? {
@@ -6531,11 +6541,11 @@ function createMinimalWorkerCtx(
           }
           enqueueMapProgressUpdate({
             completed: totalRowsWritten,
-            total: rowCountHint ?? undefined,
+            total: netNewRowKeys?.size ?? rowCountHint ?? undefined,
             ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
             message: formatMapProgressMessage(
               totalRowsWritten,
-              rowCountHint ?? undefined,
+              netNewRowKeys?.size ?? rowCountHint ?? undefined,
             ),
           });
           if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {

package/dist/bundling-sources/sdk/src/play.ts

@@ -560,6 +560,11 @@ export type DatasetBuilder<
    */
   run(options?: {
     description?: string;
+    /**
+     * `upsert` (default) returns every input row, reusing persisted work for
+     * existing keys. `net_new` atomically inserts and returns only unseen keys.
+     */
+    mode?: 'upsert' | 'net_new';
     key?:
       | (keyof InputRow & string)
       | readonly (keyof InputRow & string)[]

package/dist/bundling-sources/sdk/src/release.ts

@@ -108,10 +108,10 @@ export const SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: '0.1.226',
+  version: '0.1.227',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.226',
+    latest: '0.1.227',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.219',
     commandMinimumSupported: [

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -3943,7 +3943,10 @@ export class PlayContextImpl {
         this.#options.runId,
       );
     if (shouldStreamRuntimeBackedDataset) {
-      totalInputCount = await items.count();
+      // Net-new admission happens one source page at a time. Its progress and
+      // returned dataset describe admitted rows, not every provider/CSV row
+      // supplied to this run, so accumulate the admitted count below.
+      totalInputCount = options?.mode === 'net_new' ? 0 : await items.count();
       const mapScope = this.createMapExecutionScope({
         logicalNamespace: normalizedMapNamespace,
         artifactTableNamespace: resolvedTableNamespace,
@@ -4097,6 +4100,7 @@ export class PlayContextImpl {
             staticPipeline: this.currentStaticPipeline,
             forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true,
             inputOffset: page.offset,
+            mode: options?.mode,
           },
         );
         resolvedTableNamespace = normalizeTableNamespace(
@@ -4140,12 +4144,21 @@ export class PlayContextImpl {
             index: originalIndex,
           };
         });
+        const admittedSeededItems =
+          options?.mode === 'net_new'
+            ? seededItems.filter(({ row, index }) =>
+                pendingRowsByKey.has(seedRowIdentity(row, index)),
+              )
+            : seededItems;
+        if (options?.mode === 'net_new') {
+          totalInputCount += admittedSeededItems.length;
+        }
 
         const rowsToExecuteByKey = new Map<
           string,
           { row: Record<string, unknown>; originalIndex: number }
         >();
-        for (const { row, index } of seededItems) {
+        for (const { row, index } of admittedSeededItems) {
           if (explicitKeyResolver) {
             const explicitKey = explicitKeyResolver(row, index);
             if (seenExplicitKeys.has(explicitKey)) {
@@ -4280,6 +4293,45 @@ export class PlayContextImpl {
         page.rows.length = 0;
       }
 
+      // Isolated failures are a partial result only when at least one admitted
+      // row completed. A fully failed admission run is systemic: preserve the
+      // failed sheet rows for recovery, but fail the enclosing play loudly.
+      if (totalInputCount > 0 && failedCount === totalInputCount) {
+        const completedRows = 0;
+        this.setMapFrame({
+          mapInvocationId: mapScope.mapInvocationId,
+          mapNodeId: mapScope.mapNodeId ?? null,
+          logicalNamespace: mapScope.logicalNamespace,
+          artifactTableNamespace: resolvedTableNamespace,
+          status: 'failed',
+          totalRows: totalInputCount,
+          completedRowKeys: [],
+          pendingRowKeys: [],
+          completedRowsCount: completedRows,
+          pendingRowsCount: 0,
+          failedRowsCount: failedCount,
+          startedAt:
+            this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
+            Date.now(),
+          updatedAt: Date.now(),
+        });
+        this.emitExecutionEvent({
+          type: 'map.failed',
+          mapInvocationId: mapScope.mapInvocationId,
+          mapNodeId: mapScope.mapNodeId ?? null,
+          logicalNamespace: mapScope.logicalNamespace,
+          artifactTableNamespace: resolvedTableNamespace,
+          completedRows,
+          failedRows: failedCount,
+          totalRows: totalInputCount,
+          at: Date.now(),
+        });
+        throw new Error(
+          `ctx.dataset("${normalizedMapNamespace}") failed for every admitted row (${failedCount}/${totalInputCount}). ` +
+            'Failed rows were persisted and will be retried on the next run.',
+        );
+      }
+
       this.setMapFrame({
         mapInvocationId: mapScope.mapInvocationId,
         mapNodeId: mapScope.mapNodeId ?? null,
@@ -4531,6 +4583,7 @@ export class PlayContextImpl {
           executorToken: this.#options.executorToken,
           staticPipeline: this.currentStaticPipeline,
           forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true,
+          mode: options?.mode,
         },
       );
       resolvedTableNamespace = normalizeTableNamespace(
@@ -4587,8 +4640,16 @@ export class PlayContextImpl {
             index,
           };
         });
-      itemsToProcess = seededItems.map((item) => item.row);
-      itemOriginalIndexes = seededItems.map((item) => item.index);
+      const admittedItems =
+        options?.mode === 'net_new'
+          ? seededItems.filter(({ row, index }) =>
+              pendingRowsByKey.has(seedRowIdentity(row, index)),
+            )
+          : seededItems;
+      rawItems = admittedItems.map((item) => item.row);
+      itemsToProcess = admittedItems.map((item) => item.row);
+      itemOriginalIndexes = admittedItems.map((item) => item.index);
+      totalInputCount = admittedItems.length;
     }
 
     const mapScope = this.createMapExecutionScope({

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -248,6 +248,16 @@ export interface DatasetOptions<TItem = Record<string, unknown>> {
    * and are reused on re-run.
    */
   onRowError?: 'isolate' | 'fail';
+  /**
+   * Controls how a persisted dataset admits rows whose key already exists.
+   *
+   * `upsert` (the default) preserves the row-preserving enrichment contract:
+   * existing rows are reused and returned with the current input. `net_new`
+   * atomically inserts only previously unseen keys and returns/processes only
+   * those inserted rows. Use `net_new` for a durable sourcing table, not for
+   * ordinary CSV enrichment reruns.
+   */
+  mode?: 'upsert' | 'net_new';
 }
 
 export type DatasetDefinitionOptions<TItem = Record<string, unknown>> = Omit<
@@ -257,7 +267,7 @@ export type DatasetDefinitionOptions<TItem = Record<string, unknown>> = Omit<
 
 export type DatasetRunOptions<TItem = Record<string, unknown>> = Pick<
   DatasetOptions<TItem>,
-  'description' | 'key' | 'onRowError'
+  'description' | 'key' | 'onRowError' | 'mode'
 >;
 
 export interface ToolCallOptions {
@@ -597,6 +607,7 @@ export interface ContextOptions {
       staticPipeline?: PlayStaticPipeline | null;
       forceRefresh?: boolean;
       inputOffset?: number;
+      mode?: DatasetRunOptions['mode'];
     },
   ) => Promise<MapStartResult>;
   /**

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -273,6 +273,8 @@ type RuntimeSheetPreparedRowDisposition = {
   key: string;
   disposition: RuntimeSheetPrepareDisposition;
 };
+
+type RuntimeSheetDatasetMode = 'upsert' | 'net_new';
 const dbSessionCache = new Map<string, DbSessionCacheEntry>();
 const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
 const postgresPools = new Map<string, RuntimePool>();
@@ -2976,6 +2978,7 @@ async function prepareRuntimeSheetDatasetRows(
     physicalUpsertSetSql: string;
     outputPhysicalColumns: PhysicalSheetColumnProjection[];
     force?: boolean;
+    mode?: RuntimeSheetDatasetMode;
   },
 ): Promise<{
   inserted: number;
@@ -3378,6 +3381,121 @@ async function prepareRuntimeSheetDatasetRows(
   return { inserted, rowDispositions };
 }
 
+/**
+ * Atomically admit source-table keys that have not completed. A completed row
+ * is consumed forever; failed rows and rows owned by this logical run remain
+ * recoverable. This is deliberately a separate insert-only path: deriving
+ * "new" from a read before the normal insert would race a concurrent sourcing
+ * run.
+ */
+async function prepareNetNewRuntimeSheetDatasetRows(
+  client: RuntimeQueryClient,
+  session: RuntimePostgresSession,
+  input: {
+    chunks: RuntimeDatasetRowEntry[][];
+    runId: string;
+    attemptId: string;
+    attemptOwnerRunId: string;
+    attemptExpiresAt: string;
+    attemptSeq: number;
+    writeVersion: number;
+    physicalInsertColumnsSql: string;
+    physicalInsertValuesSql: string;
+  },
+): Promise<{
+  inserted: number;
+  rowDispositions: RuntimeSheetPreparedRowDisposition[];
+}> {
+  let inserted = 0;
+  const rowDispositions: RuntimeSheetPreparedRowDisposition[] = [];
+  for (const chunk of input.chunks) {
+    const { rows } = await client.query<{
+      inserted_count: number;
+      row_dispositions: RuntimeSheetPreparedRowDisposition[];
+    }>(
+      `WITH input_rows AS (
+         SELECT DISTINCT ON (key_values._key)
+                key_values._key, payload_values.payload, index_values._input_index
+         FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
+         JOIN unnest($2::jsonb[]) WITH ORDINALITY AS payload_values(payload, ord)
+           ON payload_values.ord = key_values.ord
+         JOIN unnest($3::bigint[]) WITH ORDINALITY AS index_values(_input_index, ord)
+           ON index_values.ord = key_values.ord
+         ORDER BY key_values._key, key_values.ord
+       ),
+       reclaimed_rows AS (
+         UPDATE ${sheetTable(session)} AS target
+            SET _status = 'pending',
+                _run_id = $4::text,
+                _input_index = input_rows._input_index,
+                _attempt_id = $5::text,
+                _attempt_owner_run_id = $6::text,
+                _attempt_expires_at = $7::timestamptz,
+                _attempt_seq = $8::integer,
+                _write_version = $9::bigint,
+                _writer_run_id = $4::text,
+                _error = NULL,
+                _updated_at = now(),
+                _version = ${nextRuntimeSheetVersionExpression(session)}
+           FROM input_rows
+          WHERE target._key = input_rows._key
+            AND (
+              target._status = 'failed'
+              OR (
+                target._status IN ('pending', 'running')
+                AND coalesce(target._attempt_owner_run_id, target._run_id) = $6::text
+              )
+            )
+         RETURNING target._key
+       ),
+       inserted_rows AS (
+         INSERT INTO ${sheetTable(session)}
+         (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id,
+          _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id${input.physicalInsertColumnsSql})
+       SELECT _key, 'pending', $4::text, _input_index, $5::text, $6::text,
+              $7::timestamptz, $8::integer, $9::bigint, $4::text${input.physicalInsertValuesSql}
+       FROM input_rows
+       ON CONFLICT (_key) DO NOTHING
+       RETURNING _key
+       ),
+       pending_rows AS (
+         SELECT _key FROM inserted_rows
+         UNION
+         SELECT _key FROM reclaimed_rows
+       )
+       SELECT
+         (SELECT count(*)::int FROM inserted_rows) AS inserted_count,
+         coalesce(
+           (SELECT jsonb_agg(jsonb_build_object('key', _key, 'disposition', 'pending')) FROM pending_rows),
+           '[]'::jsonb
+         ) AS row_dispositions`,
+      [
+        chunk.map((entry) => entry.key),
+        chunk.map((entry) => stringifyPostgresJson(entry.row)),
+        chunk.map((entry) => entry.inputIndex),
+        input.runId,
+        input.attemptId,
+        input.attemptOwnerRunId,
+        input.attemptExpiresAt,
+        input.attemptSeq,
+        input.writeVersion,
+      ],
+    );
+    inserted += Number(rows[0]?.inserted_count ?? 0);
+    if (Array.isArray(rows[0]?.row_dispositions)) {
+      rowDispositions.push(
+        ...rows[0].row_dispositions.filter(
+          (row): row is RuntimeSheetPreparedRowDisposition =>
+            row != null &&
+            typeof row.key === 'string' &&
+            row.disposition === 'pending',
+        ),
+      );
+    }
+  }
+  return { inserted, rowDispositions };
+}
+
 async function buildRuntimeSheetDatasetStartResult(
   client: RuntimeQueryClient,
   session: RuntimePostgresSession,
@@ -5561,6 +5679,7 @@ export async function startRuntimeSheetDataset(
     writeVersion?: number | null;
     inputOffset?: number;
     force?: boolean;
+    mode?: RuntimeSheetDatasetMode;
   },
 ): Promise<PrepareRuntimeSheetResult> {
   if (context.dbSessionStrategy === 'gateway_only') {
@@ -5737,24 +5856,37 @@ export async function startRuntimeSheetDataset(
         rows: rowEntries.length,
       });
       const prepareStartedAt = Date.now();
-      const prepared = await prepareRuntimeSheetDatasetRows(client, session, {
-        chunks,
-        runId: input.runId,
-        normalizedPlayName,
-        normalizedTableNamespace,
-        attemptId,
-        attemptOwnerRunId,
-        attemptExpiresAt,
-        attemptSeq,
-        writeVersion,
-        physicalInsertColumnsSql,
-        physicalInsertValuesSql,
-        physicalRefreshSetSql:
-          input.force === true ? physicalRefreshSetSql : '',
-        physicalUpsertSetSql,
-        outputPhysicalColumns,
-        force: input.force === true,
-      });
+      const prepared =
+        input.mode === 'net_new'
+          ? await prepareNetNewRuntimeSheetDatasetRows(client, session, {
+              chunks,
+              runId: input.runId,
+              attemptId,
+              attemptOwnerRunId,
+              attemptExpiresAt,
+              attemptSeq,
+              writeVersion,
+              physicalInsertColumnsSql,
+              physicalInsertValuesSql,
+            })
+          : await prepareRuntimeSheetDatasetRows(client, session, {
+              chunks,
+              runId: input.runId,
+              normalizedPlayName,
+              normalizedTableNamespace,
+              attemptId,
+              attemptOwnerRunId,
+              attemptExpiresAt,
+              attemptSeq,
+              writeVersion,
+              physicalInsertColumnsSql,
+              physicalInsertValuesSql,
+              physicalRefreshSetSql:
+                input.force === true ? physicalRefreshSetSql : '',
+              physicalUpsertSetSql,
+              outputPhysicalColumns,
+              force: input.force === true,
+            });
       timings.push({
         phase: 'prepare_rows_sql',
         ms: Date.now() - prepareStartedAt,

package/dist/cli/index.js

@@ -625,10 +625,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.226",
+  version: "0.1.227",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.226",
+    latest: "0.1.227",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/dist/cli/index.mjs

@@ -610,10 +610,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.226",
+  version: "0.1.227",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.226",
+    latest: "0.1.227",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/dist/index.d.mts

@@ -3426,6 +3426,11 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
      */
     run(options?: {
         description?: string;
+        /**
+         * `upsert` (default) returns every input row, reusing persisted work for
+         * existing keys. `net_new` atomically inserts and returns only unseen keys.
+         */
+        mode?: 'upsert' | 'net_new';
         key?: (keyof InputRow & string) | readonly (keyof InputRow & string)[] | ((row: InputRow, index: number) => string | number | readonly unknown[]);
     }): Promise<PlayDataset<OutputRow>>;
 };

package/dist/index.d.ts

@@ -3426,6 +3426,11 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
      */
     run(options?: {
         description?: string;
+        /**
+         * `upsert` (default) returns every input row, reusing persisted work for
+         * existing keys. `net_new` atomically inserts and returns only unseen keys.
+         */
+        mode?: 'upsert' | 'net_new';
         key?: (keyof InputRow & string) | readonly (keyof InputRow & string)[] | ((row: InputRow, index: number) => string | number | readonly unknown[]);
     }): Promise<PlayDataset<OutputRow>>;
 };

package/dist/index.js

@@ -424,10 +424,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.226",
+  version: "0.1.227",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.226",
+    latest: "0.1.227",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/dist/index.mjs

@@ -354,10 +354,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.226",
+  version: "0.1.227",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.226",
+    latest: "0.1.227",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.226",
+  "version": "0.1.227",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {