deepline 0.1.226 → 0.1.228

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -140,6 +140,11 @@ import type {
   PlayRunLedgerStepProgress,
   PlayRunLedgerStepStatus,
 } from '../../../shared_libs/play-runtime/run-ledger';
+import {
+  MAX_LEDGER_LOG_LINE_LENGTH,
+  MAX_LEDGER_LOG_LINES_PER_EVENT,
+} from '../../../shared_libs/play-runtime/ledger-safe-payload';
+import { partitionWorkerLedgerEvents } from './runtime/ledger-event-batches';
 import type { PlayHarnessRpc } from '../../play-harness-worker/src/rpc-types';
 import {
   createCsvDatasetHandle,
@@ -2539,6 +2544,7 @@ type WorkerStepProgram = {
 
 type WorkerMapOptions = {
   description?: string;
+  mode?: 'upsert' | 'net_new';
   concurrency?: number;
   key?:
     | string
@@ -4870,9 +4876,18 @@ function createMinimalWorkerCtx(
             runId: req.runId,
             inputOffset: 0,
             attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
+            ...(opts?.mode ? { mode: opts.mode } : {}),
             ...mapWriteAttempt,
           });
     const mapWriteVersion = mapReservation?.writeVersion ?? null;
+    const netNewRowKeys =
+      opts?.mode === 'net_new'
+        ? new Set(
+            (mapReservation?.pendingRows ?? [])
+              .map((row) => resolveMapRowOutcomeKey(row))
+              .filter((key): key is string => Boolean(key)),
+          )
+        : null;
 
     let totalRowsWritten = 0;
     let datasetLifecycleRegistered = false;
@@ -4896,7 +4911,7 @@ function createMinimalWorkerCtx(
         chunkIndex,
       });
       const keyStartedAt = nowMs();
-      const chunkEntries = chunkRows.map((row, localIndex) => {
+      let chunkEntries = chunkRows.map((row, localIndex) => {
         const absoluteIndex = baseOffset + chunkStart + localIndex;
         const rowKey = resolveRowKey(
           row,
@@ -4906,6 +4921,11 @@ function createMinimalWorkerCtx(
         );
         return { row, absoluteIndex, rowKey };
       });
+      if (netNewRowKeys) {
+        chunkEntries = chunkEntries.filter(({ rowKey }) =>
+          netNewRowKeys.has(rowKey),
+        );
+      }
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.keys',
@@ -4968,7 +4988,10 @@ function createMinimalWorkerCtx(
         },
       });
       mapLatencyProfile.recordPhase('prepare_rows', nowMs() - prepareStartedAt);
-      const progressTotalRows = rowCountHint ?? chunkRows.length;
+      // net_new reserves the entire source before chunk execution, so its
+      // progress describes the admitted subset rather than the source CSV.
+      const progressTotalRows =
+        netNewRowKeys?.size ?? rowCountHint ?? chunkRows.length;
       const preparedCompletedRows = Math.min(
         progressTotalRows,
         totalRowsWritten,
@@ -5023,15 +5046,8 @@ function createMinimalWorkerCtx(
         ({ rowKey }) => !preparedKeys.has(rowKey),
       );
       if (missingPreparedRows.length > 0) {
-        const rowKeySamples = missingPreparedRows
-          .map(({ rowKey }) => rowKey)
-          .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
         throw new Error(
-          `ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
-            `Every input row must be claimed, completed, or blocked before execution starts.` +
-            (rowKeySamples.length > 0
-              ? ` Missing row keys: ${rowKeySamples.join(', ')}`
-              : ''),
+          `ctx.dataset("${name}") preparation omitted ${missingPreparedRows.length} row disposition(s).`,
         );
       }
       const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
@@ -5986,9 +6002,8 @@ function createMinimalWorkerCtx(
         // dataset (their recoverable state lives in the runtime sheet).
         if (key && executedRow) resultByKey.set(key, executedRow);
       }
-      const outEntries = chunkRows
-        .map((_row, index) => {
-          const entry = chunkEntries[index]!;
+      const outEntries = chunkEntries
+        .map((entry) => {
           const row = resultByKey.get(entry.rowKey);
           return row
             ? {
@@ -6531,11 +6546,11 @@ function createMinimalWorkerCtx(
           }
           enqueueMapProgressUpdate({
             completed: totalRowsWritten,
-            total: rowCountHint ?? undefined,
+            total: netNewRowKeys?.size ?? rowCountHint ?? undefined,
             ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
             message: formatMapProgressMessage(
               totalRowsWritten,
-              rowCountHint ?? undefined,
+              netNewRowKeys?.size ?? rowCountHint ?? undefined,
             ),
           });
           if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
@@ -7768,7 +7783,10 @@ async function executeRunRequest(
   let ledgerFlushQueueDepth = 0;
 
   const appendRunLogLine = (line: string) => {
-    const trimmed = redactSecretsFromLogString(line.trim());
+    const trimmed = redactSecretsFromLogString(line.trim()).slice(
+      0,
+      MAX_LEDGER_LOG_LINE_LENGTH,
+    );
     if (!trimmed) return;
     workBudgetMeter.count('log');
     totalEmittedLogLines += 1;
@@ -7851,17 +7869,27 @@ async function executeRunRequest(
     pendingLedgerEvents = [];
 
     if (pendingRunLogLines.length > 0) {
-      events.push({
-        type: 'log.appended',
-        runId: req.runId,
-        source: 'worker',
-        occurredAt,
-        lines: pendingRunLogLines,
-        // Positional cursor: pendingRunLogLines always holds the LAST
-        // pending lines emitted on this channel, so the offset of its first
-        // line is total-emitted minus pending length.
-        channelOffset: totalEmittedLogLines - pendingRunLogLines.length,
-      });
+      const channelOffset = totalEmittedLogLines - pendingRunLogLines.length;
+      for (
+        let start = 0;
+        start < pendingRunLogLines.length;
+        start += MAX_LEDGER_LOG_LINES_PER_EVENT
+      ) {
+        events.push({
+          type: 'log.appended',
+          runId: req.runId,
+          source: 'worker',
+          occurredAt,
+          lines: pendingRunLogLines.slice(
+            start,
+            start + MAX_LEDGER_LOG_LINES_PER_EVENT,
+          ),
+          // Positional cursor: pendingRunLogLines always holds the LAST
+          // pending lines emitted on this channel, so the offset of its first
+          // line is total-emitted minus pending length.
+          channelOffset: channelOffset + start,
+        });
+      }
       pendingRunLogLines = [];
     }
 
@@ -7946,19 +7974,26 @@ async function executeRunRequest(
     ledgerFlushInFlight = ledgerFlushInFlight
       .catch(() => undefined)
       .then(async () => {
-        try {
-          await postRuntimeApi(req.baseUrl, req.executorToken, {
-            action: 'append_run_events',
-            playId: req.runId,
-            events,
-          });
-        } catch {
-          pendingLedgerEvents = [...events, ...pendingLedgerEvents];
-          throw new Error('runtime run-ledger append failed');
-        } finally {
-          ledgerFlushQueueDepth = Math.max(0, ledgerFlushQueueDepth - 1);
+        const batches = partitionWorkerLedgerEvents(events);
+        for (let index = 0; index < batches.length; index += 1) {
+          try {
+            await postRuntimeApi(req.baseUrl, req.executorToken, {
+              action: 'append_run_events',
+              playId: req.runId,
+              events: batches[index]!,
+            });
+          } catch {
+            pendingLedgerEvents = [
+              ...batches.slice(index).flat(),
+              ...pendingLedgerEvents,
+            ];
+            throw new Error('runtime run-ledger append failed');
+          }
         }
       })
+      .finally(() => {
+        ledgerFlushQueueDepth = Math.max(0, ledgerFlushQueueDepth - 1);
+      })
       .catch(() => undefined);
     return force ? ledgerFlushInFlight : Promise.resolve();
   };
@@ -7978,15 +8013,21 @@ async function executeRunRequest(
     ]);
     const events = [...drainPendingLedgerEvents(now), terminalEvent];
     if (events.length === 0) return;
-    try {
-      await postRuntimeApi(req.baseUrl, req.executorToken, {
-        action: 'append_run_events',
-        playId: req.runId,
-        events,
-      });
-    } catch (error) {
-      pendingLedgerEvents = [...events, ...pendingLedgerEvents];
-      throw error;
+    const batches = partitionWorkerLedgerEvents(events);
+    for (let index = 0; index < batches.length; index += 1) {
+      try {
+        await postRuntimeApi(req.baseUrl, req.executorToken, {
+          action: 'append_run_events',
+          playId: req.runId,
+          events: batches[index]!,
+        });
+      } catch (error) {
+        pendingLedgerEvents = [
+          ...batches.slice(index).flat(),
+          ...pendingLedgerEvents,
+        ];
+        throw error;
+      }
     }
   };
 

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts

@@ -0,0 +1,81 @@
+import {
+  MAX_LEDGER_LOG_LINE_LENGTH,
+  MAX_LEDGER_LOG_LINES_PER_EVENT,
+} from '../../../../shared_libs/play-runtime/ledger-safe-payload';
+import type { PlayRunLedgerEvent } from '../../../../shared_libs/play-runtime/run-ledger';
+
+/** Keep a Worker-to-runtime append comfortably below the API and Convex limits. */
+export const WORKER_LEDGER_EVENT_BATCH_MAX_BYTES = 128 * 1024;
+export const WORKER_LEDGER_EVENTS_PER_APPEND_MAX = 100;
+
+const encoder = new TextEncoder();
+
+function encodedJsonBytes(value: unknown): number {
+  return encoder.encode(JSON.stringify(value)).length;
+}
+
+function splitLogEvent(
+  event: Extract<PlayRunLedgerEvent, { type: 'log.appended' }>,
+): PlayRunLedgerEvent[] {
+  const events: PlayRunLedgerEvent[] = [];
+  const lines = event.lines.map((line) =>
+    line.slice(0, MAX_LEDGER_LOG_LINE_LENGTH),
+  );
+  let start = 0;
+  while (start < lines.length) {
+    let end = start;
+    let bytes = encodedJsonBytes({ ...event, lines: [] });
+    while (end < lines.length && end - start < MAX_LEDGER_LOG_LINES_PER_EVENT) {
+      const lineBytes = encodedJsonBytes(lines[end]!);
+      if (
+        end > start &&
+        bytes + lineBytes + 1 > WORKER_LEDGER_EVENT_BATCH_MAX_BYTES
+      ) {
+        break;
+      }
+      bytes += lineBytes + 1;
+      end += 1;
+    }
+    events.push({
+      ...event,
+      lines: lines.slice(start, end),
+      ...(typeof event.channelOffset === 'number'
+        ? { channelOffset: event.channelOffset + start }
+        : {}),
+    });
+    start = end;
+  }
+  return events;
+}
+
+/**
+ * Partition ledger delivery without dropping positional log lines. The worker
+ * retries only unacknowledged batches, so each batch must be independently
+ * valid for the app-runtime and Convex boundaries.
+ */
+export function partitionWorkerLedgerEvents(
+  events: readonly PlayRunLedgerEvent[],
+): PlayRunLedgerEvent[][] {
+  const batches: PlayRunLedgerEvent[][] = [];
+  let batch: PlayRunLedgerEvent[] = [];
+  let batchBytes = 2;
+
+  for (const event of events.flatMap((event) =>
+    event.type === 'log.appended' ? splitLogEvent(event) : [event],
+  )) {
+    const eventBytes = encodedJsonBytes(event);
+    if (
+      batch.length > 0 &&
+      (batch.length >= WORKER_LEDGER_EVENTS_PER_APPEND_MAX ||
+        batchBytes + eventBytes + 1 > WORKER_LEDGER_EVENT_BATCH_MAX_BYTES)
+    ) {
+      batches.push(batch);
+      batch = [];
+      batchBytes = 2;
+    }
+    batch.push(event);
+    batchBytes += eventBytes + 1;
+  }
+  if (batch.length > 0) batches.push(batch);
+  return batches;
+}

package/dist/bundling-sources/sdk/src/client.ts

@@ -243,8 +243,30 @@ type ExecuteToolRawOptions = {
   maxRetries?: number;
 };
 
-function resolveToolExecuteTimeoutMs(toolId: string): number | undefined {
+/**
+ * The server launches the Apify run, polls it for `payload.timeoutMs`, then
+ * reads its dataset before replying. Keep the client request alive through
+ * that final hand-off so a pollable run id is never lost to the SDK default
+ * timeout.
+ */
+const APIFY_SYNC_DEFAULT_TIMEOUT_MS = 300_000;
+const APIFY_SYNC_RESPONSE_GRACE_MS = 90_000;
+
+function resolveToolExecuteTimeoutMs(
+  toolId: string,
+  input: Record<string, unknown>,
+): number | undefined {
   const normalized = toolId.trim().toLowerCase();
+  if (normalized === 'apify_run_actor_sync') {
+    const requestedTimeoutMs = input.timeoutMs ?? APIFY_SYNC_DEFAULT_TIMEOUT_MS;
+    if (
+      typeof requestedTimeoutMs === 'number' &&
+      Number.isFinite(requestedTimeoutMs) &&
+      requestedTimeoutMs > 0
+    ) {
+      return Math.floor(requestedTimeoutMs) + APIFY_SYNC_RESPONSE_GRACE_MS;
+    }
+  }
   return normalized === 'deeplineagent' ||
     normalized === 'deeplineagent_deeplineagent' ||
     normalized === 'ai_inference' ||
@@ -1417,7 +1439,7 @@ export class DeeplineClient {
       headers,
       {
         forbiddenAsApiError: true,
-        timeout: options?.timeout ?? resolveToolExecuteTimeoutMs(toolId),
+        timeout: options?.timeout ?? resolveToolExecuteTimeoutMs(toolId, input),
         maxRetries: options?.maxRetries ?? 0,
         exactUrlOnly: true,
       },

package/dist/bundling-sources/sdk/src/play.ts

@@ -161,7 +161,7 @@ export interface PlayCallOptions {
  *   sqlListeners: [
  *     {
  *       id: 'jobs',
- *       tool: 'tamradar.company_radar',
+ *       tool: 'deepline_native.company_radar',
  *       stream: 'company_job_openings',
  *       operations: ['INSERT'],
  *     },
@@ -237,7 +237,7 @@ export type SqlListenerWhere = {
 export type SqlListenerDeclaration = {
   /** Short id unique inside this play. Deepline stores it as playName.id. */
   id: string;
-  /** Modeled monitor tool id, for example "tamradar.company_radar". */
+  /** Modeled monitor tool id, for example "deepline_native.company_radar". */
   tool: string;
   /** Stream key exposed by the modeled monitor tool, for example "company_job_openings". */
   stream: string;
@@ -560,6 +560,11 @@ export type DatasetBuilder<
    */
   run(options?: {
     description?: string;
+    /**
+     * `upsert` (default) returns every input row, reusing persisted work for
+     * existing keys. `net_new` atomically inserts and returns only unseen keys.
+     */
+    mode?: 'upsert' | 'net_new';
     key?:
       | (keyof InputRow & string)
       | readonly (keyof InputRow & string)[]

package/dist/bundling-sources/sdk/src/release.ts

@@ -108,10 +108,10 @@ export const SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: '0.1.226',
+  version: '0.1.228',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.226',
+    latest: '0.1.228',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.219',
     commandMinimumSupported: [