deepline 0.3.285 → 0.3.287

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts

@@ -22,6 +22,11 @@ const EVENTS_TABLE = 'runtime_observability.events';
 const RUNTIME_EVENT_LOG_DEDUPE_INDEX = 'runtime_events_log_attempt_digest_idx';
 const RUNTIME_EVENT_LOG_POSITION_INDEX =
   'runtime_events_log_attempt_position_idx';
+const RUNTIME_EVENT_LOG_POSITION_CHANNELS = [
+  'worker',
+  'replay',
+  'envelope',
+] as const;
 const RUNTIME_EVENT_LOG_LEGACY_INDEX =
   'runtime_events_legacy_log_normalize_idx';
 const RUNTIME_EVENT_LOG_BODY_PAGE_INDEX = 'runtime_events_run_log_body_seq_idx';
@@ -51,7 +56,7 @@ const RUNTIME_EVENT_LOG_POSITION_EXPRESSION = `jsonb_set(
 // Migration readiness tracks this projection body independently of scheduler
 // wire and schema versions. Bump when a changed trigger must be reinstalled.
 export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
-  'runtime-event-search-log-context-v13-index-first-position';
+  'runtime-event-search-log-context-v14-btree-position';
 export const RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER =
   'runtime-event-canonical-retention-v1';
 export const RUNTIME_EVENTS_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
@@ -1263,10 +1268,7 @@ export async function attachRuntimeEventSearchTriggers(
                   'logPositions', jsonb_build_object(log_channel, log_position));
                 position_alias := jsonb_build_object(
                   'producerAttempt', (item->>'producerAttempt')::integer,
-                  '_runtimeScope:' || ${quoteLiteral(scope)}, true,
-                  '_orgId:' || run_org, true,
-                  '_runId:' || observation.run_id, true)
-                  || jsonb_build_object(log_channel, log_position);
+                  log_channel, log_position);
                 matched_event_id := NULL;
                 position_event_ids := ARRAY[
                   legacy_position_prefix || log_position::text,
@@ -1293,24 +1295,26 @@ export async function attachRuntimeEventSearchTriggers(
                  ORDER BY event.event_id
                  LIMIT 1;
                 IF matched_event_id IS NULL THEN
-                  -- A transport may already be aliased onto an event whose
-                  -- primary key belongs to another channel. Find that alias
-                  -- independently of content so changed bytes still fail the
-                  -- immutable-position check below. Materialize the scoped
-                  -- position lookup before applying run filters: otherwise
-                  -- the planner can choose the run-history index and inspect
-                  -- every old log for each incoming position, even with a
-                  -- custom plan. The indexed document already binds scope,
-                  -- org, run, attempt and channel position; retain the outer
-                  -- identity checks as defense in depth.
+                  -- Pack scope, organization and run into one indexed key.
+                  -- The materialization keeps generic plans from choosing a
+                  -- run-history scan based on the outer identity checks.
                   WITH position_matches AS MATERIALIZED (
+                    ${RUNTIME_EVENT_LOG_POSITION_CHANNELS.map(
+                      (channel) => `
                     SELECT event.event_id, event.source_digest, event.search_doc,
-                           event.runtime_scope, event.org_id, event.run_id, event.log_seq
+                           event.log_seq, event.runtime_scope, event.org_id, event.run_id
                       FROM runtime_observability.events AS event
-                     WHERE event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
+                     WHERE log_channel = '${channel}'
+                       AND ARRAY[event.runtime_scope, event.org_id, event.run_id]
+                         = ARRAY[${quoteLiteral(scope)}, run_org, observation.run_id]
+                       AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
                        AND event.search_doc ? 'producerAttempt'
                        AND event.search_doc ? 'logPositions'
-                       AND (${RUNTIME_EVENT_LOG_POSITION_EXPRESSION.replaceAll('search_doc', 'event.search_doc')}) @> position_alias
+                       AND (event.search_doc->'logPositions') ? '${channel}'
+                       AND event.search_doc->'producerAttempt' = position_alias->'producerAttempt'
+                       AND event.search_doc->'logPositions'->'${channel}' = position_alias->'${channel}'
+                    `,
+                    ).join(' UNION ALL ')}
                   )
                   SELECT event.event_id, event.source_digest, event.search_doc
                     INTO matched_event_id, existing_digest, matched_doc
@@ -2034,8 +2038,27 @@ const RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC: RuntimeEventIndexExpectation = {
       WHERE ${RUNTIME_EVENT_LOG_POSITION_PREDICATE}`,
 };
 
+const RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES: RuntimeEventIndexExpectation[] =
+  RUNTIME_EVENT_LOG_POSITION_CHANNELS.map((channel) => ({
+    name: `runtime_events_log_${channel}_position_idx`,
+    keys: [
+      'ARRAY[runtime_scope, org_id, run_id]',
+      "(search_doc -> 'producerAttempt'::text)",
+      `(search_doc -> 'logPositions'::text) -> '${channel}'::text`,
+    ],
+    predicate: `${RUNTIME_EVENT_LOG_POSITION_PREDICATE}
+      AND (search_doc->'logPositions') ? '${channel}'`,
+    createSql: `CREATE INDEX CONCURRENTLY IF NOT EXISTS runtime_events_log_${channel}_position_idx
+      ON runtime_observability.events
+        ((ARRAY[runtime_scope, org_id, run_id]), (search_doc->'producerAttempt'),
+         (search_doc->'logPositions'->'${channel}'))
+      WHERE ${RUNTIME_EVENT_LOG_POSITION_PREDICATE}
+        AND (search_doc->'logPositions') ? '${channel}'`,
+  }));
+
 const RUNTIME_EVENT_QUERY_INDEXES: RuntimeEventIndexExpectation[] = [
   RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
+  ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
   {
     name: RUNTIME_EVENT_LOG_DEDUPE_INDEX,
     keys: [
@@ -2167,20 +2190,25 @@ async function prepareRuntimeEventQueryIndex(
 export async function hasRuntimeEventLogPositionIndex(
   client: PostgresSchedulerQueryClient,
 ): Promise<boolean> {
-  return hasRuntimeEventQueryIndex(
-    client,
+  for (const expected of [
     RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
-  );
+    ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
+  ]) {
+    if (!(await hasRuntimeEventQueryIndex(client, expected))) return false;
+  }
+  return true;
 }
 
-/** Prepare the existing position index before replacing the projection body. */
+/** Prepare old and new position indexes before replacing the projection body. */
 export async function prepareRuntimeEventLogPositionIndex(
   client: PostgresSchedulerQueryClient,
 ): Promise<void> {
-  await prepareRuntimeEventQueryIndex(
-    client,
+  for (const expected of [
     RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
-  );
+    ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
+  ]) {
+    await prepareRuntimeEventQueryIndex(client, expected);
+  }
 }
 
 /** Build only operator-owned indexes, concurrently and outside a transaction. */

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts

@@ -337,7 +337,6 @@ export async function reserveRuntimeSandboxCapacityLease(
           AND operation.state = $4
           AND lease.released_at IS NULL
           AND lease.sandbox_id IS NULL
-          AND lease.expires_at > clock_timestamp()
         ORDER BY operation.created_at ASC
         LIMIT 2`,
       [orgId, runId, input.provider, RuntimeSandboxOperationState.Reserved],
@@ -770,7 +769,10 @@ export async function bindRuntimeSandboxCapacityLease(
     `
       UPDATE ${leases}
       SET provider = $2, sandbox_id = $3, routing_domain = $4
-      WHERE lease_id = $1 AND released_at IS NULL AND expires_at > clock_timestamp()
+      WHERE lease_id = $1 AND released_at IS NULL AND provider = $2
+        AND (expires_at > clock_timestamp() OR sandbox_id = $3)
+        AND (sandbox_id IS NULL OR sandbox_id = $3)
+        AND (sandbox_id IS NULL OR routing_domain IS NOT DISTINCT FROM $4)
       RETURNING lease_id
     `,
     [
@@ -793,7 +795,7 @@ export async function releaseRuntimeSandboxCapacityLease(
     `
       UPDATE ${leases}
       SET released_at = clock_timestamp()
-      WHERE lease_id = $1 AND released_at IS NULL
+      WHERE lease_id = $1 AND released_at IS NULL AND sandbox_id IS NULL
       RETURNING lease_id
     `,
     [assertId(input.leaseId, 'leaseId')],
@@ -811,15 +813,29 @@ export async function abandonRuntimeSandboxOperationReservation(
   input: { operationKey: string },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<boolean> {
-  const { leases, operations } = schemaTables(options);
+  const { leases, operations, runs } = schemaTables(options);
   const result = await client.query<{ operation_key: string }>(
     `
-      WITH abandoned AS (
+      WITH custody_run AS MATERIALIZED (
+        SELECT run_id FROM ${runs}
+        WHERE run_id = (SELECT run_id FROM ${operations} WHERE operation_key = $1)
+        FOR NO KEY UPDATE
+      ), custody_attempts AS MATERIALIZED (
+        SELECT attempt.* FROM ${quoteIdent(options?.schema ?? 'runtime_scheduler')}."attempts" AS attempt
+        WHERE attempt.run_id IN (SELECT run_id FROM custody_run)
+        ORDER BY attempt.attempt FOR UPDATE
+      ), eligible_lease AS MATERIALIZED (
+        SELECT lease.lease_id FROM ${leases} AS lease
+        CROSS JOIN (SELECT count(*) FROM custody_attempts) AS attempt_locks
+        WHERE lease_id = $1 AND sandbox_id IS NULL AND released_at IS NULL
+        FOR UPDATE OF lease
+      ), abandoned AS (
         UPDATE ${operations} AS operation
         SET state = $2,
             updated_at = clock_timestamp()
         WHERE operation.operation_key = $1
           AND operation.state = $3
+          AND EXISTS (SELECT 1 FROM eligible_lease)
         RETURNING operation.operation_key
       ), released AS (
         UPDATE ${leases} AS lease
@@ -898,14 +914,31 @@ export async function recordRecoveredDaytonaCancellationSandbox(
   const { operations, leases, cleanup } = schemaTables(options);
   const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
   const recorded = await client.query<{ operation_key: string }>(
-    `WITH promoted AS (
+    `WITH custody_run AS MATERIALIZED (
+       SELECT run.run_id FROM ${schema}."runs" AS run
+       WHERE run.run_id = (SELECT run_id FROM ${operations} WHERE operation_key = $1)
+       FOR NO KEY UPDATE
+     ), custody_attempts AS MATERIALIZED (
+       SELECT attempt.* FROM ${schema}."attempts" AS attempt
+       WHERE attempt.run_id IN (SELECT run_id FROM custody_run)
+       ORDER BY attempt.attempt FOR UPDATE
+     ), custody_lease AS MATERIALIZED (
+       SELECT lease.lease_id FROM ${leases} AS lease
+       CROSS JOIN (SELECT count(*) FROM custody_attempts) AS attempt_locks
+       WHERE lease.lease_id = $1 FOR UPDATE OF lease
+     ), custody_operation AS MATERIALIZED (
+       SELECT operation.operation_key FROM ${operations} AS operation
+       CROSS JOIN (SELECT count(*) FROM custody_lease) AS lease_locks
+       WHERE operation.operation_key = $1 FOR UPDATE OF operation
+     ), promoted AS (
        UPDATE ${operations} AS operation
        SET state = 'deleting',
            sandbox_id = $2,
            routing_domain = $3,
            provider_created_at = clock_timestamp(),
            updated_at = clock_timestamp()
-       WHERE operation.operation_key = $1
+       FROM custody_operation
+       WHERE operation.operation_key = custody_operation.operation_key
          AND operation.provider = 'daytona'
          AND operation.state = 'reserved'
        RETURNING operation.operation_key, operation.run_id, operation.attempt,

package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts

@@ -1,4 +1,17 @@
-import type { PlayStaticSubstep } from '../plays/static-pipeline';
+import {
+  getTopLevelPipelineSubsteps,
+  type PlayStaticPipeline,
+  type PlayStaticSubstep,
+} from '../plays/static-pipeline';
+import {
+  docflowOutputRoots,
+  flattenSubsteps,
+  resolveDocflowOwnerDataset,
+} from '../plays/docflow-binding-owner';
+import {
+  normalizePlayConditionCounts,
+  type PlayConditionCounts,
+} from './cell-provenance';
 import {
   normalizePlayDocflowNodeIoState,
   type PlayDocflowNodeIoState,
@@ -32,6 +45,7 @@ export type PlayVisualNodeProgress = {
   completedAt?: number | null;
   artifactTableNamespace?: string | null;
   nodeIo?: PlayDocflowNodeIoState;
+  conditionCounts?: PlayConditionCounts;
 };
 
 export type PlayVisualNodeState = {
@@ -162,6 +176,9 @@ export function normalizePlayVisualNodeProgressMap(
           ? null
           : undefined;
     const nodeIo = normalizePlayDocflowNodeIoState(rawProgress.nodeIo);
+    const conditionCounts = normalizePlayConditionCounts(
+      rawProgress.conditionCounts,
+    );
 
     normalized[nodeId] = {
       ...(completed !== undefined ? { completed } : {}),
@@ -180,11 +197,76 @@ export function normalizePlayVisualNodeProgressMap(
         ? { artifactTableNamespace }
         : {}),
       ...(nodeIo ? { nodeIo } : {}),
+      ...(conditionCounts ? { conditionCounts } : {}),
     };
   }
   return normalized;
 }
 
+/** Join observed predicates to authored decision nodes by their owning column. */
+export function projectDocflowConditionCounts(
+  pipeline: PlayStaticPipeline | null | undefined,
+  sheets: readonly {
+    tableNamespace: string;
+    conditions?: Record<string, PlayConditionCounts>;
+  }[],
+): Record<string, PlayConditionCounts> {
+  if (!pipeline?.docflow) return {};
+  const substeps = getTopLevelPipelineSubsteps(pipeline);
+  const namespaces = flattenSubsteps(substeps)
+    .filter((step) => step.type === 'dataset' || step.type === 'csv')
+    .map((step) =>
+      step.type === 'dataset'
+        ? (step.tableNamespace ?? step.field)
+        : step.field,
+    )
+    .sort((a, b) => b.length - a.length);
+  const decisions = new Set(
+    pipeline.docflow.nodes
+      .filter((node) => node.kind === 'decision' && !node.sketch)
+      .map((node) => node.id),
+  );
+  const result: Record<string, PlayConditionCounts> = {};
+  for (const binding of pipeline.docflow.bindings) {
+    if (!decisions.has(binding.nodeId)) continue;
+    const owner = resolveDocflowOwnerDataset(binding, substeps).substep;
+    if (!owner) continue;
+    const namespace = owner.tableNamespace ?? owner.field;
+    const roots = docflowOutputRoots(binding);
+    // Several output columns can carry different predicates. Do not combine them.
+    if (roots.length !== 1) continue;
+    const root = roots[0]!;
+    const column = owner.sheetContract?.columns.find(
+      (candidate) =>
+        candidate.field === root ||
+        candidate.id === root ||
+        candidate.sqlName === root,
+    );
+    const key = column?.sqlName ?? root;
+    for (const sheet of sheets) {
+      const sheetOwner = namespaces.find(
+        (candidate) =>
+          sheet.tableNamespace === candidate ||
+          sheet.tableNamespace.startsWith(`${candidate}_`),
+      );
+      if (sheetOwner !== namespace) continue;
+      const counts = normalizePlayConditionCounts(
+        sheet.conditions?.[key] ?? sheet.conditions?.[root],
+      );
+      if (!counts) continue;
+      const current = result[`docflow:${binding.nodeId}`] ?? {
+        true: 0,
+        false: 0,
+      };
+      result[`docflow:${binding.nodeId}`] = {
+        true: current.true + counts.true,
+        false: current.false + counts.false,
+      };
+    }
+  }
+  return result;
+}
+
 export function applyVisualNodeProgress(input: {
   nodeStates: readonly PlayVisualNodeState[];
   progressByNode: PlayVisualNodeProgressMap;

package/dist/bundling-sources/shared_libs/play-runtime/test-async-batching.ts

@@ -87,7 +87,10 @@ export const testAsyncContractLookupBatchStrategy: BatchOperationStrategy<
     const first = payloads[0]!;
     const items = payloads.map((payload) => ({
       itemKey: payload.key,
-      payload: { key: payload.key, row_number: payload.row_number },
+      payload: {
+        key: payload.key,
+        row_number: payload.row_number,
+      },
     }));
     return {
       batchOperation: 'test_async_contract_start',

package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts

@@ -2159,3 +2159,71 @@ export function cloneToolExecuteResultWithExecution<TResult>(
     execution,
   }) as ToolExecuteResult<TResult>;
 }
+
+/**
+ * Native splitters return the logical source response; the historical adapter
+ * stores it inside an extra `{ data }` layer. Keep that immutable receipt and
+ * its legacy raw/getter projections, and select the source response only at a
+ * newly admitted Play's boundary. This is one provenance-backed projection,
+ * never recursive provider-envelope unwrapping.
+ */
+export function projectNativeBatchToolResponse(
+  value: ToolExecuteResult,
+  policy: 'legacy_envelope' | 'logical_response' = 'logical_response',
+): ToolExecuteResult {
+  const stored = value.toolResponse.rawV2;
+  const materialized = value.meta?.nativeBatchResponseMaterialized === true;
+  if (policy === 'legacy_envelope' && !materialized) return value;
+  if (policy === 'logical_response' && materialized) return value;
+  if (!materialized && (!isRecord(stored) || !Object.hasOwn(stored, 'data')))
+    return value;
+  const recordedView = value.meta?.nativeBatchResponseView;
+  let view: ToolResponseView | undefined =
+    recordedView === 'data' || recordedView === 'rawV2'
+      ? recordedView
+      : undefined;
+  if (view !== 'data' && view !== 'rawV2') {
+    // Retained BounceBan receipts predate the provenance fact. Its scalar
+    // action returns `{ data: verdict }`; only the old batch adapter adds
+    // `{ data: { data: verdict } }`. Do not apply this repair to other tools
+    // or infer it from a provider's arbitrary `data` field.
+    const source = (stored as Record<string, unknown>).data;
+    const verdict = isRecord(source) ? source.data : undefined;
+    if (
+      value._metadata.toolId !== 'bounceban_verify_single' ||
+      !isRecord(source) ||
+      !isRecord(verdict) ||
+      typeof verdict.email !== 'string' ||
+      (verdict.status !== undefined && verdict.status !== 'success') ||
+      !['deliverable', 'undeliverable', 'risky', 'unknown'].includes(
+        String(verdict.result),
+      )
+    ) {
+      return value;
+    }
+    view = 'data';
+  }
+  const response = {
+    ...value.toolResponse,
+    rawV2: materialized
+      ? { data: stored }
+      : (stored as Record<string, unknown>).data,
+    view: materialized ? ('data' as const) : view,
+  };
+  // Preserve non-enumerable metadata/aliases and live list/getter accessors.
+  // Do not mutate a result shared with another artifact or the receipt cache.
+  const descriptors = Object.getOwnPropertyDescriptors(value);
+  descriptors.toolResponse = { ...descriptors.toolResponse, value: response };
+  descriptors.toolOutput = { ...descriptors.toolOutput, value: response };
+  descriptors.meta = {
+    value: {
+      ...value.meta,
+      nativeBatchResponseView: view,
+      nativeBatchResponseMaterialized: !materialized,
+    },
+    writable: true,
+    enumerable: true,
+    configurable: true,
+  };
+  return Object.create(Object.getPrototypeOf(value), descriptors);
+}

package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts

@@ -1,5 +1,6 @@
 import type { PlayDocflowNodeIoState } from './docflow-node-io';
 import type { PlayMapRowActivitySource } from './run-ledger';
+import type { PlayConditionCounts } from './cell-provenance';
 
 export type PlayStagedFileRef = {
   storageKind: 'r2';
@@ -33,6 +34,7 @@ export type PlayVisualNodeProgress = {
   completedAt?: number;
   artifactTableNamespace?: string | null;
   nodeIo?: PlayDocflowNodeIoState;
+  conditionCounts?: PlayConditionCounts;
 };
 
 export type PlayVisualNodeProgressMap = Record<string, PlayVisualNodeProgress>;

package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts

@@ -7,10 +7,12 @@ import type { ToolExecutionErrorSchemaVersion } from './tool-execution-error';
 import type { PlayDataset, PlayDatasetInput, PlayDatasetRow } from './dataset';
 
 export const LEGACY_PLAY_AUTHORING_CONTRACT_EDITION = 1 as const;
+// Reader-first rollout: accept edition 9, but keep new bundles on edition 8
+// until the edition-9 runtime reader has completed deployment.
 export const PLAY_AUTHORING_CONTRACT_EDITION = 8 as const;
 export const PLAY_AUTHORING_INPUT_SCHEMA_SNAPSHOT_EDITION = 3 as const;
 export const SUPPORTED_PLAY_AUTHORING_CONTRACT_EDITIONS = [
-  1, 2, 3, 4, 5, 6, 7, 8,
+  1, 2, 3, 4, 5, 6, 7, 8, 9,
 ] as const;
 
 export type PlayAuthoringContractEdition =
@@ -75,6 +77,13 @@ const PLAY_AUTHORING_CONTRACT_CHANGELOG_DETAILS = [
     newWritesEnd: null,
     readerRemoval: null,
   },
+  {
+    changed:
+      'Projects native-batch rawV2 to the logical source response at the Play boundary. Editions 1–8 retain their admitted response nesting; completed tool receipts and provider identities are unchanged.',
+    compatibilityOwner: 'Plays Runtime',
+    newWritesEnd: null,
+    readerRemoval: null,
+  },
 ] as const;
 
 export const PLAY_AUTHORING_CONTRACT_CHANGELOG =

package/dist/bundling-sources/shared_libs/plays/enrich-compat-adapter.ts

@@ -1,5 +1,7 @@
 export const ENRICH_COMPAT_DEFAULT_PLAY_NAME = 'deepline-enrich-v1-compat';
 export const ENRICH_COMPAT_DEFAULT_MAP_NAME = 'deepline_enrich_rows';
+export const ENRICH_COMMAND_DEPRECATION_MESSAGE =
+  'The `deepline enrich` command is deprecated. Deepline Plays gives much better results. Run `deepline plays --help` to get started. If you still need the legacy command, submit feedback explaining why.';
 
 export type EnrichCompatibilityOptions = {
   playName?: string;