deepline 0.1.55 → 0.1.57

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -120,6 +120,7 @@ import {
 import {
   applyCsvRenameProjection,
   stripCsvProjectedFields,
+  stripCsvProjectionMetadata,
   cloneCsvAliasedRow,
   type CsvRenameOptions,
 } from '../../../shared_libs/play-runtime/csv-rename';
@@ -129,6 +130,12 @@ import type {
   LiveNodeProgressMap,
   LiveNodeProgressSnapshot,
 } from './runtime/live-progress';
+import {
+  ToolHttpError,
+  extractErrorBilling,
+  isHardBillingToolHttpError,
+  normalizeToolHttpErrorMessage,
+} from './runtime/tool-http-errors';
 
 // The play's default export. The bundler injects this — see bundle-play-file.ts.
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -194,23 +201,6 @@ const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
 const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
 const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
 
-class ToolHttpError extends Error {
-  readonly billing: Record<string, unknown> | null;
-
-  constructor(message: string, billing: Record<string, unknown> | null) {
-    super(message);
-    this.name = 'ToolHttpError';
-    this.billing = billing;
-  }
-}
-
-function formatCreditAmount(value: unknown): string {
-  if (typeof value !== 'number' || !Number.isFinite(value)) {
-    return String(value ?? '-');
-  }
-  return Number(value.toFixed(8)).toString();
-}
-
 function getStringField(value: unknown, key: string): string | null {
   if (!isRecord(value)) return null;
   const field = value[key];
@@ -226,71 +216,6 @@ function getObjectField(
   return isRecord(field) ? field : null;
 }
 
-function isInsufficientCreditsBilling(
-  billing: Record<string, unknown> | null,
-): billing is Record<string, unknown> {
-  return billing?.kind === 'insufficient_credits';
-}
-
-function formatInsufficientCreditsMessage(input: {
-  billing: Record<string, unknown>;
-  toolId: string;
-}): string {
-  const operation =
-    getStringField(input.billing, 'operation_id') ??
-    getStringField(input.billing, 'operation') ??
-    input.toolId;
-  const balance = formatCreditAmount(input.billing.balance_credits);
-  const required = formatCreditAmount(input.billing.required_credits);
-  const recommended = formatCreditAmount(
-    input.billing.recommended_add_credits ?? input.billing.needed_credits,
-  );
-  const billingUrl = getStringField(input.billing, 'billing_url');
-  const addSuffix =
-    billingUrl && recommended !== '-'
-      ? ` Add >=${recommended} at ${billingUrl}.`
-      : billingUrl
-        ? ` Add credits at ${billingUrl}.`
-        : '';
-  return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
-}
-
-function normalizeToolHttpErrorMessage(input: {
-  toolId: string;
-  status: number;
-  attempt: number;
-  maxAttempts: number;
-  bodyText: string;
-}): ToolHttpError {
-  let parsed: Record<string, unknown> | null = null;
-  try {
-    const candidate = JSON.parse(input.bodyText);
-    parsed = isRecord(candidate) ? candidate : null;
-  } catch {
-    parsed = null;
-  }
-  const billing = getObjectField(parsed, 'billing');
-  if (isInsufficientCreditsBilling(billing)) {
-    return new ToolHttpError(
-      `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
-        {
-          billing,
-          toolId: input.toolId,
-        },
-      )}`,
-      billing,
-    );
-  }
-  return new ToolHttpError(
-    `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${input.bodyText.slice(0, 500)}`,
-    billing,
-  );
-}
-
-function extractErrorBilling(error: unknown): Record<string, unknown> | null {
-  return error instanceof ToolHttpError ? error.billing : null;
-}
-
 /** R2 binding injected by the Worker runtime (when present in deploy metadata). */
 type WorkerEnv = {
   PLAYS_BUCKET?: R2Bucket;
@@ -690,6 +615,23 @@ function publicCsvInputRow<T extends Record<string, unknown>>(row: T): T {
   ) as T;
 }
 
+function publicCsvOutputRow<T extends Record<string, unknown>>(row: T): T {
+  const stripped = stripCsvProjectionMetadata(row) as Record<string, unknown>;
+  const publicRow: Record<string, unknown> = {};
+  for (const fieldName of Reflect.ownKeys(stripped)) {
+    if (
+      typeof fieldName === 'string' &&
+      fieldName.startsWith('__deepline')
+    ) {
+      continue;
+    }
+    const descriptor = Object.getOwnPropertyDescriptor(stripped, fieldName);
+    if (!descriptor) continue;
+    Object.defineProperty(publicRow, fieldName, descriptor);
+  }
+  return publicRow as T;
+}
+
 /**
  * Strip credentials and JWT-shaped tokens from any string before it lands in
  * a log buffer or upstream error message. The harness routinely echoes
@@ -1330,7 +1272,7 @@ async function callToolDirect(
       bodyText: text,
     });
     const retryable =
-      res.status === 429 ||
+      (res.status === 429 && !isHardBillingToolHttpError(lastError)) ||
       (res.status >= 500 && WORKER_RETRY_SAFE_5XX_TOOLS.has(toolId));
     if (!retryable || attempt >= maxAttempts) {
       throw lastError;
@@ -3742,9 +3684,8 @@ function createMinimalWorkerCtx(
             ? completedRow.__deeplineRowKey
             : derivePlayRowIdentity(publicCsvInputRow(completedRow), name);
         if (key) {
-          const { __deeplineRowKey: _rowKey, ...cleanedRow } =
-            publicCsvInputRow(completedRow);
-          void _rowKey;
+          const cleanedRow = publicCsvOutputRow(completedRow);
+          delete cleanedRow.__deeplineRowKey;
           resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
         }
       }
@@ -3763,8 +3704,9 @@ function createMinimalWorkerCtx(
           return resultByKey.get(key);
         })
         .filter((row): row is T & Record<string, unknown> => Boolean(row));
+      const publicOut = out.map((row) => publicCsvOutputRow(row));
       const hashStartedAt = nowMs();
-      const hash = await hashJson(out);
+      const hash = await hashJson(publicOut);
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.hash',
@@ -3797,7 +3739,7 @@ function createMinimalWorkerCtx(
         rowsSkipped,
         outputDatasetId: `map:${name}`,
         hash,
-        preview: serializeDurableStepValue(out.slice(0, 5)),
+        preview: toWorkflowSerializableValue(publicOut.slice(0, 5)),
         cachedRows:
           out.length <= WORKER_DATASET_IN_MEMORY_ROWS
             ? serializeDurableStepValue(out)
@@ -4333,13 +4275,15 @@ function createMinimalWorkerCtx(
               req,
               allowInline:
                 options?.timeoutMs == null && !childNeedsWorkflowScheduler,
-              body: {
-                name: resolvedName,
-                input: isRecord(input) ? input : {},
-                orgId: req.orgId,
-                parentExecutorToken: req.executorToken,
-                userEmail: req.userEmail ?? '',
-                profile: 'workers_edge',
+	              body: {
+	                name: resolvedName,
+	                input: isRecord(input) ? input : {},
+	                orgId: req.orgId,
+	                callbackBaseUrl: req.callbackUrl,
+	                baseUrl: req.baseUrl,
+	                parentExecutorToken: req.executorToken,
+	                userEmail: req.userEmail ?? '',
+	                profile: 'workers_edge',
                 manifest: childManifest,
                 childPlayManifests: req.childPlayManifests ?? null,
                 internalRunPlay: {
@@ -5344,6 +5288,10 @@ function serializePlayReturnValue(value: unknown): unknown {
   return serializeValue(value, 0);
 }
 
+function toWorkflowSerializableValue<T>(value: T): T {
+  return serializeValue(value, 0) as T;
+}
+
 /**
  * Hard cap on the trimmed result body persisted into Convex run state.
  * Convex docs are bounded (~1 MiB per field) so we keep this comfortably

package/dist/repo/apps/play-runner-workers/src/runtime/dataset-handles.ts

@@ -1,5 +1,7 @@
 import {
   applyCsvRenameProjection,
+  stripCsvProjectedFields,
+  stripCsvProjectionMetadata,
   type CsvRenameOptions,
 } from '../../../../shared_libs/play-runtime/csv-rename';
 import {
@@ -38,7 +40,32 @@ const datasetCountHints = new WeakMap<object, number | null>();
 const datasetCapabilities = new WeakMap<object, WorkerDatasetCapabilities>();
 
 function cloneRow<T extends DatasetRow>(row: T): T {
-  return { ...row };
+  const cloned: DatasetRow = {};
+  for (const key of Reflect.ownKeys(row)) {
+    const descriptor = Object.getOwnPropertyDescriptor(row, key);
+    if (!descriptor) continue;
+    Object.defineProperty(cloned, key, descriptor);
+  }
+  return cloned as T;
+}
+
+function internalDatasetRow<T extends DatasetRow>(row: T): T {
+  const stripped = stripCsvProjectionMetadata(row) as DatasetRow;
+  const publicRow: DatasetRow = {};
+  for (const key of Reflect.ownKeys(stripped)) {
+    if (typeof key === 'string' && key.startsWith('__deepline')) continue;
+    const descriptor = Object.getOwnPropertyDescriptor(stripped, key);
+    if (!descriptor) continue;
+    Object.defineProperty(publicRow, key, descriptor);
+  }
+  return publicRow as T;
+}
+
+function materializedDatasetRow<T extends DatasetRow>(row: T): T {
+  const stripped = stripCsvProjectedFields(row) as DatasetRow;
+  return Object.fromEntries(
+    Object.entries(stripped).filter(([key]) => !key.startsWith('__deepline')),
+  ) as T;
 }
 
 function registerChunkReader<T extends DatasetRow>(
@@ -138,10 +165,10 @@ export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
   const count = Math.max(0, Math.floor(input.count));
   const previewRows = (input.previewRows ?? [])
     .slice(0, WORKER_DATASET_PREVIEW_ROWS)
-    .map(cloneRow);
+    .map(materializedDatasetRow);
   const cachedRows =
     input.cachedRows && input.cachedRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
-      ? input.cachedRows.map(cloneRow)
+      ? input.cachedRows.map(internalDatasetRow)
       : null;
 
   async function loadRows(limit: number, offset: number): Promise<T[]> {
@@ -157,7 +184,7 @@ export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
     ) {
       return cachedRows
         .slice(normalizedOffset, normalizedOffset + normalizedLimit)
-        .map(cloneRow);
+        .map(internalDatasetRow);
     }
     const startedAt = input.nowMs();
     const rows = await input.readRows({
@@ -172,7 +199,7 @@ export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
       offset: normalizedOffset,
       rows: rows.length,
     });
-    return rows.map(cloneRow);
+    return rows.map(internalDatasetRow);
   }
 
   async function* readChunks(chunkSize: number): AsyncGenerator<T[], void, void> {
@@ -205,14 +232,15 @@ export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
     workProgress: input.workProgress,
     resolvers: {
       count: async () => count,
-      peek: async (limit) => await loadRows(Math.max(0, limit), 0),
+      peek: async (limit) =>
+        (await loadRows(Math.max(0, limit), 0)).map(materializedDatasetRow),
       materialize: async (limit) => {
         const rows: T[] = [];
         const maxRows = limit ?? count;
         for await (const chunk of readChunks(STREAM_MATERIALIZE_CHUNK_ROWS)) {
           for (const row of chunk) {
             if (rows.length >= maxRows) return rows;
-            rows.push(row);
+            rows.push(materializedDatasetRow(row));
           }
         }
         return rows;

package/dist/repo/apps/play-runner-workers/src/runtime/tool-http-errors.ts

@@ -0,0 +1,198 @@
+export class ToolHttpError extends Error {
+  readonly billing: Record<string, unknown> | null;
+
+  constructor(message: string, billing: Record<string, unknown> | null) {
+    super(message);
+    this.name = 'ToolHttpError';
+    this.billing = billing;
+  }
+}
+
+function formatCreditAmount(value: unknown): string {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return String(value ?? '-');
+  }
+  return Number(value.toFixed(8)).toString();
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function getStringField(value: unknown, key: string): string | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return typeof field === 'string' && field.trim() ? field : null;
+}
+
+function getObjectField(
+  value: unknown,
+  key: string,
+): Record<string, unknown> | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return isRecord(field) ? field : null;
+}
+
+function isInsufficientCreditsBilling(
+  billing: Record<string, unknown> | null,
+): billing is Record<string, unknown> {
+  return billing?.kind === 'insufficient_credits';
+}
+
+function isHardBillingFailurePayload(
+  payload: Record<string, unknown> | null,
+): payload is Record<string, unknown> {
+  if (!payload) return false;
+  const category = String(
+    payload.error_category ?? payload.errorCategory ?? '',
+  ).toLowerCase();
+  const code = String(payload.code ?? payload.error_code ?? '').toUpperCase();
+  const message = String(
+    payload.error ?? payload.message ?? payload.failure_description ?? '',
+  ).toLowerCase();
+  if (category === 'billing') return true;
+  if (
+    code === 'INSUFFICIENT_CREDITS' ||
+    code === 'BILLING_CAP_EXCEEDED' ||
+    code === 'MONTHLY_BILLING_LIMIT_EXCEEDED'
+  ) {
+    return true;
+  }
+  return (
+    (message.includes('billing cap') ||
+      message.includes('monthly billing limit') ||
+      message.includes('rolling 30-day organization billing cap') ||
+      message.includes('insufficient credits')) &&
+    !message.includes('rate limit')
+  );
+}
+
+function normalizeHardBillingPayload(
+  payload: Record<string, unknown>,
+): Record<string, unknown> {
+  return {
+    kind: 'billing_cap_exceeded',
+    code:
+      typeof payload.code === 'string' && payload.code.trim()
+        ? payload.code
+        : 'MONTHLY_BILLING_LIMIT_EXCEEDED',
+    error_category: 'billing',
+    failure_origin:
+      typeof payload.failure_origin === 'string' && payload.failure_origin.trim()
+        ? payload.failure_origin
+        : 'deepline_billing',
+    message:
+      typeof payload.error === 'string' && payload.error.trim()
+        ? payload.error
+        : typeof payload.message === 'string' && payload.message.trim()
+          ? payload.message
+          : 'Deepline billing cap exceeded.',
+    ...payload,
+  };
+}
+
+function formatHardBillingFailureMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+}): string {
+  const code = getStringField(input.billing, 'code');
+  const message =
+    getStringField(input.billing, 'message') ??
+    getStringField(input.billing, 'error') ??
+    'Deepline billing cap exceeded.';
+  return [
+    `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}:`,
+    'Deepline billing cap exceeded.',
+    'Run halted before marking remaining rows processed.',
+    code ? `code=${code}.` : '',
+    message,
+  ]
+    .filter(Boolean)
+    .join(' ');
+}
+
+function formatInsufficientCreditsMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+}): string {
+  const operation =
+    getStringField(input.billing, 'operation_id') ??
+    getStringField(input.billing, 'operation') ??
+    input.toolId;
+  const balance = formatCreditAmount(input.billing.balance_credits);
+  const required = formatCreditAmount(input.billing.required_credits);
+  const recommended = formatCreditAmount(
+    input.billing.recommended_add_credits ?? input.billing.needed_credits,
+  );
+  const billingUrl = getStringField(input.billing, 'billing_url');
+  const addSuffix =
+    billingUrl && recommended !== '-'
+      ? ` Add >=${recommended} at ${billingUrl}.`
+      : billingUrl
+        ? ` Add credits at ${billingUrl}.`
+        : '';
+  return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
+}
+
+export function normalizeToolHttpErrorMessage(input: {
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+  bodyText: string;
+}): ToolHttpError {
+  let parsed: Record<string, unknown> | null = null;
+  try {
+    const candidate = JSON.parse(input.bodyText);
+    parsed = isRecord(candidate) ? candidate : null;
+  } catch {
+    parsed = null;
+  }
+  const billing = getObjectField(parsed, 'billing');
+  if (isInsufficientCreditsBilling(billing)) {
+    return new ToolHttpError(
+      `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
+        {
+          billing,
+          toolId: input.toolId,
+        },
+      )}`,
+      billing,
+    );
+  }
+  const hardBillingPayload = isHardBillingFailurePayload(billing)
+    ? normalizeHardBillingPayload(billing)
+    : isHardBillingFailurePayload(parsed)
+      ? normalizeHardBillingPayload(parsed)
+      : null;
+  if (hardBillingPayload) {
+    return new ToolHttpError(
+      formatHardBillingFailureMessage({
+        billing: hardBillingPayload,
+        toolId: input.toolId,
+        status: input.status,
+        attempt: input.attempt,
+        maxAttempts: input.maxAttempts,
+      }),
+      hardBillingPayload,
+    );
+  }
+  return new ToolHttpError(
+    `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${input.bodyText.slice(0, 500)}`,
+    billing,
+  );
+}
+
+export function extractErrorBilling(
+  error: unknown,
+): Record<string, unknown> | null {
+  return error instanceof ToolHttpError ? error.billing : null;
+}
+
+export function isHardBillingToolHttpError(error: unknown): boolean {
+  return error instanceof ToolHttpError && isHardBillingFailurePayload(error.billing);
+}

package/dist/repo/sdk/src/client.ts

@@ -485,6 +485,37 @@ export class DeeplineClient {
     return `deepline plays run ${target} --input '{...}' --watch`;
   }
 
+  private starterPlayPath(play: Pick<PlayListItem, 'name' | 'reference'>): string {
+    const target = play.reference || play.name;
+    const unqualifiedName = target.split('/').pop() || play.name;
+    const safeName = unqualifiedName
+      .trim()
+      .toLowerCase()
+      .replace(/[^a-z0-9-]/g, '-')
+      .replace(/-+/g, '-')
+      .replace(/^-|-$/g, '');
+    return `./${safeName || 'play'}.play.ts`;
+  }
+
+  private playCloneEditStarter(
+    play: Pick<
+      PlayListItem,
+      'name' | 'reference' | 'canClone' | 'canEdit' | 'origin' | 'ownerType'
+    >,
+  ): PlayDescription['cloneEditStarter'] | undefined {
+    const readonlyPrebuilt =
+      (play.origin === 'prebuilt' || play.ownerType === 'deepline') &&
+      !play.canEdit;
+    if (!play.canClone && !readonlyPrebuilt) return undefined;
+    const target = play.reference || play.name;
+    const path = this.starterPlayPath(play);
+    return {
+      path,
+      command: `deepline plays get ${target} --source --out ${path}`,
+      checkCommand: `deepline plays check ${path}`,
+    };
+  }
+
   private summarizePlayListItem(
     play: PlayListItem,
     options?: { compact?: boolean },
@@ -496,6 +527,7 @@ export class DeeplineClient {
       'rowOutputSchema',
     );
     const runCommand = this.playRunCommand(play, { csvInput });
+    const cloneEditStarter = this.playCloneEditStarter(play);
     return {
       name: play.name,
       ...(play.reference ? { reference: play.reference } : {}),
@@ -515,6 +547,7 @@ export class DeeplineClient {
       ...(rowOutputSchema ? { rowOutputSchema } : {}),
       runCommand,
       examples: [runCommand],
+      ...(cloneEditStarter ? { cloneEditStarter } : {}),
       currentPublishedVersion: play.currentPublishedVersion ?? null,
       isDraftDirty: play.isDraftDirty,
     };
@@ -1472,12 +1505,10 @@ export class DeeplineClient {
 
   async searchPlays(options: {
     query: string;
-    origin?: 'prebuilt' | 'owned';
     compact?: boolean;
   }): Promise<PlayDescription[]> {
     const params = new URLSearchParams();
     params.set('search', options.query.trim());
-    if (options.origin) params.set('origin', options.origin);
     const response = await this.http.get<{ plays: PlayListItem[] }>(
       `/api/v2/plays?${params.toString()}`,
     );