deepline 0.3.247 → 0.3.249

package/dist/bundling-sources/sdk/src/runs/overview.ts

@@ -0,0 +1,457 @@
+import type { PlayStatus } from '../types';
+import { DATASET_EXECUTION_BUCKETS } from '../../../shared_libs/plays/dataset-summary';
+
+type RecordValue = Record<string, unknown>;
+const record = (value: unknown): RecordValue | undefined =>
+  value !== null && typeof value === 'object' && !Array.isArray(value)
+    ? (value as RecordValue)
+    : undefined;
+
+/** Presentation only: exports and retained evidence keep the original metadata. */
+function excludePreviewMetadata(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(excludePreviewMetadata);
+  const fields = record(value);
+  if (!fields) return value;
+  return Object.fromEntries(
+    Object.entries(fields).map(([name, child]) => [
+      name,
+      name === '_dl_meta'
+        ? '<excluded from preview>'
+        : excludePreviewMetadata(child),
+    ]),
+  );
+}
+
+function lifecycleActions(status: PlayStatus, id: string, state?: string) {
+  return {
+    ...(['started', 'queued', 'running', 'waiting'].includes(state ?? '')
+      ? { stopRunCmd: `deepline runs stop ${id} --json` }
+      : {}),
+    ...(state === 'failed'
+      ? {
+          getFailedLogsCmd: `deepline runs get ${id} --log-failed --json`,
+          ...(status.rerunCommand ? { rerunCmd: status.rerunCommand } : {}),
+        }
+      : {}),
+  };
+}
+
+function withoutProviderSpend(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(withoutProviderSpend);
+  const source = record(value);
+  if (!source) return value;
+  return Object.fromEntries(
+    Object.entries(source)
+      .filter(
+        ([key]) => key !== 'providerCostUsd' && key !== 'totalProviderCostUsd',
+      )
+      .map(([key, child]) => [key, withoutProviderSpend(child)]),
+  );
+}
+
+function executionSummary(value: unknown, conflictingPopulation: boolean) {
+  const source = record(value);
+  if (!source)
+    return {
+      recordedAttempts: null,
+      countConsistency: 'unknown',
+      coverage: 'unknown',
+      byStatus: {},
+    };
+  const states: Record<string, { count: number; percent: number }> = {};
+  const totals = new Set<number>();
+  let sum = 0;
+  let valid = true;
+  for (const [name, text] of Object.entries(source)) {
+    const match =
+      typeof text === 'string' && /^(\d+)\/(\d+) \((\d+)%\)$/.exec(text);
+    if (!match) {
+      valid = false;
+      continue;
+    }
+    const [, numerator, denominator, percentage] = match;
+    const count = Number(numerator),
+      total = Number(denominator),
+      percent = Number(percentage);
+    if (
+      ![count, total, percent].every(Number.isSafeInteger) ||
+      count > total ||
+      percent !== (total ? Math.round((count * 100) / total) : 0)
+    )
+      valid = false;
+    totals.add(total);
+    sum += count;
+    if (count !== 0) states[name] = { count, percent };
+  }
+  const total = totals.size === 1 ? [...totals][0] : null;
+  const reconciled =
+    valid &&
+    total !== null &&
+    sum === total &&
+    DATASET_EXECUTION_BUCKETS.every((name) => name in source);
+  return {
+    recordedAttempts: total,
+    countConsistency: reconciled ? 'consistent' : 'inconsistent',
+    coverage: conflictingPopulation ? 'conflicting' : 'unknown',
+    byStatus: states,
+    ...(!valid ? { unparsed: source } : {}),
+  };
+}
+
+function datasetSummary(value: unknown, conflictingPopulation: boolean) {
+  const summary = record(value);
+  if (!summary) return null;
+  const columns = record(summary.columnStats);
+  const {
+    total_rows,
+    rowCounts,
+    columnStats: _columnStats,
+    ...other
+  } = summary;
+  return {
+    ...other,
+    reportedPopulation: {
+      totalRows: total_rows ?? record(rowCounts)?.persisted ?? null,
+      persistedRows: record(rowCounts)?.persisted ?? null,
+      succeededRows: record(rowCounts)?.succeeded ?? null,
+      failedRows: record(rowCounts)?.failed ?? null,
+    },
+    coverage: conflictingPopulation ? 'conflicting' : 'unknown',
+    ...(columns
+      ? {
+          columnStats: Object.fromEntries(
+            Object.entries(columns).map(([name, value]) => {
+              const column = record(value);
+              const { execution, ...rest } = column ?? {};
+              return [
+                name,
+                column && 'execution' in column
+                  ? {
+                      ...rest,
+                      executionStatusCounts: executionSummary(
+                        execution,
+                        conflictingPopulation,
+                      ),
+                    }
+                  : value,
+              ];
+            }),
+          ),
+        }
+      : {}),
+  };
+}
+
+/** Action descriptions are affordances, never evidence that an action ran. */
+function actions(value: unknown, names: Record<string, string>) {
+  return Object.fromEntries(
+    Object.entries(record(value) ?? {}).flatMap(([name, value]) => {
+      const action = record(value);
+      if (!action || action.deprecated) return [];
+      if (!(name in names) || typeof action.command !== 'string') return [];
+      return [
+        [
+          names[name],
+          action.kind === 'deepline_run_billing'
+            ? action.command.replace(
+                "jq '.billing'",
+                "jq '.diagnostics.billing'",
+              )
+            : action.command,
+        ],
+      ];
+    }),
+  );
+}
+
+/** A read-only overview of recorded run facts, never a complete row export. */
+export function buildRunOverview(status: PlayStatus, fullStatus?: PlayStatus) {
+  const pkg = status.package;
+  if (!pkg) {
+    throw new Error(
+      'This server did not return a run package; use runs get --full --json.',
+    );
+  }
+  const outputs = Object.entries(pkg.outputs);
+  const resultPackage = fullStatus?.package?.resultState
+    ? fullStatus.package
+    : pkg;
+  const warnings = [...(pkg.warnings ?? [])];
+  const { outcome: _outcome, ...run } = pkg.run;
+  const steps = pkg.steps.map((step) => {
+    const timing = record(step.timing);
+    const started = timing?.startedAt,
+      finished = timing?.completedAt;
+    return {
+      id: step.id,
+      kind: step.kind,
+      status: step.status,
+      progress: step.progress,
+      timing: step.timing,
+      ...(typeof started === 'number' &&
+      typeof finished === 'number' &&
+      finished >= started
+        ? { durationMs: finished - started }
+        : {}),
+      ...(step.error ? { error: step.error } : {}),
+    };
+  });
+  const catalog = [...(pkg.datasets ?? [])];
+  for (const value of Object.values(pkg.outputs)) {
+    if (
+      value.kind === 'dataset' &&
+      typeof value.path === 'string' &&
+      !catalog.some((entry) =>
+        value.datasetId
+          ? entry.datasetId === value.datasetId
+          : entry.path === value.path,
+      )
+    ) {
+      catalog.push({
+        kind: 'dataset',
+        path: value.path,
+        ...(typeof value.datasetId === 'string'
+          ? { datasetId: value.datasetId }
+          : {}),
+        ...(typeof value.rowCount === 'number'
+          ? { rowCount: value.rowCount }
+          : {}),
+      });
+    }
+  }
+  const uniqueCatalog = catalog.filter(
+    (entry, index) =>
+      catalog.findIndex((candidate) =>
+        entry.datasetId
+          ? candidate.datasetId === entry.datasetId
+          : candidate.path === entry.path && !candidate.datasetId,
+      ) === index,
+  );
+  const datasets = uniqueCatalog.map((dataset) => {
+    const step = pkg.steps.find((candidate) => {
+      const output = record(candidate.output);
+      return (
+        output &&
+        (dataset.datasetId
+          ? output.datasetId === dataset.datasetId
+          : output.path === dataset.path)
+      );
+    });
+    const stepOutput = record(step?.output);
+    const preview = record(stepOutput?.preview) ?? dataset.preview;
+    // Also project historical/server previews that still contain debug columns.
+    const rows = Array.isArray(preview?.rows)
+      ? preview.rows.map((row: unknown) => {
+          const fields = record(row);
+          return fields
+            ? Object.fromEntries(
+                Object.entries(fields)
+                  .filter(([name]) => !name.startsWith('_'))
+                  .map(([name, value]) => [
+                    name,
+                    excludePreviewMetadata(value),
+                  ]),
+              )
+            : row;
+        })
+      : [];
+    const total =
+      typeof preview?.totalRows === 'number'
+        ? preview.totalRows
+        : dataset.rowCount;
+    const named = outputs.filter(
+      ([, value]) =>
+        value.kind === 'dataset' &&
+        (dataset.datasetId
+          ? value.datasetId === dataset.datasetId
+          : value.path === dataset.path),
+    );
+    const summary = record(record(dataset)?.summary ?? stepOutput?.summary);
+    const rowCounts = record(summary?.rowCounts);
+    const progress = record(step?.progress);
+    const countConflict =
+      (typeof rowCounts?.persisted === 'number' &&
+        typeof dataset.rowCount === 'number' &&
+        rowCounts.persisted !== dataset.rowCount) ||
+      (typeof rowCounts?.failed === 'number' &&
+        typeof progress?.failed === 'number' &&
+        rowCounts.failed !== progress.failed);
+    if (countConflict)
+      warnings.push(
+        `Output ${dataset.path}: dataset summary and step/row counts disagree. Column execution counts may cover only a subset; inspect step progress and exported rows.`,
+      );
+    return {
+      datasetId: dataset.datasetId,
+      name: dataset.tableNamespace,
+      stepId: step?.id ?? named[0]?.[1].stepId,
+      rowCount: dataset.rowCount ?? null,
+      ...(dataset.lifecycle ? { lifecycle: dataset.lifecycle } : {}),
+      summary: datasetSummary(summary, countConflict),
+      storage: {
+        ...(dataset.sqlQualifiedTableName?.includes('.')
+          ? {
+              schema: dataset.sqlQualifiedTableName
+                .slice(0, dataset.sqlQualifiedTableName.lastIndexOf('.'))
+                .replace(/^"(.*)"$/, '$1')
+                .replace(/""/g, '"'),
+            }
+          : {}),
+        table: dataset.sqlTableName,
+      },
+      preview: {
+        population: 'source-reported dataset sample; not an export count',
+        availability: rows.length
+          ? 'available'
+          : total === 0
+            ? 'empty'
+            : 'unavailable',
+        returnedRows: rows.length,
+        totalRows: total ?? null,
+        partial:
+          preview?.truncated === true ||
+          total === undefined ||
+          rows.length < total,
+        rows,
+      },
+      ...(dataset.exportUnavailable
+        ? { exportUnavailable: dataset.exportUnavailable }
+        : {}),
+      recovered: dataset.recovered === true,
+    };
+  });
+  return {
+    schemaVersion: pkg.schemaVersion,
+    kind: 'run_overview',
+    ...((status.runStatus ?? pkg.runStatus)
+      ? { runStatus: status.runStatus ?? pkg.runStatus }
+      : {}),
+    run: {
+      ...run,
+      ...(status.revisionId && !run.revisionId
+        ? { revisionId: status.revisionId }
+        : {}),
+    },
+    steps,
+    failures: {
+      steps: steps.flatMap((step) => {
+        const progress = record(step.progress);
+        const failedRows =
+          typeof progress?.failed === 'number' ? progress.failed : undefined;
+        return step.status === 'failed' ||
+          step.error ||
+          (failedRows !== undefined && failedRows > 0)
+          ? [
+              {
+                stepId: step.id,
+                status: step.status,
+                failedRows,
+                ...(step.error ? { message: step.error } : {}),
+              },
+            ]
+          : [];
+      }),
+    },
+    errorDetails: {
+      coverage: 'unknown',
+      reported: record(pkg)?.errors ?? record(status)?.errors ?? [],
+      ...(pkg.run.error ? { run: pkg.run.error } : {}),
+      steps: steps
+        .filter((step) => step.error)
+        .map((step) => ({ stepId: step.id, message: step.error })),
+      ...(pkg.next?.logs ? { getMore: 'actions.getLogsCmd' } : {}),
+    },
+    rowOutcomes: status.rowOutcomes,
+    warnings,
+    ...(status.failedLogs ? { failedLogs: status.failedLogs } : {}),
+    datasets,
+    ...(Object.hasOwn(resultPackage, 'result')
+      ? { result: resultPackage.result }
+      : {}),
+    resultState: resultPackage.resultState ?? 'unavailable',
+    actions: {
+      ...lifecycleActions(status, pkg.run.id, pkg.run.status),
+      ...actions(pkg.next, {
+        inspect: 'getRunCmd',
+        full: 'getFullRunCmd',
+        billing: 'getBillingCmd',
+        logs: 'getLogsCmd',
+      }),
+      ...receiptAction(pkg.run.id),
+      ...(uniqueCatalog.some((dataset) => dataset.actions?.exportCsv)
+        ? {
+            exportDatasetCmds: uniqueCatalog.flatMap((dataset) => {
+              const command = dataset.actions?.exportCsv?.command;
+              return command && !dataset.exportUnavailable ? [command] : [];
+            }),
+          }
+        : {}),
+      ...(uniqueCatalog.some((dataset) => dataset.actions?.query)
+        ? {
+            queryTableSqlCmds: uniqueCatalog.flatMap((dataset) =>
+              dataset.actions?.query?.command
+                ? [dataset.actions.query.command]
+                : [],
+            ),
+          }
+        : {}),
+    },
+    // Keep the full diagnostic payload except its duplicate presentation package.
+    // Do not maintain an allowlist: it silently lost child runs, usage and future
+    // diagnostic fields. Full mode also resolves an explicitly full-only result.
+    ...(fullStatus
+      ? {
+          diagnostics: Object.fromEntries(
+            Object.entries(fullStatus)
+              .filter(([key]) => key !== 'package')
+              .map(([key, value]) => [key, withoutProviderSpend(value)]),
+          ),
+        }
+      : {}),
+  };
+}
+
+function receiptAction(id: string) {
+  const quotedId = `'${id.replace(/'/g, "'\\''")}'`;
+  return {
+    getReceiptCmd: `deepline runs receipt ${quotedId} --key '<receipt-key>' --json`,
+  };
+}
+
+/** Same facts and commands as get; launch intentionally omits row content/statistics. */
+export function buildRunNavigation(status: PlayStatus) {
+  if (
+    !status.package?.run ||
+    !status.package.steps ||
+    !status.package.outputs
+  ) {
+    const id = status.runId;
+    return {
+      run: {
+        id,
+        playName: status.playName ?? status.name,
+        status: status.status,
+      },
+      datasets: [],
+      actions: {
+        ...lifecycleActions(status, id, status.status),
+        getRunCmd: `deepline runs get ${id} --json`,
+        getFullRunCmd: `deepline runs get ${id} --full --json`,
+        getLogsCmd: `deepline runs logs ${id} --limit 100 --json`,
+        ...receiptAction(id),
+      },
+    };
+  }
+  const { run, datasets, actions } = buildRunOverview(status);
+  return {
+    run,
+    datasets: datasets.map(
+      ({ datasetId, name, storage, exportUnavailable }) => ({
+        datasetId,
+        name,
+        storage,
+        ...(exportUnavailable ? { exportUnavailable } : {}),
+      }),
+    ),
+    actions,
+  };
+}

package/dist/bundling-sources/sdk/src/runs/receipts.ts

@@ -0,0 +1,23 @@
+/** A retained result, not a new tool execution or a billing receipt. */
+export type RunToolReceiptResult = {
+  receiptKey: string;
+  status?: string;
+} & (
+  | {
+      result: {
+        status: string;
+        job_id?: string;
+        toolResponse: Record<string, unknown>;
+        targetValues?: Record<string, unknown>;
+        listDatasets?: Record<string, unknown>;
+        listRows?: Record<string, unknown>;
+      };
+      error?: never;
+    }
+  | { result?: never; error: { code: string; message: string } }
+);
+
+export type RunToolReceiptsResponse = {
+  runId: string;
+  receipts: RunToolReceiptResult[];
+};

package/dist/bundling-sources/sdk/src/tool-calls.ts

@@ -168,11 +168,25 @@ export type ExecutionByKeyResult = {
  */
 const APIFY_SYNC_DEFAULT_TIMEOUT_MS = 300_000;
 const APIFY_SYNC_RESPONSE_GRACE_MS = 90_000;
+/**
+ * JSearch inherits the server HTTP owner's 120s per-attempt deadline and one
+ * eligible network retry, with at most 250ms backoff. HTTP errors and timeouts
+ * are not retried by this action. Add the existing response-policy 15s grace.
+ * This encloses provider transport with usual route headroom, not the entire
+ * route: auth, catalog and financial awaits have no shared aggregate deadline.
+ * Keep this client-only policy beside the SDK resolver; it changes no provider
+ * request, retry, admission or billing behavior.
+ */
+const JSEARCH_SEARCH_CLIENT_TIMEOUT_MS = 2 * 120_000 + 250 + 15_000;
 function resolveToolExecuteTimeoutMs(
   toolId: string,
   input: Record<string, unknown>,
+  configuredRequestTimeoutMs?: number,
 ): number | undefined {
   const normalized = toolId.trim().toLowerCase();
+  if (normalized === 'openwebninja_jsearch_search') {
+    return configuredRequestTimeoutMs ?? JSEARCH_SEARCH_CLIENT_TIMEOUT_MS;
+  }
   const lookupTimeoutMs = resolveLookupClientTimeoutMs(normalized);
   if (lookupTimeoutMs !== undefined) return lookupTimeoutMs;
   if (normalized === 'apify_run_actor_sync') {
@@ -304,6 +318,7 @@ export class ToolCalls {
   constructor(
     private readonly http: HttpClient,
     private readonly defaultRequestTimeoutMs: number,
+    private readonly configuredRequestTimeoutMs?: number,
   ) {}
 
   async call<TData = unknown, TMeta = Record<string, unknown>>(
@@ -342,7 +357,12 @@ export class ToolCalls {
       await options?.onExecution?.({ idempotencyKey });
     }
     const timeoutMs =
-      options?.timeout ?? resolveToolExecuteTimeoutMs(toolId, inputSnapshot);
+      options?.timeout ??
+      resolveToolExecuteTimeoutMs(
+        toolId,
+        inputSnapshot,
+        this.configuredRequestTimeoutMs,
+      );
     if (idempotencyKey !== undefined) {
       await this.lookupExecutionByKey(idempotencyKey, timeoutMs);
     }

package/dist/bundling-sources/sdk/src/types.ts

@@ -747,6 +747,14 @@ export interface PlayRunPackage {
   schemaVersion: 1;
   /** Package discriminator. */
   kind: 'play_run';
+  /** Retained return structure with dataset references, not flattened output previews. */
+  result?: unknown;
+  /** Full-only values are omitted as a whole, never replaced with partial authored data. */
+  resultState?:
+    | 'available'
+    | 'available_in_full'
+    | 'not_returned'
+    | 'unavailable';
   /** Ledger position of the detailed snapshot, when provided by this server. */
   source?: { ledgerNextSeq: number | null };
   /** Legacy status evidence returned by older servers. */
@@ -756,6 +764,8 @@ export interface PlayRunPackage {
   /** Run identity, status, timing, and dashboard metadata. */
   run: {
     id: string;
+    /** Revision pinned at admission, never the play's current revision. */
+    revisionId?: string;
     playName: string;
     status: string;
     dashboardUrl?: string;
@@ -785,6 +795,8 @@ export interface PlayRunPackage {
     kind: 'dataset';
     datasetId?: string;
     path: string;
+    /** Other proven catalog/return paths for this same identity. */
+    aliases?: string[];
     tableNamespace?: string;
     rowCount?: number;
     /** The materialized dataset count is exact only after this lifecycle completes. */
@@ -802,6 +814,7 @@ export interface PlayRunPackage {
       reason:
         | 'empty_dataset'
         | 'shared_table_namespace'
+        | 'ambiguous_selector'
         | 'returned_dataset_not_persisted';
       message: string;
     };
@@ -1840,7 +1853,9 @@ export interface PlayCheckRecognizedSummary {
   triggers?: PlayCheckTriggersSummary;
   tools?: string[];
   /**
-   * Durable datasets the play produces. `undrawnColumns` echoes the columns the
+   * Durable datasets the play produces. `columns` contains statically known
+   * physical SQL column names, not logical step ids or a complete CSV schema.
+   * `undrawnColumns` echoes the authored field identifiers the
    * author declared out of an authored `@mermaid` diagram with
    * `.run({ undrawnColumns: [...] })`, so an opt-out is visible in check output.
    */

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -323,6 +323,24 @@
     "failureAlert": "github_workflow_run",
     "domainReportRequired": true
   },
+  {
+    "id": "github.runtime-event-retention",
+    "name": "Runtime Event Retention",
+    "scheduler": "github_actions",
+    "cadence": "17 7 * * *",
+    "timezone": "UTC",
+    "source": ".github/workflows/runtime-event-retention.yml",
+    "locator": "schedule:",
+    "owner": "runtime",
+    "criticality": "operational",
+    "purpose": "Expire Preview and production Play runtime search rows at their declared seven- and thirty-day boundaries.",
+    "successDefinition": "Both target-verified cleanup jobs finish and prove no rows remain before their fixed cutoff.",
+    "failureDefinition": "A database target is ambiguous, a delete batch fails, the backlog does not converge, or either cleanup job times out.",
+    "startGraceMs": 600000,
+    "runTimeoutMs": 1800000,
+    "failureAlert": "github_workflow_run",
+    "domainReportRequired": true
+  },
   {
     "id": "github.runtime-shape",
     "name": "Twice-Daily Play Runtime Dataset Matrix",
@@ -377,7 +395,7 @@
     "successDefinition": "Due financial work settles or is durably parked with recovery, witnesses are recorded, and backlog advances.",
     "failureDefinition": "A ledger/provider mismatch is unresolved, parked work lacks recovery, or continuation cannot advance.",
     "startGraceMs": 300000,
-    "runTimeoutMs": 1800000,
+    "runTimeoutMs": 300000,
     "failureAlert": "vercel_route_wrapper",
     "domainReportRequired": true
   },

package/dist/bundling-sources/shared_libs/play-data-plane/column-names.ts

@@ -1,14 +1,20 @@
 const POSTGRES_IDENTIFIER_MAX_BYTES = 63;
 const HASH_SUFFIX_LENGTH = 10;
 
-function normalizePlayColumnName(id: string): string {
+function normalizePlayColumnName(
+  id: string,
+  preserveAuthoredPrefix = true,
+): string {
+  const prefix = preserveAuthoredPrefix
+    ? (id.trim().match(/^_+/)?.[0] ?? '')
+    : '';
   const normalized = id
     .trim()
     .replace(/\.+/g, '__')
     .replace(/[^A-Za-z0-9_]+/g, '_')
     .replace(/^_+|_+$/g, '')
     .toLowerCase();
-  const safe = normalized || 'column';
+  const safe = `${prefix}${normalized || 'column'}`;
   return /^[A-Za-z_]/.test(safe) ? safe : `c_${safe}`;
 }
 
@@ -48,10 +54,18 @@ export function sqlSafePlayColumnName(id: string): string {
   return `${prefix}_${hash}`;
 }
 
+/** Physical debug column only; never use this as a step or checkpoint id. */
+export function sqlSafePlayStepColumnName(
+  parent: string,
+  step: string,
+): string {
+  return sqlSafePlayColumnName(`_${parent}__step__${step}`);
+}
+
 export function legacyPostgresTruncatedPlayColumnName(
   id: string,
 ): string | null {
-  const prefixed = normalizePlayColumnName(id);
+  const prefixed = normalizePlayColumnName(id, false);
   if (prefixed.length <= POSTGRES_IDENTIFIER_MAX_BYTES) {
     return null;
   }