deepline 0.3.87 → 0.3.89

package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts

@@ -0,0 +1,263 @@
+/**
+ * Shared, provider-agnostic vocabulary for a durable monitor operation.
+ *
+ * A batch operation is deliberately a small control document.  It names a
+ * stable scope and a retry key; monitor definitions remain the source of
+ * truth for every individual item and the retained operation evidence is only
+ * a log of what the runner observed or did.  Keeping this vocabulary pure
+ * makes CLI/API parsing, Convex admission and the background runner agree on
+ * the exact same request identity.
+ */
+
+import { requireMonitorDeactivationReason } from './deactivation-reasons';
+
+export const PROVIDER_MONITOR_BATCH_OPERATION_KINDS = [
+  'deploy',
+  'deactivate',
+  'reactivate',
+  'audit',
+  'repair',
+] as const;
+
+export type ProviderMonitorBatchOperationKind =
+  (typeof PROVIDER_MONITOR_BATCH_OPERATION_KINDS)[number];
+
+export const PROVIDER_MONITOR_BATCH_TARGETS = [
+  'all',
+  'all-active',
+  'billing-suspended',
+  'needs-repair',
+] as const;
+
+export type ProviderMonitorBatchTarget =
+  (typeof PROVIDER_MONITOR_BATCH_TARGETS)[number];
+
+/**
+ * A manifest lives in durable blob storage.  The runner reads it in bounded
+ * pages; never embed its definitions or a growing result array in a run
+ * record.  `count` is declared at upload time and is used for honest progress
+ * before the first item has been processed.
+ */
+export type ProviderMonitorBatchManifestScope = {
+  kind: 'manifest';
+  storageId: string;
+  count: number;
+  /** Content address used for idempotent retries; storage ids are ephemeral. */
+  sha256?: string;
+};
+
+export type ProviderMonitorBatchScope =
+  | { kind: 'keys'; monitorKeys: readonly string[] }
+  | { kind: 'target'; target: ProviderMonitorBatchTarget }
+  | { kind: 'fleet'; fleetId: string }
+  | ProviderMonitorBatchManifestScope;
+
+export type ProviderMonitorBatchOperationRequest = {
+  kind: ProviderMonitorBatchOperationKind;
+  scope: ProviderMonitorBatchScope;
+  dryRun: boolean;
+  /** Required for deactivate; absent for every other operation. */
+  deactivationReason?: string;
+};
+
+export type ProviderMonitorBatchOperationState =
+  | 'running'
+  | 'retry_scheduled'
+  | 'completed'
+  | 'failed';
+
+const IDEMPOTENCY_KEY = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/;
+
+function canonicalJson(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
+  if (value && typeof value === 'object') {
+    return `{${Object.entries(value as Record<string, unknown>)
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([key, child]) => `${JSON.stringify(key)}:${canonicalJson(child)}`)
+      .join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+/**
+ * A job id is deterministic on purpose.  Convex's existing `by_run` point
+ * lookup is then the idempotency index: the same org/key can only own one run,
+ * and its canonical request below decides whether it is a rejoin or conflict.
+ */
+export function providerMonitorBatchOperationId(input: {
+  orgId: string;
+  idempotencyKey: string;
+}): string {
+  const orgId = input.orgId.trim();
+  const idempotencyKey = input.idempotencyKey.trim();
+  if (!orgId)
+    throw new Error('A monitor batch operation requires an organization.');
+  if (!IDEMPOTENCY_KEY.test(idempotencyKey)) {
+    throw new Error(
+      'idempotencyKey must be 1-96 characters of letters, numbers, dot, underscore, or hyphen.',
+    );
+  }
+  return `monitor-batch:${orgId}:${idempotencyKey}`;
+}
+
+export function isProviderMonitorBatchOperationKind(
+  value: string,
+): value is ProviderMonitorBatchOperationKind {
+  return (PROVIDER_MONITOR_BATCH_OPERATION_KINDS as readonly string[]).includes(
+    value,
+  );
+}
+
+export function isProviderMonitorBatchTarget(
+  value: string,
+): value is ProviderMonitorBatchTarget {
+  return (PROVIDER_MONITOR_BATCH_TARGETS as readonly string[]).includes(value);
+}
+
+/**
+ * Validate the representation shared by all entry points before any state is
+ * mutated.  A request chooses exactly one selection mechanism.  Key lists are
+ * kept intentionally small because a large explicit selection is a manifest,
+ * not an unbounded mutation argument.
+ */
+export function validateProviderMonitorBatchOperationRequest(
+  request: ProviderMonitorBatchOperationRequest,
+): ProviderMonitorBatchOperationRequest {
+  if (!isProviderMonitorBatchOperationKind(request.kind)) {
+    throw new Error(`Unknown monitor batch operation '${request.kind}'.`);
+  }
+  if (!request.scope || typeof request.scope !== 'object') {
+    throw new Error('A monitor batch operation requires exactly one scope.');
+  }
+  const scope = request.scope;
+  if (scope.kind === 'keys') {
+    const keys = [...new Set(scope.monitorKeys.map((key) => key.trim()))];
+    if (
+      keys.length === 0 ||
+      keys.length !== scope.monitorKeys.length ||
+      keys.some((key) => key.length === 0)
+    ) {
+      throw new Error(
+        'Monitor key scopes must contain unique, non-empty keys.',
+      );
+    }
+    if (keys.length > 100) {
+      throw new Error(
+        'An explicit monitor batch contains at most 100 keys. Upload a manifest for larger selections.',
+      );
+    }
+    if (request.kind === 'deploy') {
+      throw new Error(
+        'Batch deploy requires a durable manifest of complete definitions.',
+      );
+    }
+    if (request.kind === 'deactivate' && !request.deactivationReason) {
+      throw new Error(
+        'Batch deactivate requires a declared deactivation reason.',
+      );
+    }
+    if (request.kind === 'deactivate') {
+      requireMonitorDeactivationReason(request.deactivationReason!);
+    }
+    return { ...request, scope: { kind: 'keys', monitorKeys: keys } };
+  }
+  if (scope.kind === 'target') {
+    if (request.kind === 'deploy') {
+      throw new Error(
+        'Batch deploy requires a durable manifest of complete definitions.',
+      );
+    }
+    if (!isProviderMonitorBatchTarget(scope.target)) {
+      throw new Error(`Unknown monitor batch target '${scope.target}'.`);
+    }
+    if (request.kind === 'deactivate' && !request.deactivationReason) {
+      throw new Error(
+        'Batch deactivate requires a declared deactivation reason.',
+      );
+    }
+    if (request.kind === 'deactivate') {
+      requireMonitorDeactivationReason(request.deactivationReason!);
+    }
+    return request;
+  }
+  if (scope.kind === 'fleet') {
+    if (request.kind === 'deploy') {
+      throw new Error(
+        'Batch deploy requires a durable manifest of complete definitions.',
+      );
+    }
+    if (!scope.fleetId.trim())
+      throw new Error('A Fleet scope requires fleetId.');
+    if (request.kind === 'deactivate' && !request.deactivationReason) {
+      throw new Error(
+        'Batch deactivate requires a declared deactivation reason.',
+      );
+    }
+    if (request.kind === 'deactivate') {
+      requireMonitorDeactivationReason(request.deactivationReason!);
+    }
+    return {
+      ...request,
+      scope: { kind: 'fleet', fleetId: scope.fleetId.trim() },
+    };
+  }
+  if (scope.kind === 'manifest') {
+    if (
+      !scope.storageId.trim() ||
+      !Number.isSafeInteger(scope.count) ||
+      scope.count < 1
+    ) {
+      throw new Error(
+        'A manifest scope requires a storage id and a positive item count.',
+      );
+    }
+    if (request.kind !== 'deploy') {
+      throw new Error('Only batch deploy accepts a manifest scope.');
+    }
+    if (
+      scope.sha256 !== undefined &&
+      !/^[a-f0-9]{64}$/i.test(scope.sha256.trim())
+    ) {
+      throw new Error('A manifest sha256 must be a 64-character hex digest.');
+    }
+    return {
+      ...request,
+      scope: {
+        kind: 'manifest',
+        storageId: scope.storageId.trim(),
+        count: scope.count,
+        ...(scope.sha256 === undefined
+          ? {}
+          : { sha256: scope.sha256.trim().toLowerCase() }),
+      },
+    };
+  }
+  throw new Error(
+    'A monitor batch operation requires exactly one supported scope.',
+  );
+}
+
+/**
+ * The persisted fingerprint is an equality proof, not a display hash.  It is
+ * compact for normal target/key requests, collision-free, and avoids a Node
+ * crypto dependency in code shared with Convex.
+ */
+export function providerMonitorBatchOperationRequestFingerprint(
+  request: ProviderMonitorBatchOperationRequest,
+): string {
+  const normalized = validateProviderMonitorBatchOperationRequest(request);
+  // A storage id is an implementation detail of an uploaded immutable file.
+  // Re-uploading identical bytes after a lost HTTP response must rejoin the
+  // same operation, not conflict merely because storage issued another id.
+  if (normalized.scope.kind === 'manifest' && normalized.scope.sha256) {
+    return canonicalJson({
+      ...normalized,
+      scope: {
+        kind: 'manifest',
+        count: normalized.scope.count,
+        sha256: normalized.scope.sha256,
+      },
+    });
+  }
+  return canonicalJson(normalized);
+}

package/dist/bundling-sources/shared_libs/monitors/provider-monitor-identity-claims.ts

@@ -307,6 +307,14 @@ export async function retryProviderMonitorLifecycleAdmission<T>(
 export async function beginProviderMonitorLifecycleReservation(input: {
   auth: OrgAuthContext;
   monitorKey: string;
+  /**
+   * A durable bulk operation supplies its stable item attempt id.  The
+   * ordinary scalar path intentionally leaves this unset and mints a fresh
+   * UUID.  Reusing the same id lets a reclaimed batch refresh its own fence
+   * without ever turning an already-recorded provider intent into a second
+   * create.
+   */
+  operationId?: string;
   /** Tool and payload of the monitor being provisioned, so capacity can be
    *  counted against the account it watches rather than against the row. */
   entityKey?: string | null;
@@ -315,7 +323,7 @@ export async function beginProviderMonitorLifecycleReservation(input: {
   expectedControlStateRevision?: number;
   requiredMonitorStatus?: 'active';
 }): Promise<ProviderMonitorLifecycleReservation> {
-  const operationId = randomUUID();
+  const operationId = input.operationId ?? randomUUID();
   const action =
     input.state === 'provisioning'
       ? internal.providerMonitorIdentityClaims.beginLifecycleProvisioning

package/dist/bundling-sources/shared_libs/monitors/provider-monitor-state.ts

@@ -0,0 +1,122 @@
+/**
+ * The customer-visible state of a monitor. Provider work and audit evidence
+ * intentionally live on separate axes; neither may add a third value here.
+ */
+export type ProviderMonitorState = 'active' | 'inactive';
+
+declare const providerMonitorResourceIdBrand: unique symbol;
+
+/**
+ * A non-empty provider-owned resource handle.  This deliberately says
+ * nothing about the provider or its wire format: an adapter is the only
+ * layer that knows whether this happens to be a radar, subscription, etc.
+ */
+export type ProviderMonitorResourceId = string & {
+  readonly [providerMonitorResourceIdBrand]: 'ProviderMonitorResourceId';
+};
+
+/** Turn untrusted storage/API input into the proof used by active state. */
+export function providerMonitorResourceId(
+  value: string | null | undefined,
+): ProviderMonitorResourceId | null {
+  const normalized = typeof value === 'string' ? value.trim() : '';
+  return normalized === '' ? null : (normalized as ProviderMonitorResourceId);
+}
+
+/**
+ * A provider-neutral proof that this control record has an upstream
+ * counterpart. The identifier might be a radar id today and a different
+ * provider's resource id tomorrow; callers outside an adapter must never need
+ * to know which.
+ */
+export type ProviderMonitorResourceBinding =
+  | { kind: 'bound'; providerResourceId: ProviderMonitorResourceId }
+  | { kind: 'unbound'; providerResourceId: null };
+
+/**
+ * The state safe to show as a monitor's current operational state.
+ *
+ * An active intent without a resource is valid *work* (a create is pending or
+ * a historical record needs repair), but it is not an active monitor. The
+ * discriminated `active` branch therefore proves a non-empty provider id to
+ * TypeScript at every API/worker boundary that administers a live resource.
+ */
+export type OperationalProviderMonitorState =
+  | {
+      state: 'active';
+      desired: 'active';
+      providerResourceId: ProviderMonitorResourceId;
+      binding: 'bound';
+    }
+  | {
+      state: 'inactive';
+      desired: ProviderMonitorState;
+      providerResourceId: string | null;
+      binding: 'bound' | 'unbound';
+    };
+
+export type ActiveOperationalProviderMonitorState = Extract<
+  OperationalProviderMonitorState,
+  { state: 'active' }
+>;
+
+export function providerMonitorOperationalState(input: {
+  desired: ProviderMonitorState;
+  providerResourceId: string | null | undefined;
+}): OperationalProviderMonitorState {
+  const providerResourceId = providerMonitorResourceId(input.providerResourceId);
+  if (input.desired === 'active' && providerResourceId !== null) {
+    return {
+      state: 'active',
+      desired: 'active',
+      providerResourceId,
+      binding: 'bound',
+    };
+  }
+  return {
+    state: 'inactive',
+    desired: input.desired,
+    providerResourceId,
+    binding: providerResourceId === null ? 'unbound' : 'bound',
+  };
+}
+
+/**
+ * Runtime boundary for code that is about to publish or administer an active
+ * resource. Its return type is the same proof exposed to callers: an active
+ * record necessarily has a non-empty, provider-neutral resource id.
+ */
+export function requireActiveProviderMonitorBinding(input: {
+  providerResourceId: string | null | undefined;
+}): ActiveOperationalProviderMonitorState {
+  const operational = providerMonitorOperationalState({
+    desired: 'active',
+    providerResourceId: input.providerResourceId,
+  });
+  if (operational.state !== 'active') {
+    throw new Error(
+      'An active provider monitor requires a non-empty provider resource id. Keep it pending until upstream provisioning returns one.',
+    );
+  }
+  return operational;
+}
+
+/**
+ * Translate the historical control-record lifecycle label at the sole
+ * compatibility boundary. New code persists `desired` and returns this type;
+ * this mapping exists only while the online backfill drains existing rows.
+ */
+export function providerMonitorStateForLegacyStatus(
+  status: string | undefined,
+): ProviderMonitorState {
+  switch (status) {
+    case 'active':
+    case 'pending_create':
+    case 'replacement_pending':
+    case 'replacement_active_old_cleanup_pending':
+    case 'replacement_cleanup_failed':
+      return 'active';
+    default:
+      return 'inactive';
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts

@@ -101,7 +101,11 @@ export type PlayActivityState =
     };
 
 export type PlayActivityObservation = {
-  schemaVersion: 1;
+  /**
+   * Retired marker. Producers stopped writing it; envelopes still in flight
+   * from an older producer carry `1`, so readers tolerate both shapes.
+   */
+  schemaVersion?: 1;
   /** Stable within one run, for example `tool:discover_ctos` or `dataset:leads`. */
   activityId: string;
   /** Optional owning play step. */
@@ -126,7 +130,7 @@ export function isPlayActivityObservation(
 ): value is PlayActivityObservation {
   if (
     !isRecord(value) ||
-    value.schemaVersion !== 1 ||
+    (value.schemaVersion !== undefined && value.schemaVersion !== 1) ||
     typeof value.activityId !== 'string' ||
     !value.activityId.trim() ||
     typeof value.observedAt !== 'number' ||
@@ -316,7 +320,6 @@ export function createPlayActivityReporter(input: {
         type: 'activity.observed',
         observation: {
           ...observation,
-          schemaVersion: 1,
           observedAt: observation.observedAt ?? input.now?.() ?? Date.now(),
         },
       });
@@ -402,7 +405,6 @@ export function projectPlayRunActivity(input: {
   if (input.waitKind === 'detached_runner') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'runner:managed',
         target: {
           kind: 'runner',
@@ -418,7 +420,6 @@ export function projectPlayRunActivity(input: {
   if (input.waitKind === 'sleep') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'run:sleep',
         target: {
           kind: 'step',
@@ -441,7 +442,6 @@ export function projectPlayRunActivity(input: {
   ) {
     return projection(
       {
-        schemaVersion: 1,
         activityId: `event:${input.activeNodeId ?? 'run'}`,
         stepId: input.activeNodeId ?? undefined,
         target: {
@@ -468,7 +468,6 @@ export function projectPlayRunActivity(input: {
   if (normalizedStatus === 'waiting') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'run:legacy-wait',
         target: {
           kind: 'step',
@@ -506,7 +505,6 @@ export function projectPlayRunActivity(input: {
     const datasetProgress = datasetStep?.progress ?? null;
     return projection(
       {
-        schemaVersion: 1,
         activityId: `dataset:${pendingDataset.datasetId ?? pendingDataset.tableNamespace}`,
         target: {
           kind: 'dataset',
@@ -548,7 +546,6 @@ export function projectPlayRunActivity(input: {
     const provider = toolId?.split(/[._]/)[0]?.trim() || null;
     return projection(
       {
-        schemaVersion: 1,
         activityId: `step:${activeStep.nodeId}`,
         stepId: activeStep.nodeId,
         target:
@@ -577,7 +574,6 @@ export function projectPlayRunActivity(input: {
   if (normalizedStatus === 'queued') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'capacity:worker',
         target: { kind: 'capacity', pool: 'worker' },
         state: { kind: 'queued', reason: 'capacity' },
@@ -588,7 +584,6 @@ export function projectPlayRunActivity(input: {
   }
 
   const fallback: PlayActivityObservation = {
-    schemaVersion: 1,
     activityId: 'run:legacy',
     target: {
       kind: 'step',

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -4,6 +4,7 @@ import { stat } from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import type { PlaySheetContract } from '../plays/static-pipeline';
+import type { AsyncPlayRunRef } from '../plays/authoring-contract';
 import type { PlayRowUpdate } from './/ctx-types';
 import type { PlayRunTimelineEntry } from './/live-events';
 import type { PlayDatasetBornFrom } from './/cell-provenance';
@@ -179,6 +180,51 @@ type RuntimeApiRequest =
       runId: string;
       firstClaimLatencyMs: number;
     }
+  | {
+      action: 'launch_play_async';
+      key: string;
+      invocationKey: string;
+      playRef: string;
+      input: Record<string, unknown>;
+      description: string;
+      schedulerSchema?: string | null;
+      ancestryPlayIds: string[];
+    }
+  | {
+      action: 'project_run_created';
+      idempotencyKey: string;
+      playName: string;
+      playReference?: string | null;
+      workflowId: string;
+      runId: string;
+      workflowFamilyKey: string;
+      definitionScope?: 'org' | 'system';
+      revisionId?: string | null;
+      artifactStorageKey?: string | null;
+      artifactHash?: string | null;
+      graphHash?: string | null;
+      runtimeBackend?: string | null;
+      schedulerBackend?: string | null;
+      schedulerSchema?: string | null;
+      executionProfile?: string | null;
+      runtimeReleaseId?: string | null;
+      runtimeDeployVersion?: string | null;
+      runtimeCallbackBaseUrl?: string | null;
+      coordinatorUrl?: string | null;
+      coordinatorWorkerName?: string | null;
+      runtimeWorkflowName?: string | null;
+      runtimeHarnessWorkerName?: string | null;
+      maxCreditsPerRun?: number | null;
+      staticPipeline?: unknown;
+      source?: 'published' | 'ad_hoc' | 'draft';
+      triggerSource?: 'webhook' | 'cron' | 'sql_listener' | 'api';
+      inputFileId?: string;
+      inputBytes?: number;
+      inputSha256?: string;
+      replayedFromRunId?: string | null;
+      secretRefs?: unknown[];
+      inputSummary?: Record<string, unknown>;
+    }
   | ({
       action: 'save_results';
     } & RuntimeSaveResults)
@@ -719,7 +765,8 @@ function isRetryableAppRuntimeAction(
     action === 'release_runtime_step_receipt' ||
     action === 'save_results' ||
     action === 'skip_runtime_step_receipt' ||
-    action === 'start_run'
+    action === 'start_run' ||
+    action === 'launch_play_async'
   );
 }
 
@@ -1073,7 +1120,8 @@ export class AppRuntimeApiResponseError extends Error {
     this.requestId = input.requestId?.trim() || null;
     this.retryable = input.retryable;
     this.retryAfterMs =
-      typeof input.retryAfterMs === 'number' && Number.isFinite(input.retryAfterMs)
+      typeof input.retryAfterMs === 'number' &&
+      Number.isFinite(input.retryAfterMs)
         ? Math.max(0, Math.floor(input.retryAfterMs))
         : null;
     this.detail = input.detail;
@@ -1963,16 +2011,14 @@ export async function appendRunEventsViaAppRuntime(
   let terminalBatchIndex = -1;
   for (let index = batches.length - 1; index >= 0; index -= 1) {
     if (
-      batches[index]!.some(
-        (event) => {
-          const key = event.key ?? event.type;
-          return (
-            key === 'run.completed' ||
-            key === 'run.failed' ||
-            key === 'run.cancelled'
-          );
-        },
-      )
+      batches[index]!.some((event) => {
+        const key = event.key ?? event.type;
+        return (
+          key === 'run.completed' ||
+          key === 'run.failed' ||
+          key === 'run.cancelled'
+        );
+      })
     ) {
       terminalBatchIndex = index;
       break;
@@ -2021,6 +2067,19 @@ export async function startRunViaAppRuntime(
   });
 }
 
+export async function projectRunCreatedViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<RuntimeApiRequest, { action: 'project_run_created' }>,
+    'action'
+  >,
+): Promise<void> {
+  await postAppRuntimeApi<{ ok: true; status: 'queued' }>(context, {
+    action: 'project_run_created',
+    ...input,
+  });
+}
+
 export async function recordFirstClaimLatencyViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: { runId: string; firstClaimLatencyMs: number },
@@ -2031,6 +2090,25 @@ export async function recordFirstClaimLatencyViaAppRuntime(
   });
 }
 
+export async function launchPlayAsyncViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: {
+    key: string;
+    invocationKey: string;
+    playRef: string;
+    input: Record<string, unknown>;
+    description: string;
+    schedulerSchema?: string | null;
+    ancestryPlayIds: readonly string[];
+  },
+): Promise<AsyncPlayRunRef> {
+  return await postAppRuntimeApi<AsyncPlayRunRef>(context, {
+    action: 'launch_play_async',
+    ...input,
+    ancestryPlayIds: [...input.ancestryPlayIds],
+  });
+}
+
 export async function saveResultsViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: RuntimeSaveResults,