deepline 0.3.229 → 0.3.230

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.229',
+  version: '0.3.230',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -1,4 +1,22 @@
 [
+  {
+    "id": "github.agentic-deepline-install",
+    "name": "Agentic Deepline Installation",
+    "scheduler": "github_actions",
+    "cadence": "15 7 * * *",
+    "timezone": "America/New_York",
+    "source": ".github/workflows/agentic-install.yml",
+    "locator": "schedule:",
+    "owner": "product",
+    "criticality": "critical",
+    "purpose": "Run clean Sonnet/Claude Code and Luna/Codex installs from the production homepage prompt through browser claim completion and the agent’s complete starter handoff.",
+    "successDefinition": "Both latest coding harnesses install Deepline without installation warnings or errors, finish browser authorization and setup, present all four starter choices as a card or text, and verify test key revocation; after seven valid daily runs each finishes within 240 seconds.",
+    "failureDefinition": "Either agent, browser claim completion, read-only setup verification, complete starter handoff, verified key cleanup, or the applicable timing gate fails.",
+    "startGraceMs": 900000,
+    "runTimeoutMs": 2700000,
+    "failureAlert": "github_workflow_run",
+    "domainReportRequired": true
+  },
   {
     "id": "github.billing-identity-sync",
     "name": "Billing identity lookup sync",

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -114,6 +114,16 @@ export type RuntimeStatusUpdate = {
    * attempt starts at offset zero without colliding with earlier attempts.
    */
   liveLogProducerAttempt?: number;
+  /** Sampled attempt-local calls; terminal samples supersede missed live updates. */
+  workCounters?: {
+    attempt: number;
+    invocationSeq: number;
+    segmentId: string;
+    logicalCalls: number;
+    providerRequests: number;
+    provider429s: number;
+    final: boolean;
+  };
   datasetLifecycleEvents?: Array<{
     datasetId: string;
     path: string;
@@ -2205,6 +2215,15 @@ async function updateRunStatusViaAppRuntimeUnlocked(
     now: update.lastCheckpointAt ?? Date.now(),
     source: 'worker',
   });
+  if (update.workCounters) {
+    events.push({
+      type: 'work.counters',
+      runId: update.playId,
+      source: 'worker',
+      occurredAt: update.lastCheckpointAt ?? Date.now(),
+      ...update.workCounters,
+    });
+  }
   for (const event of update.datasetLifecycleEvents ?? []) {
     const current = previousSnapshot.datasetsById[event.datasetId];
     const changesSnapshot =

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -202,6 +202,7 @@ import {
   TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
   TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER,
   TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER,
+  TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER,
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
   TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS,
@@ -2634,9 +2635,7 @@ const WAITING_ROW = Symbol('deepline.waiting_row');
 const FAILED_ROW = Symbol('deepline.failed_row');
 const COMPLETED_ROW = Symbol('deepline.completed_row');
 type MapRowExecutionResult =
-  | typeof COMPLETED_ROW
-  | typeof WAITING_ROW
-  | typeof FAILED_ROW;
+  typeof COMPLETED_ROW | typeof WAITING_ROW | typeof FAILED_ROW;
 
 function normalizeFetchHeaders(
   headers: RequestInit['headers'],
@@ -4434,8 +4433,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     artifactTableNamespace: string;
     mapNodeId?: string | null;
     explicitKey?:
-      | ((row: Record<string, unknown>, index: number) => string)
-      | null;
+      ((row: Record<string, unknown>, index: number) => string) | null;
   }): MapExecutionScope {
     const mapInvocationId = `${input.logicalNamespace}:${this.mapInvocationIndex}`;
     this.mapInvocationIndex += 1;
@@ -4685,12 +4683,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       // run actually owns. The map/cohort publisher keeps the claim-free
       // batched write, where the pointer genuinely shares the canonical
       // result's single bounded request.
-      if (
-        !(
-          this.#options.claimRuntimeStepReceipt ||
-          this.#options.claimRuntimeStepReceipts
-        )
-      ) {
+      if (!(
+        this.#options.claimRuntimeStepReceipt ||
+        this.#options.claimRuntimeStepReceipts
+      )) {
         return;
       }
       const existing = await this.getRuntimeStepReceipts(
@@ -8474,8 +8470,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     _key: string,
     _items: PlayDatasetInput<T>,
     _input?:
-      | MapFieldDefinition<T, Record<string, unknown>>
-      | RuntimeStepProgram,
+      MapFieldDefinition<T, Record<string, unknown>> | RuntimeStepProgram,
     _options?: RuntimeDatasetOptions<T>,
   ): never {
     void _key;
@@ -8599,8 +8594,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
     const userKeyOption = options?.key;
     let explicitKeyResolver:
-      | ((row: Record<string, unknown>, index: number) => string)
-      | null = null;
+      ((row: Record<string, unknown>, index: number) => string) | null = null;
     if (userKeyOption !== undefined) {
       explicitKeyResolver = (row, index) => {
         const inputRow = stripFieldOutputs(row);
@@ -13007,6 +13001,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const callIndex = this.toolCallIndexByKey.get(callIndexKey) ?? 0;
     this.toolCallIndexByKey.set(callIndexKey, callIndex + 1);
     const logicalCallId = stableDigest(`${callIndexKey}:${callIndex}`);
+    this.resourceGovernor.recordLogicalToolCall();
     let executionAuthScopeDigest =
       (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
     const eventWaitHandler =
@@ -15496,6 +15491,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       // Terminal logs are intentionally bounded; keep the small aggregate in
       // stats so every retained terminal snapshot can explain queueing.
       runtimeResources: this.getRuntimeResourceSnapshot(),
+      runtimeWorkCounters: this.getRuntimeWorkCounters(),
     };
   }
 
@@ -15504,6 +15500,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     return this.resourceGovernor.snapshot();
   }
 
+  /** Attempt-local counts without serializing the diagnostic resource map. */
+  getRuntimeWorkCounters(): {
+    logicalCalls: number;
+    providerRequests: number;
+    provider429s: number;
+  } {
+    return this.resourceGovernor.workCounters();
+  }
+
   /**
    * Record what one receipt gateway response said about itself.
    *
@@ -16680,8 +16685,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               },
             })),
             buildPhysicalCall: async ({ members }) => {
-              const owned = members.map(
-                (member) => batchRequestByCallId.get(member.memberId)!,
+              const owned = members.map((member) =>
+                batchRequestByCallId.get(member.memberId)!,
               );
               const compiled = compileOwnedPhysicalBatch(owned);
               const providerActionVersion =
@@ -19007,9 +19012,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     // running for.
     const activeRowScope = rowContext.getStore();
     const memoryLedgerRowScopes: readonly (
-      | MemoryLedgerRowScope
-      | null
-      | undefined
+      MemoryLedgerRowScope | null | undefined
     )[] = options?.memoryLedgerRowScopes ?? [
       { rowKey: activeRowScope?.rowKey, fieldName: activeRowScope?.fieldName },
     ];
@@ -19852,6 +19855,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           logProvider429RetryPhase('http_dispatch_started');
                           activeProviderRequest =
                             this.beginProviderRequest(provider);
+                          // Count at dispatch, including ambiguous transport
+                          // failures that never produce a residence sample.
+                          this.resourceGovernor.recordProviderDispatch();
                           const dispatchedResponse = await fetch(url, {
                             method: 'POST',
                             signal: abortController?.signal,
@@ -19907,21 +19913,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           // it has been read and verified below.
                           response = dispatchedResponse;
                           if (dispatchedResponse.status === 429) {
-                            provider429RetryDiagnosticStartedAt =
-                              performance.now();
-                            provider429RetryDiagnosticNextInvocationAttempt =
-                              invocationAttempt + 1;
-                            logProvider429RetryPhase(
-                              'http_429_response_received',
-                            );
-                            providerRequestsInFlightAt429 =
-                              activeProviderRequest?.inFlightCount() ?? null;
                             const observedAt = Date.now();
                             const retryAfter =
                               dispatchedResponse.headers.get('retry-after');
-                            // Without a provider deadline the classifier owns
-                            // the conservative bare-429 cooldown. Do not replace
-                            // it with the generic one-second HTTP retry fallback.
+                            // Preserve absolute provider deadlines even when the
+                            // response did not carry the gateway's counter
+                            // attestation. Classification below still decides
+                            // whether this 429 may update shared provider pacing.
                             providerRetryAtMs = retryAfter?.trim()
                               ? parseToolExecuteRetryAtMs(
                                   retryAfter,
@@ -19929,6 +19927,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                                 )
                               : undefined;
                           }
+                          if (
+                            dispatchedResponse.status === 429 &&
+                            dispatchedResponse.headers.get(
+                              TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER,
+                            ) === '1'
+                          ) {
+                            this.resourceGovernor.recordProvider429();
+                            provider429RetryDiagnosticStartedAt =
+                              performance.now();
+                            provider429RetryDiagnosticNextInvocationAttempt =
+                              invocationAttempt + 1;
+                            logProvider429RetryPhase(
+                              'http_429_response_received',
+                            );
+                            providerRequestsInFlightAt429 =
+                              activeProviderRequest?.inFlightCount() ?? null;
+                          }
                           const dispatchedFetchToHeadersMs =
                             Date.now() - fetchedAt;
                           // The response writer owns these named wall times.

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -1171,6 +1171,9 @@ export interface ContextOptions {
 }
 
 export interface PlayCheckpoint {
+  /** Last completed runner invocation within the current scheduler attempt. */
+  workCounterInvocationAttempt?: number;
+  workCounterInvocationSeq?: number;
   /**
    * Clock captured when this logical run first creates its context. All
    * staleAfterSeconds receipt buckets use this value across durable resumes.

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts

@@ -566,6 +566,7 @@ export class FlyMachinesClient {
     method: string,
     path: string,
     body?: unknown,
+    leaseNonce?: string,
   ): Promise<unknown> {
     const retryDelaysMs =
       this.rateLimitOptions.retryDelaysMs ??
@@ -585,6 +586,7 @@ export class FlyMachinesClient {
             headers: {
               authorization: `Bearer ${this.token}`,
               'content-type': 'application/json',
+              ...(leaseNonce ? { 'fly-machine-lease-nonce': leaseNonce } : {}),
             },
             body: body === undefined ? undefined : JSON.stringify(body),
             signal,
@@ -689,10 +691,43 @@ export class FlyMachinesClient {
     );
   }
 
-  async deleteMachine(id: string, force: boolean): Promise<void> {
+  /** Hold exclusive provider ownership while rechecking and reclaiming a
+   * stopped Machine. A concurrent rollback cannot update its hold or restart
+   * it until this lease is released or expires. */
+  async acquireMachineLease(
+    id: string,
+    options: { ttl: number; description: string },
+  ): Promise<string> {
+    const result = await this.request('POST', `/machines/${id}/lease`, options);
+    const nonce =
+      result &&
+      typeof result === 'object' &&
+      'data' in result &&
+      result.data &&
+      typeof result.data === 'object' &&
+      'nonce' in result.data
+        ? result.data.nonce
+        : undefined;
+    if (typeof nonce !== 'string' || nonce.trim() === '') {
+      throw new Error('Fly Machines API did not return a lease nonce.');
+    }
+    return nonce;
+  }
+
+  async releaseMachineLease(id: string, nonce: string): Promise<void> {
+    await this.request('DELETE', `/machines/${id}/lease`, undefined, nonce);
+  }
+
+  async deleteMachine(
+    id: string,
+    force: boolean,
+    leaseNonce?: string,
+  ): Promise<void> {
     await this.request(
       'DELETE',
       `/machines/${id}?force=${force ? 'true' : 'false'}`,
+      undefined,
+      leaseNonce,
     );
   }
 
@@ -718,8 +753,9 @@ export class FlyMachinesClient {
   async stopMachine(
     id: string,
     options?: { signal?: string; timeout?: string },
+    leaseNonce?: string,
   ): Promise<void> {
-    await this.request('POST', `/machines/${id}/stop`, options);
+    await this.request('POST', `/machines/${id}/stop`, options, leaseNonce);
   }
 
   async machinesInLane(releaseId: string): Promise<FlyMachine[]> {

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts

@@ -12,6 +12,10 @@ import type { RunLedgerProjectionPorts } from '../../play-runs/projection/ports'
 
 export type GatewayRunLedgerProjectionAdapterInput = {
   runId: string;
+  /**
+   * Launch-pinned runtime context. Gateway authority refreshes its credentials
+   * against the active outbox lease while retaining supported transport hooks.
+   */
   runtimeApi: WorkerRuntimeApiContext;
   withClient: <T>(
     operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
@@ -20,9 +24,10 @@ export type GatewayRunLedgerProjectionAdapterInput = {
 };
 
 /**
- * Gateway Adapter for the durable Run Ledger outbox. It retains the worker's
- * no-signing-key rule and turns both normal and targeted delivery into the
- * one `RunLedgerProjection.deliver` port.
+ * Gateway adapter for the durable Run Ledger outbox. It retains the worker's
+ * no-signing-key rule, refreshes credentials for each claimed batch, and turns
+ * normal and targeted delivery into the one `RunLedgerProjection.deliver`
+ * port.
  */
 export function createGatewayRunLedgerProjectionAdapter(
   input: GatewayRunLedgerProjectionAdapterInput,

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -317,6 +317,15 @@ export type PlayRunnerLogEvent = {
 
 export type PlayRunnerEvent =
   | PlayRunnerLogEvent
+  | {
+      type: 'work_counters';
+      attempt: number;
+      invocationSeq: number;
+      segmentId: string;
+      logicalCalls: number;
+      providerRequests: number;
+      provider429s: number;
+    }
   | {
       type: 'checkpoint';
       checkpoint: PlayCheckpoint;

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -734,6 +734,12 @@ export interface RuntimeResourceGovernor {
     inFlightAtDispatch: number;
     release(): void;
   };
+  /** Count one actual tool HTTP dispatch before transport can fail ambiguously. */
+  recordProviderDispatch(): void;
+  /** Count a provider HTTP 429 as soon as response headers arrive. */
+  recordProvider429(): void;
+  /** Count one admitted ctx.tools.execute invocation, including a later failure. */
+  recordLogicalToolCall(): void;
   /**
    * Probe app transport capacity at the physical provider-dispatch edge. A
    * caller that misses it must cancel and re-admit its provider ticket.
@@ -757,6 +763,12 @@ export interface RuntimeResourceGovernor {
   }): Promise<void>;
   observe(input: RuntimeResourceObservation): void;
   snapshot(): RuntimeResourceSnapshot;
+  /** O(1) counters for sampled live status; no provider-map scan. */
+  workCounters(): {
+    logicalCalls: number;
+    providerRequests: number;
+    provider429s: number;
+  };
   /** Stop the event-loop lag sampler. The timer is unref'd either way. */
   stopSampling(): void;
   /** Smoothed event-loop lag from the sampler, or null before its first tick. */
@@ -1047,6 +1059,13 @@ export function createRuntimeResourceGovernor(input: {
   const toolCounters = {
     acquired: 0,
   };
+  // Dispatch is counted at the fetch edge. A failed transport is still a
+  // physical request attempt; response-residence observations omit it.
+  const workCounters = {
+    logicalCalls: 0,
+    providerRequests: 0,
+    provider429s: 0,
+  };
   const sheetFlushCounters = {
     acquired: 0,
     bytes: 0,
@@ -1642,6 +1661,18 @@ export function createRuntimeResourceGovernor(input: {
       };
     },
 
+    recordProviderDispatch() {
+      workCounters.providerRequests += 1;
+    },
+
+    recordProvider429() {
+      workCounters.provider429s += 1;
+    },
+
+    recordLogicalToolCall() {
+      workCounters.logicalCalls += 1;
+    },
+
     tryAcquireIntegrationRequestSlot() {
       return input.executionGovernor.tryAcquireIntegrationRequestSlot();
     },
@@ -1672,6 +1703,14 @@ export function createRuntimeResourceGovernor(input: {
 
     observe,
 
+    workCounters() {
+      return {
+        logicalCalls: workCounters.logicalCalls,
+        providerRequests: workCounters.providerRequests,
+        provider429s: workCounters.provider429s,
+      };
+    },
+
     stopSampling: stopEventLoopLagSampler,
     eventLoopLagEwmaMs: () => phases.eventLoopLag.snapshot().ewmaMs,