deepline 0.1.198 → 0.1.200

package/dist/bundling-sources/sdk/src/release.ts

@@ -106,10 +106,10 @@ export const SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: '0.1.198',
+  version: '0.1.200',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.198',
+    latest: '0.1.200',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/sdk/src/worker-play-entry.ts

@@ -112,9 +112,6 @@ class WorkerStepProgram<Input, Output, ReturnValue> implements StepProgram<
       | StepProgramResolver<Output, Value>,
     options?: StepOptions<Output, Value>,
   ): StepProgram<Input, Output & Record<Name, Value | null>, ReturnValue> {
-    if (!name.trim()) {
-      throw new Error('Step name required.');
-    }
     const stepResolver =
       options?.runIf && !isConditionalStepResolver(resolver)
         ? new WorkerConditionalStepResolver(
@@ -155,11 +152,7 @@ class WorkerStepProgram<Input, Output, ReturnValue> implements StepProgram<
 function isConditionalStepResolver(
   value: unknown,
 ): value is ConditionalStepResolver<unknown, unknown> {
-  return (
-    value !== null &&
-    typeof value === 'object' &&
-    (value as { kind?: unknown }).kind === 'conditional'
-  );
+  return (value as { kind?: unknown } | null)?.kind === 'conditional';
 }
 
 export function steps<TInput>(): StepProgram<TInput, TInput, TInput> {
@@ -176,9 +169,6 @@ export function runIf<Row, Value>(
 export function defineInput<TInput>(
   schema: Record<string, unknown>,
 ): PlayInputContract<TInput> {
-  if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
-    throw new Error('defineInput schema must be an object');
-  }
   return { schema };
 }
 
@@ -220,24 +210,6 @@ export function definePlay<TInput, TOutput extends PlayReturnObject>(
   const name = config.name;
   const fn = config.fn;
   const description = config.description?.trim();
-  if (typeof fn !== 'function') {
-    throw new Error('definePlay run must be async');
-  }
-  if (name.includes('/')) {
-    throw new Error('Play name cannot contain /');
-  }
-  const normalizedName = name
-    .trim()
-    .replace(/[^a-z0-9]+/gi, '_')
-    .replace(/_+/g, '_')
-    .replace(/^_+|_+$/g, '')
-    .toLowerCase();
-  if (!normalizedName) {
-    throw new Error('Play name required');
-  }
-  if (normalizedName.length > 63) {
-    throw new Error('Play name >63');
-  }
 
   const metadata: PlayMetadata = {
     name,
@@ -248,37 +220,21 @@ export function definePlay<TInput, TOutput extends PlayReturnObject>(
   };
   const play = fn as DefinedPlay<TInput, TOutput>;
 
-  Object.defineProperty(play, PLAY_METADATA_SYMBOL, {
-    value: metadata,
-  });
-  Object.defineProperty(play, 'playName', {
-    value: name,
-    enumerable: true,
+  Object.defineProperties(play, {
+    [PLAY_METADATA_SYMBOL]: { value: metadata },
+    playName: { value: name, enumerable: true },
+    bindings: { value: config.bindings, enumerable: true },
+    name: { value: name },
   });
-  Object.defineProperty(play, 'bindings', {
-    value: config.bindings,
-    enumerable: true,
-  });
-
-  Object.defineProperty(play, 'name', { value: name });
   return play;
 }
 
 export const defineWorkflow = definePlay;
 
 export function getDefinedPlayMetadata(value: unknown): PlayMetadata | null {
-  if (typeof value !== 'function') {
-    return null;
-  }
-  const metadata = (value as unknown as Record<PropertyKey, unknown>)[
-    PLAY_METADATA_SYMBOL
-  ];
-  if (!metadata || typeof metadata !== 'object') {
-    return null;
-  }
-  const candidate = metadata as PlayMetadata;
-  if (!candidate.name || typeof candidate.name !== 'string') {
-    return null;
-  }
-  return candidate;
+  return typeof value === 'function'
+    ? (((value as unknown as Record<PropertyKey, unknown>)[
+        PLAY_METADATA_SYMBOL
+      ] as PlayMetadata | undefined) ?? null)
+    : null;
 }

package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts

@@ -0,0 +1,355 @@
+export type AdaptiveToolDispatchErrorKind = 'rate_limit' | 'other';
+
+export type AdaptiveToolDispatchError = {
+  kind: AdaptiveToolDispatchErrorKind;
+  retryAfterMs?: number | null;
+};
+
+export type AdaptiveToolDispatcherOptions = {
+  initialRequestsPerSecond: number;
+  maxRequestsPerSecond: number;
+  maxConcurrency: number;
+  minConcurrency?: number;
+  additiveIncreasePerWindow?: number;
+  cleanWindowMs?: number;
+  defaultLatencyMs?: number;
+  safetyFactor?: number;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  classifyError?: (error: unknown) => AdaptiveToolDispatchError | null;
+};
+
+export type AdaptiveToolDispatchResult<TJob, TResult> = {
+  job: TJob;
+  index: number;
+  result: TResult | null;
+  latencyMs: number | null;
+  error?: unknown;
+};
+
+export type AdaptiveToolDispatcherSnapshot = {
+  currentRequestsPerSecond: number;
+  targetConcurrency: number;
+  inFlight: number;
+  queued: number;
+  maxObservedInFlight: number;
+  successCount: number;
+  errorCount: number;
+  rateLimitCount: number;
+  latencyP95Ms: number | null;
+  cooldownUntilMs: number | null;
+};
+
+type QueuedJob<TJob> = {
+  job: TJob;
+  index: number;
+};
+
+function positiveInteger(value: number, fallback: number): number {
+  return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
+}
+
+function clamp(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}
+
+function percentile(
+  values: readonly number[],
+  percentileValue: number,
+): number {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = clamp(
+    Math.ceil(sorted.length * percentileValue) - 1,
+    0,
+    sorted.length - 1,
+  );
+  return sorted[index] ?? 0;
+}
+
+export class AdaptiveToolDispatcher<TJob, TResult> {
+  private readonly maxRequestsPerSecond: number;
+  private readonly maxConcurrency: number;
+  private readonly minConcurrency: number;
+  private readonly additiveIncreasePerWindow: number;
+  private readonly cleanWindowMs: number;
+  private readonly defaultLatencyMs: number;
+  private readonly safetyFactor: number;
+  private readonly now: () => number;
+  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly classifyError: (
+    error: unknown,
+  ) => AdaptiveToolDispatchError | null;
+  private readonly latencySamples: number[] = [];
+  private readonly launchTimestamps: number[] = [];
+  private readonly progressWaiters: Array<() => void> = [];
+  private currentRequestsPerSecond: number;
+  private lastIncreaseAtMs: number;
+  private successCountAtLastIncrease = 0;
+  private successCount = 0;
+  private errorCount = 0;
+  private rateLimitCount = 0;
+  private inFlight = 0;
+  private queued = 0;
+  private maxObservedInFlight = 0;
+  private cooldownUntilMs: number | null = null;
+
+  constructor(options: AdaptiveToolDispatcherOptions) {
+    this.maxRequestsPerSecond = positiveInteger(
+      options.maxRequestsPerSecond,
+      1,
+    );
+    this.minConcurrency = positiveInteger(options.minConcurrency ?? 1, 1);
+    this.maxConcurrency = Math.max(
+      this.minConcurrency,
+      positiveInteger(options.maxConcurrency, this.minConcurrency),
+    );
+    this.currentRequestsPerSecond = clamp(
+      positiveInteger(options.initialRequestsPerSecond, 1),
+      1,
+      this.maxRequestsPerSecond,
+    );
+    this.additiveIncreasePerWindow = positiveInteger(
+      options.additiveIncreasePerWindow ?? 5,
+      5,
+    );
+    this.cleanWindowMs = positiveInteger(options.cleanWindowMs ?? 1_000, 1_000);
+    this.defaultLatencyMs = positiveInteger(
+      options.defaultLatencyMs ?? 1_000,
+      1_000,
+    );
+    this.safetyFactor =
+      Number.isFinite(options.safetyFactor) && options.safetyFactor! > 0
+        ? options.safetyFactor!
+        : 1.5;
+    this.now = options.now ?? Date.now;
+    this.sleep =
+      options.sleep ??
+      ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.classifyError = options.classifyError ?? (() => null);
+    this.lastIncreaseAtMs = this.now();
+  }
+
+  async dispatch(
+    jobs: readonly TJob[],
+    execute: (job: TJob) => Promise<TResult>,
+  ): Promise<Array<AdaptiveToolDispatchResult<TJob, TResult>>> {
+    const queue = jobs.map((job, index) => ({ job, index }));
+    const results = new Array<AdaptiveToolDispatchResult<TJob, TResult>>(
+      jobs.length,
+    );
+    this.queued = queue.length;
+
+    while (queue.length > 0 || this.inFlight > 0) {
+      let launched = false;
+      while (queue.length > 0 && this.canLaunch()) {
+        const next = queue.shift();
+        if (!next) break;
+        this.launch(next, execute, results);
+        launched = true;
+      }
+      this.queued = queue.length;
+      if (queue.length === 0 && this.inFlight === 0) break;
+      if (!launched) {
+        await this.waitForProgress(this.nextLaunchDelayMs());
+      }
+    }
+
+    return results.filter(
+      (result): result is AdaptiveToolDispatchResult<TJob, TResult> =>
+        result !== undefined,
+    );
+  }
+
+  snapshot(): AdaptiveToolDispatcherSnapshot {
+    return {
+      currentRequestsPerSecond: this.currentRequestsPerSecond,
+      targetConcurrency: this.targetConcurrency(),
+      inFlight: this.inFlight,
+      queued: this.queued,
+      maxObservedInFlight: this.maxObservedInFlight,
+      successCount: this.successCount,
+      errorCount: this.errorCount,
+      rateLimitCount: this.rateLimitCount,
+      latencyP95Ms: this.latencyP95Ms(),
+      cooldownUntilMs: this.cooldownUntilMs,
+    };
+  }
+
+  private canLaunch(): boolean {
+    return (
+      this.inFlight < this.targetConcurrency() &&
+      this.now() >= (this.cooldownUntilMs ?? 0) &&
+      this.startsAvailable() > 0
+    );
+  }
+
+  private launch(
+    queued: QueuedJob<TJob>,
+    execute: (job: TJob) => Promise<TResult>,
+    results: Array<AdaptiveToolDispatchResult<TJob, TResult> | undefined>,
+  ): void {
+    const startedAt = this.now();
+    this.launchTimestamps.push(startedAt);
+    this.inFlight += 1;
+    this.maxObservedInFlight = Math.max(
+      this.maxObservedInFlight,
+      this.inFlight,
+    );
+
+    let execution: Promise<TResult>;
+    try {
+      execution = Promise.resolve(execute(queued.job));
+    } catch (error) {
+      execution = Promise.reject(error);
+    }
+
+    void execution
+      .then((result) => {
+        const latencyMs = Math.max(0, this.now() - startedAt);
+        this.observeSuccess(latencyMs);
+        results[queued.index] = {
+          job: queued.job,
+          index: queued.index,
+          result,
+          latencyMs,
+        };
+      })
+      .catch((error) => {
+        this.observeError(error);
+        results[queued.index] = {
+          job: queued.job,
+          index: queued.index,
+          result: null,
+          latencyMs: Math.max(0, this.now() - startedAt),
+          error,
+        };
+      })
+      .finally(() => {
+        this.inFlight = Math.max(0, this.inFlight - 1);
+        this.signalProgress();
+      });
+  }
+
+  private observeSuccess(latencyMs: number): void {
+    this.successCount += 1;
+    this.latencySamples.push(latencyMs);
+    if (this.latencySamples.length > 128) {
+      this.latencySamples.splice(0, this.latencySamples.length - 128);
+    }
+    const currentTime = this.now();
+    if (this.cooldownUntilMs != null && this.cooldownUntilMs > currentTime) {
+      this.successCountAtLastIncrease = this.successCount;
+      return;
+    }
+    this.cooldownUntilMs = null;
+    const successesSinceIncrease =
+      this.successCount - this.successCountAtLastIncrease;
+    if (
+      currentTime - this.lastIncreaseAtMs >= this.cleanWindowMs &&
+      successesSinceIncrease >= this.currentRequestsPerSecond
+    ) {
+      this.currentRequestsPerSecond = Math.min(
+        this.maxRequestsPerSecond,
+        this.currentRequestsPerSecond + this.additiveIncreasePerWindow,
+      );
+      this.lastIncreaseAtMs = currentTime;
+      this.successCountAtLastIncrease = this.successCount;
+    }
+  }
+
+  private observeError(error: unknown): void {
+    this.errorCount += 1;
+    let classified: AdaptiveToolDispatchError | null = null;
+    try {
+      classified = this.classifyError(error);
+    } catch {
+      classified = null;
+    }
+    if (classified?.kind !== 'rate_limit') return;
+    this.rateLimitCount += 1;
+    this.currentRequestsPerSecond = Math.max(
+      1,
+      Math.floor(this.currentRequestsPerSecond / 2),
+    );
+    const retryAfterMs = Math.max(0, classified.retryAfterMs ?? 0);
+    const currentTime = this.now();
+    const cooldownUntilMs = currentTime + retryAfterMs;
+    this.cooldownUntilMs =
+      this.cooldownUntilMs == null
+        ? cooldownUntilMs
+        : Math.max(this.cooldownUntilMs, cooldownUntilMs);
+    this.lastIncreaseAtMs = this.cooldownUntilMs;
+    this.successCountAtLastIncrease = this.successCount;
+  }
+
+  private targetConcurrency(): number {
+    const latencyMs = this.latencyP95Ms() ?? this.defaultLatencyMs;
+    const target = Math.ceil(
+      this.currentRequestsPerSecond *
+        (Math.max(1, latencyMs) / 1_000) *
+        this.safetyFactor,
+    );
+    return clamp(target, this.minConcurrency, this.maxConcurrency);
+  }
+
+  private latencyP95Ms(): number | null {
+    return this.latencySamples.length > 0
+      ? percentile(this.latencySamples, 0.95)
+      : null;
+  }
+
+  private startsAvailable(): number {
+    this.pruneLaunchTimestamps();
+    return Math.max(
+      0,
+      this.currentRequestsPerSecond - this.launchTimestamps.length,
+    );
+  }
+
+  private nextLaunchDelayMs(): number {
+    const currentTime = this.now();
+    const cooldownWait =
+      this.cooldownUntilMs != null
+        ? Math.max(0, this.cooldownUntilMs - currentTime)
+        : 0;
+    this.pruneLaunchTimestamps();
+    const oldest = this.launchTimestamps[0];
+    const tokenWait =
+      oldest == null ||
+      this.launchTimestamps.length < this.currentRequestsPerSecond
+        ? 0
+        : Math.max(1, oldest + 1_000 - currentTime);
+    return Math.max(1, cooldownWait, tokenWait);
+  }
+
+  private pruneLaunchTimestamps(): void {
+    const cutoff = this.now() - 1_000;
+    while (
+      this.launchTimestamps.length > 0 &&
+      (this.launchTimestamps[0] ?? 0) <= cutoff
+    ) {
+      this.launchTimestamps.shift();
+    }
+  }
+
+  private async waitForProgress(delayMs: number): Promise<void> {
+    let waiter: (() => void) | null = null;
+    await Promise.race([
+      new Promise<void>((resolve) => {
+        waiter = resolve;
+        this.progressWaiters.push(resolve);
+      }),
+      this.sleep(delayMs),
+    ]);
+    if (waiter) {
+      const index = this.progressWaiters.indexOf(waiter);
+      if (index >= 0) this.progressWaiters.splice(index, 1);
+    }
+  }
+
+  private signalProgress(): void {
+    const waiters = this.progressWaiters.splice(0);
+    for (const waiter of waiters) waiter();
+  }
+}