deepline 0.1.11 → 0.1.19

package/dist/repo/shared_libs/plays/rate-limit-scheduler.ts

@@ -1,206 +0,0 @@
-const DEFAULT_SUGGESTED_MAX_PARALLELISM = 50;
-const MIN_CONCURRENCY_WAIT_MS = 10;
-
-type BucketState = {
-  windowStartedAt: number;
-  startedInWindow: number;
-  activeCount: number;
-};
-
-export interface PlayQueueHint {
-  bucketId: string;
-  provider: string;
-  operation: string;
-  ruleId: string;
-  requestsPerWindow: number;
-  windowMs: number;
-  maxConcurrency: number | null;
-}
-
-type SchedulerOptions = {
-  now?: () => number;
-  sleep?: (ms: number) => Promise<void>;
-  maxSuggestedParallelism?: number;
-  getQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
-};
-
-export async function getSuggestedPlayToolParallelism(
-  toolId: string,
-  fallback: number,
-  maxSuggestedParallelism = DEFAULT_SUGGESTED_MAX_PARALLELISM,
-  getQueueHints: (toolId: string) => Promise<readonly PlayQueueHint[]> = async () => [],
-): Promise<number> {
-  const hints = await getQueueHints(toolId);
-  if (hints.length === 0) {
-    return fallback;
-  }
-
-  const limits = hints.flatMap((hint) =>
-    hint.maxConcurrency != null
-      ? [hint.requestsPerWindow, hint.maxConcurrency]
-      : [hint.requestsPerWindow],
-  );
-
-  return Math.max(1, Math.min(maxSuggestedParallelism, ...limits));
-}
-
-export class PlayRateLimitScheduler {
-  private readonly bucketStates = new Map<string, BucketState>();
-  private lock: Promise<void> = Promise.resolve();
-  private readonly now;
-  private readonly sleep;
-  private readonly maxSuggestedParallelism;
-  private readonly getQueueHints;
-
-  constructor(options: SchedulerOptions = {}) {
-    this.now = options.now ?? (() => Date.now());
-    this.sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
-    this.maxSuggestedParallelism =
-      options.maxSuggestedParallelism ?? DEFAULT_SUGGESTED_MAX_PARALLELISM;
-    this.getQueueHints = options.getQueueHints ?? (async () => []);
-  }
-
-  async getSuggestedParallelism(toolId: string, fallback: number): Promise<number> {
-    return await getSuggestedPlayToolParallelism(
-      toolId,
-      fallback,
-      this.maxSuggestedParallelism,
-      this.getQueueHints,
-    );
-  }
-
-  async run<T>(toolId: string, task: () => Promise<T>): Promise<T> {
-    const hints = await this.getQueueHints(toolId);
-    if (hints.length === 0) {
-      return await task();
-    }
-
-    await this.acquire(hints);
-    try {
-      return await task();
-    } finally {
-      await this.release(hints);
-    }
-  }
-
-  private async acquire(hints: readonly PlayQueueHint[]): Promise<void> {
-    const orderedHints = [...hints].sort((left, right) =>
-      left.bucketId.localeCompare(right.bucketId),
-    );
-
-    while (true) {
-      const decision = await this.withLock(() => {
-        const now = this.now();
-        let waitMs = 0;
-
-        for (const hint of orderedHints) {
-          const state = this.getBucketState(hint.bucketId, now);
-          this.resetExpiredWindow(state, hint.windowMs, now);
-
-          if (state.startedInWindow >= hint.requestsPerWindow) {
-            waitMs = Math.max(
-              waitMs,
-              state.windowStartedAt + hint.windowMs - now,
-            );
-          }
-
-          if (
-            hint.maxConcurrency != null &&
-            state.activeCount >= hint.maxConcurrency
-          ) {
-            waitMs = Math.max(waitMs, MIN_CONCURRENCY_WAIT_MS);
-          }
-        }
-
-        if (waitMs > 0) {
-          return { acquired: false, waitMs: Math.max(1, waitMs) } as const;
-        }
-
-        for (const hint of orderedHints) {
-          const state = this.getBucketState(hint.bucketId, now);
-          state.startedInWindow += 1;
-          if (hint.maxConcurrency != null) {
-            state.activeCount += 1;
-          }
-        }
-
-        return { acquired: true, waitMs: 0 } as const;
-      });
-
-      if (decision.acquired) {
-        return;
-      }
-
-      await this.sleep(decision.waitMs);
-    }
-  }
-
-  private async release(hints: readonly PlayQueueHint[]): Promise<void> {
-    await this.withLock(() => {
-      for (const hint of hints) {
-        if (hint.maxConcurrency == null) {
-          continue;
-        }
-        const state = this.bucketStates.get(hint.bucketId);
-        if (!state || state.activeCount <= 0) {
-          continue;
-        }
-        state.activeCount -= 1;
-      }
-    });
-  }
-
-  private getBucketState(bucketId: string, now: number): BucketState {
-    const existing = this.bucketStates.get(bucketId);
-    if (existing) {
-      return existing;
-    }
-
-    const created: BucketState = {
-      windowStartedAt: now,
-      startedInWindow: 0,
-      activeCount: 0,
-    };
-    this.bucketStates.set(bucketId, created);
-    return created;
-  }
-
-  private resetExpiredWindow(
-    state: BucketState,
-    windowMs: number,
-    now: number,
-  ): void {
-    if (windowMs <= 0) {
-      state.windowStartedAt = now;
-      state.startedInWindow = 0;
-      return;
-    }
-
-    if (now - state.windowStartedAt < windowMs) {
-      return;
-    }
-
-    const windowsElapsed = Math.floor((now - state.windowStartedAt) / windowMs);
-    state.windowStartedAt += windowsElapsed * windowMs;
-    state.startedInWindow = 0;
-
-    if (now - state.windowStartedAt >= windowMs) {
-      state.windowStartedAt = now;
-    }
-  }
-
-  private async withLock<T>(fn: () => T | Promise<T>): Promise<T> {
-    const previous = this.lock;
-    let releaseLock!: () => void;
-    this.lock = new Promise<void>((resolve) => {
-      releaseLock = resolve;
-    });
-
-    await previous;
-    try {
-      return await fn();
-    } finally {
-      releaseLock();
-    }
-  }
-}

package/dist/repo/shared_libs/plays/resolve-static-pipeline.ts

@@ -1,164 +0,0 @@
-import type {
-  PlaySheetContract,
-  PlayStaticPipeline,
-  PlayStaticSubstep,
-} from './static-pipeline';
-import { compileSheetContract } from './static-pipeline';
-
-export type ResolvablePlayStaticShape = {
-  staticPipeline?: PlayStaticPipeline | null;
-};
-
-export type ResolveStaticPipelineInput = {
-  rootPlayId: string;
-  root: ResolvablePlayStaticShape;
-  resolveReferencedPlay: (playId: string) => Promise<ResolvablePlayStaticShape | null>;
-  maxDepth?: number;
-};
-
-function clonePipeline(pipeline: PlayStaticPipeline): PlayStaticPipeline {
-  return {
-    ...pipeline,
-    inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
-    fields: [...pipeline.fields],
-    stages: (pipeline.stages ?? []).map((stage) => ({ ...stage })),
-    substeps: pipeline.substeps.map((substep) => ({ ...substep })),
-    sheetContract: cloneSheetContract(pipeline.sheetContract),
-    sheetContractErrors: undefined,
-  };
-}
-
-function cloneSheetContract(contract: PlaySheetContract | null | undefined) {
-  if (!contract) return contract;
-  return {
-    ...contract,
-    columns: contract.columns.map((column) => ({ ...column })),
-  };
-}
-
-function materializePipeline(shape: ResolvablePlayStaticShape | null): PlayStaticPipeline | null {
-  if (!shape) return null;
-  if (shape.staticPipeline) {
-    return clonePipeline(shape.staticPipeline);
-  }
-  return null;
-}
-
-function annotateSubstep(
-  substep: PlayStaticSubstep,
-  input: { callDepth: number; callPath: string[] },
-): PlayStaticSubstep {
-  return {
-    ...substep,
-    callDepth: input.callDepth,
-    callPath: input.callPath,
-  };
-}
-
-async function resolvePipelineSubsteps(input: {
-  steps: PlayStaticSubstep[];
-  resolveReferencedPlay: (playId: string) => Promise<ResolvablePlayStaticShape | null>;
-  stack: string[];
-  maxDepth: number;
-}): Promise<PlayStaticSubstep[]> {
-  const { steps, resolveReferencedPlay, stack, maxDepth } = input;
-  const callDepth = Math.max(0, stack.length - 1);
-
-  return await Promise.all(
-    steps.map(async (substep) => {
-      const annotated = annotateSubstep(substep, {
-        callDepth,
-        callPath: [...stack],
-      });
-
-      if (annotated.type !== 'play_call') {
-        return annotated;
-      }
-
-      if (stack.includes(annotated.playId)) {
-        return {
-          ...annotated,
-          cycleDetected: true,
-          resolutionError: `Static cycle detected: ${[...stack, annotated.playId].join(' -> ')}`,
-          pipeline: null,
-        } satisfies PlayStaticSubstep;
-      }
-
-      if (stack.length >= maxDepth) {
-        return {
-          ...annotated,
-          resolutionError: `Static nesting limit reached at ${annotated.playId}`,
-          pipeline: null,
-        } satisfies PlayStaticSubstep;
-      }
-
-      const childShape = await resolveReferencedPlay(annotated.playId);
-      const childPipeline = materializePipeline(childShape);
-      if (!childPipeline) {
-        return {
-          ...annotated,
-          resolutionError: `Unable to statically resolve play "${annotated.playId}"`,
-          pipeline: null,
-        } satisfies PlayStaticSubstep;
-      }
-
-      const resolvedChildPipeline = {
-        ...childPipeline,
-        stages: await resolvePipelineSubsteps({
-          steps: childPipeline.stages ?? [],
-          resolveReferencedPlay,
-          stack: [...stack, annotated.playId],
-          maxDepth,
-        }),
-        substeps: await resolvePipelineSubsteps({
-          steps: childPipeline.substeps,
-          resolveReferencedPlay,
-          stack: [...stack, annotated.playId],
-          maxDepth,
-        }),
-      };
-      const childContract = compileSheetContract(resolvedChildPipeline);
-
-      return {
-        ...annotated,
-        pipeline: {
-          ...resolvedChildPipeline,
-          sheetContract: childContract.contract,
-          sheetContractErrors: childContract.errors,
-        },
-      } satisfies PlayStaticSubstep;
-    }),
-  );
-}
-
-export async function resolveStaticPipelineTree(
-  input: ResolveStaticPipelineInput,
-): Promise<PlayStaticPipeline | null> {
-  const maxDepth = input.maxDepth ?? 6;
-  const rootPipeline = materializePipeline(input.root);
-  if (!rootPipeline) {
-    return null;
-  }
-
-  const resolvedPipeline = {
-    ...rootPipeline,
-    stages: await resolvePipelineSubsteps({
-      steps: rootPipeline.stages ?? [],
-      resolveReferencedPlay: input.resolveReferencedPlay,
-      stack: [input.rootPlayId],
-      maxDepth,
-    }),
-    substeps: await resolvePipelineSubsteps({
-      steps: rootPipeline.substeps,
-      resolveReferencedPlay: input.resolveReferencedPlay,
-      stack: [input.rootPlayId],
-      maxDepth,
-    }),
-  };
-  const contract = compileSheetContract(resolvedPipeline);
-  return {
-    ...resolvedPipeline,
-    sheetContract: contract.contract,
-    sheetContractErrors: contract.errors,
-  };
-}