deepline 0.3.461 → 0.3.462

package/dist/bundling-sources/shared_libs/play-runtime/free-company-lookup-batching.ts

@@ -0,0 +1,218 @@
+import {
+  defineBatchStrategyMap,
+  type BatchOperationStrategy,
+} from './batching-types';
+
+export const FREE_COMPANY_LOOKUP_KEY_TYPES = [
+  'domain',
+  'linkedin_url',
+  'company_name',
+] as const;
+export type FreeCompanyLookupKeyType =
+  (typeof FREE_COMPANY_LOOKUP_KEY_TYPES)[number];
+
+/** Snowflake bind count stays well inside one statement's limits. */
+export const FREE_COMPANY_BULK_LOOKUP_MAX_KEYS = 1_000;
+/** One SQL statement per batch keeps a row's wait under the query timeout. */
+export const FREE_COMPANY_LOOKUP_MAX_BATCH_SIZE = 100;
+
+/**
+ * The same steps the loader applies to PDL's `website` column, so a lookup
+ * matches `normalized_domain` exactly. Subdomains are kept, as they are there.
+ */
+export function normalizeFreeCompanyDomain(value: string): string {
+  return value
+    .trim()
+    .toLowerCase()
+    .replace(/^https?:\/\//, '')
+    .replace(/^www\./, '')
+    .replace(/[/?#].*$/, '')
+    .replace(/\.$/, '');
+}
+
+/** Stored as `linkedin.com/company/<slug>`; LinkedIn slugs ignore case. */
+export function normalizeFreeCompanyLinkedinUrl(value: string): string {
+  return value
+    .trim()
+    .toLowerCase()
+    .replace(/^https?:\/\//, '')
+    .replace(/^www\./, '')
+    .replace(/[?#].*$/, '')
+    .replace(/\/+$/, '');
+}
+
+/** Stored names are trimmed and lowercase, so this is case-insensitive equality. */
+export function normalizeFreeCompanyName(value: string): string {
+  return value.trim().toLowerCase();
+}
+
+export function normalizeFreeCompanyLookupKey(
+  keyType: FreeCompanyLookupKeyType,
+  value: string,
+): string {
+  switch (keyType) {
+    case 'domain':
+      return normalizeFreeCompanyDomain(value);
+    case 'linkedin_url':
+      return normalizeFreeCompanyLinkedinUrl(value);
+    case 'company_name':
+      return normalizeFreeCompanyName(value);
+  }
+}
+
+type LookupPayload = Record<string, unknown> & {
+  domain?: string;
+  linkedin_url?: string;
+  company_name?: string;
+};
+
+type BulkLookupPayload = Record<string, unknown> & {
+  domains?: string[];
+  linkedin_urls?: string[];
+  company_names?: string[];
+};
+
+export type FreeCompanyLookupResult = Record<string, unknown> & {
+  key_type: FreeCompanyLookupKeyType;
+  key: string;
+  matches: Record<string, unknown>[];
+};
+
+type BulkLookupResult = Record<string, unknown> & {
+  results?: unknown;
+};
+
+const BULK_FIELD: Record<FreeCompanyLookupKeyType, keyof BulkLookupPayload> = {
+  domain: 'domains',
+  linkedin_url: 'linkedin_urls',
+  company_name: 'company_names',
+};
+
+/** The one usable key a lookup names, or null when it does not name exactly one. */
+function usableFreeCompanyLookupKey(payload: LookupPayload): {
+  keyType: FreeCompanyLookupKeyType;
+  key: string;
+} | null {
+  const present = FREE_COMPANY_LOOKUP_KEY_TYPES.filter(
+    (keyType) => typeof payload[keyType] === 'string',
+  );
+  if (present.length !== 1) return null;
+  const keyType = present[0]!;
+  const key = normalizeFreeCompanyLookupKey(
+    keyType,
+    payload[keyType] as string,
+  );
+  return key ? { keyType, key } : null;
+}
+
+/** The single key a lookup names. Callers validate the payload first. */
+export function freeCompanyLookupKey(payload: LookupPayload): {
+  keyType: FreeCompanyLookupKeyType;
+  key: string;
+} {
+  const lookupKey = usableFreeCompanyLookupKey(payload);
+  if (!lookupKey) {
+    throw new Error(
+      'free_company_lookup needs exactly one usable domain, linkedin_url or company_name.',
+    );
+  }
+  return lookupKey;
+}
+
+function isLookupResult(value: unknown): value is FreeCompanyLookupResult {
+  const row = value as FreeCompanyLookupResult | null;
+  return Boolean(
+    row &&
+    typeof row === 'object' &&
+    FREE_COMPANY_LOOKUP_KEY_TYPES.includes(row.key_type) &&
+    typeof row.key === 'string' &&
+    Array.isArray(row.matches),
+  );
+}
+
+export const freeCompanyLookupBatchStrategy: BatchOperationStrategy<
+  LookupPayload,
+  BulkLookupPayload | LookupPayload,
+  BulkLookupResult | FreeCompanyLookupResult,
+  FreeCompanyLookupResult
+> = {
+  sourceOperation: 'free_company_lookup',
+  batchOperation: 'free_company_bulk_lookup',
+  kind: 'identifier_batch',
+  maxBatchSize: FREE_COMPANY_LOOKUP_MAX_BATCH_SIZE,
+  // Rows looking up by different keys share a batch; the bulk lookup takes all
+  // three lists in one statement.
+  canBatchWith() {
+    return true;
+  },
+  // The runtime batches before the app validates anything. A row without one
+  // usable key gets a bucket and item key of its own, so it never joins a
+  // batch of valid rows. compile sends it to free_company_lookup alone, which
+  // refuses it with a 422 for that row only.
+  toBucketKey(payload) {
+    return usableFreeCompanyLookupKey(payload)
+      ? 'free_company_bulk_lookup'
+      : `invalid:${JSON.stringify(payload)}`;
+  },
+  toItemKey(payload) {
+    const lookupKey = usableFreeCompanyLookupKey(payload);
+    return lookupKey
+      ? JSON.stringify([lookupKey.keyType, lookupKey.key])
+      : JSON.stringify(['invalid', payload]);
+  },
+  compile(payloads) {
+    const [only] = payloads;
+    if (payloads.length === 1 && only && !usableFreeCompanyLookupKey(only)) {
+      return {
+        batchOperation: 'free_company_lookup',
+        batchPayload: only,
+        items: [{ itemKey: JSON.stringify(['invalid', only]), payload: only }],
+      };
+    }
+    const batchPayload: BulkLookupPayload = {};
+    const items = payloads.map((payload) => {
+      const { keyType, key } = freeCompanyLookupKey(payload);
+      const field = BULK_FIELD[keyType];
+      const list = (batchPayload[field] ??= []) as string[];
+      if (!list.includes(key)) list.push(key);
+      return { itemKey: JSON.stringify([keyType, key]), payload };
+    });
+    return {
+      batchOperation: 'free_company_bulk_lookup',
+      batchPayload,
+      items,
+    };
+  },
+  splitResult(fullResult, compiled) {
+    if (compiled.batchOperation === 'free_company_lookup') {
+      throw new Error(
+        'free_company_lookup answered a payload without one usable key; it must refuse it.',
+      );
+    }
+    const result = ((fullResult as { data?: unknown }).data ??
+      fullResult) as BulkLookupResult;
+    if (!Array.isArray(result.results)) {
+      throw new Error('free_company_bulk_lookup returned no results list.');
+    }
+    const byKey = new Map<string, FreeCompanyLookupResult>();
+    for (const row of result.results) {
+      if (!isLookupResult(row)) {
+        throw new Error('free_company_bulk_lookup returned a malformed result.');
+      }
+      byKey.set(JSON.stringify([row.key_type, row.key]), row);
+    }
+    return compiled.items.map((item) => {
+      const row = byKey.get(item.itemKey);
+      if (!row) {
+        throw new Error(
+          `free_company_bulk_lookup omitted the result for ${item.itemKey}.`,
+        );
+      }
+      return { itemKey: item.itemKey, result: row, rawResult: row };
+    });
+  },
+};
+
+export const freeCompanyLookupBatchStrategies = defineBatchStrategyMap({
+  free_company_lookup: freeCompanyLookupBatchStrategy,
+});

package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts

@@ -1,4 +1,4 @@
-export type GatewayPostgresLane = 'ordinary' | 'commit';
+export type GatewayPostgresLane = 'ordinary' | 'commit' | 'heartbeat';
 
 export type GatewayPostgresAdmissionTenant = string | null;
 export const GATEWAY_POSTGRES_MACHINE_TENANT: GatewayPostgresAdmissionTenant =
@@ -89,6 +89,7 @@ export class GatewayPostgresAdmission {
   readonly #waiting: Record<GatewayPostgresLane, LaneQueue> = {
     ordinary: { ring: new Map(), size: 0 },
     commit: { ring: new Map(), size: 0 },
+    heartbeat: { ring: new Map(), size: 0 },
   };
   readonly #activeByTenant: Record<
     GatewayPostgresLane,
@@ -96,9 +97,11 @@ export class GatewayPostgresAdmission {
   > = {
     ordinary: new Map(),
     commit: new Map(),
+    heartbeat: new Map(),
   };
   #ordinaryActive = 0;
   #commitActive = 0;
+  #heartbeatActive = 0;
   #cancelled = 0;
   #overloaded = 0;
   #fairShareEvictions = 0;
@@ -110,8 +113,21 @@ export class GatewayPostgresAdmission {
       maxOrdinaryActive?: number;
       maxTotalActive?: number;
       maxWaitingPerLane?: number;
+      /** Capacity withheld from bulk work, within maxTotalActive. */
+      reservedHeartbeatActive?: number;
     } = {},
   ) {
+    const reserved = limits.reservedHeartbeatActive ?? 0;
+    if (
+      !Number.isSafeInteger(reserved) ||
+      reserved < 0 ||
+      reserved > 1 ||
+      reserved >= this.#maxTotalActive()
+    ) {
+      throw new Error(
+        'Gateway Postgres heartbeat reservation must be zero or one and leave bulk capacity.',
+      );
+    }
     if (
       limits.maxWaitingPerLane !== undefined &&
       (!Number.isSafeInteger(limits.maxWaitingPerLane) ||
@@ -135,7 +151,12 @@ export class GatewayPostgresAdmission {
       this.#cancelled += 1;
       throw new GatewayPostgresAdmissionCancelledError();
     }
-    if (this.#canGrant(lane) && this.#waiting.commit.size === 0) {
+    if (
+      this.#canGrant(lane) &&
+      (lane === 'heartbeat'
+        ? this.#waiting.heartbeat.size === 0
+        : this.#waiting.commit.size === 0)
+    ) {
       return this.#grant(lane, tenant);
     }
     if (this.#waiting[lane].size >= this.#maxWaitingPerLane()) {
@@ -172,15 +193,18 @@ export class GatewayPostgresAdmission {
       active: {
         ordinary: this.#ordinaryActive,
         commit: this.#commitActive,
+        heartbeat: this.#heartbeatActive,
         total: this.#activeTotal(),
       },
       waiting: {
         ordinary: this.#waiting.ordinary.size,
         commit: this.#waiting.commit.size,
+        heartbeat: this.#waiting.heartbeat.size,
       },
       waitingTenants: {
         ordinary: this.#waiting.ordinary.ring.size,
         commit: this.#waiting.commit.ring.size,
+        heartbeat: this.#waiting.heartbeat.ring.size,
       },
       cancelled: this.#cancelled,
       overloaded: this.#overloaded,
@@ -190,11 +214,17 @@ export class GatewayPostgresAdmission {
   }
 
   #activeTotal(): number {
-    return this.#ordinaryActive + this.#commitActive;
+    return this.#ordinaryActive + this.#commitActive + this.#heartbeatActive;
   }
 
   #canGrant(lane: GatewayPostgresLane): boolean {
     if (this.#activeTotal() >= this.#maxTotalActive()) return false;
+    if (lane === 'heartbeat') return this.#heartbeatActive < 1;
+    if (
+      this.#ordinaryActive + this.#commitActive >=
+      this.#maxTotalActive() - (this.limits.reservedHeartbeatActive ?? 0)
+    )
+      return false;
     return (
       lane === 'commit' || this.#ordinaryActive < this.#maxOrdinaryActive()
     );
@@ -221,7 +251,8 @@ export class GatewayPostgresAdmission {
     tenant: GatewayPostgresAdmissionTenant,
   ): () => void {
     if (lane === 'ordinary') this.#ordinaryActive += 1;
-    else this.#commitActive += 1;
+    else if (lane === 'commit') this.#commitActive += 1;
+    else this.#heartbeatActive += 1;
     const activeByTenant = this.#activeByTenant[lane];
     activeByTenant.set(tenant, (activeByTenant.get(tenant) ?? 0) + 1);
     let released = false;
@@ -229,7 +260,8 @@ export class GatewayPostgresAdmission {
       if (released) return;
       released = true;
       if (lane === 'ordinary') this.#ordinaryActive -= 1;
-      else this.#commitActive -= 1;
+      else if (lane === 'commit') this.#commitActive -= 1;
+      else this.#heartbeatActive -= 1;
       const remaining = (activeByTenant.get(tenant) ?? 1) - 1;
       if (remaining > 0) activeByTenant.set(tenant, remaining);
       else activeByTenant.delete(tenant);
@@ -239,6 +271,11 @@ export class GatewayPostgresAdmission {
 
   #drain(): void {
     while (this.#activeTotal() < this.#maxTotalActive()) {
+      if (this.#waiting.heartbeat.size > 0 && this.#canGrant('heartbeat')) {
+        this.#resolve(this.#dequeueLeastActiveTenant('heartbeat')!);
+        continue;
+      }
+      if (!this.#canGrant('commit')) return;
       const ordinaryCanRun =
         this.#waiting.ordinary.size > 0 &&
         this.#ordinaryActive < this.#maxOrdinaryActive();

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-budget-state-backend.ts

@@ -2,12 +2,16 @@ import {
   chargeGovernorBudgetViaAppRuntime,
   type WorkerRuntimeApiContext,
 } from '../app-runtime-api';
-import { BlockReservingBudgetStateBackend } from './block-reserving-budget-state-backend';
+import {
+  BlockReservingBudgetStateBackend,
+  type BlockReservingBudgetStateOptions,
+} from './block-reserving-budget-state-backend';
 
 export class AppRuntimeBudgetStateBackend extends BlockReservingBudgetStateBackend {
   constructor(
     context: WorkerRuntimeApiContext,
     schedulerSchema?: string | null,
+    options?: BlockReservingBudgetStateOptions,
   ) {
     super(
       async (input) =>
@@ -17,6 +21,7 @@ export class AppRuntimeBudgetStateBackend extends BlockReservingBudgetStateBacke
           charges: [...input.charges],
           schedulerSchema: schedulerSchema ?? null,
         }),
+      options,
     );
   }
 }

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -1,7 +1,7 @@
 import {
   acquireRateStateViaAppRuntime,
-  AppRuntimeApiResponseError,
-  isAppRuntimeApiCapacityError,
+  AppRuntimeApiTransportError,
+  capacityShedRetryAfterMs,
   penalizeRateStateViaAppRuntime,
   releaseRateStateViaAppRuntime,
   RateStateValidationUnavailableError,
@@ -143,20 +143,15 @@ function sleepWithAbort(
  */
 const RATE_BLOCK_HORIZON_MS = 1_000;
 const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
-const CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS = 1_000;
-
-function capacityShedRetryAfterMs(error: unknown): number | null {
-  const shed =
-    isAppRuntimeApiCapacityError(error) ||
-    (error instanceof AppRuntimeApiResponseError &&
-      error.retryable &&
-      error.status === 429);
-  if (!shed) return null;
-  const retryAfterMs = (error as AppRuntimeApiResponseError).retryAfterMs;
-  return retryAfterMs && retryAfterMs > 0
-    ? retryAfterMs
-    : CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS;
-}
+const RELEASE_SHED_BACKOFF_BASE_MS = 1_000;
+const RELEASE_SHED_BACKOFF_MAX_MS = 16_000;
+/**
+ * Longest one batch of leases keeps retrying a shed release. The store expires
+ * an unreleased lease after five minutes, so retrying only frees its slot
+ * sooner. Past this window the lease is left to that TTL.
+ */
+export const RELEASE_SHED_RETRY_WINDOW_MS = 120_000;
+
 /**
  * Jitter fraction applied to a server-supplied `waitMs` so a synchronized wave
  * of buckets that all hit granted=0 at the same instant re-probe at slightly
@@ -316,10 +311,18 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     string,
     number
   >();
-  private pendingReleaseFailure: Error | null = null;
+
+  /**
+   * The sandbox runner's transport already retries a request that gets no
+   * response through a whole network cut, so one more round here would only
+   * repeat that wait.
+   */
+  private readonly transportRetriesOutages: boolean;
 
   constructor(context: WorkerRuntimeApiContext, options: Options = {}) {
     this.port = options.port ?? contextRatePort(context);
+    this.transportRetriesOutages =
+      context.connectionRetryBudgetMs !== undefined;
     this.schedulerSchema = options.schedulerSchema?.trim() || null;
     this.now = options.now ?? (() => Date.now());
     this.sleep = options.sleep ?? sleepWithAbort;
@@ -437,9 +440,9 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   }): PacingPermit | null {
     const { bucketId, rules } = input;
     if (rules.length === 0) return noopPacingPermit();
-    // A probe reports nothing and repairs nothing: a pending release failure
-    // or an exhausted pacer surfaces from the next blocking acquire instead.
-    if (this.pendingReleaseFailure || this.pacerExhaustion(bucketId)) {
+    // A probe reports nothing and repairs nothing: an exhausted pacer surfaces
+    // from the next blocking acquire instead.
+    if (this.pacerExhaustion(bucketId)) {
       return null;
     }
     const laneKey = rateLaneKey(
@@ -535,7 +538,6 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   }): Promise<PacingPermit> {
     const { bucketId, rules, signal } = input;
     if (rules.length === 0) return noopPacingPermit();
-    this.throwPendingReleaseFailure();
     // Pre-check: zero round trips when the pacer is already holding past the
     // threshold. Fails fast with a typed PROVIDER_EXHAUSTED before touching the
     // store.
@@ -1177,9 +1179,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         consecutiveFailures += 1;
         const normalized = normalizeError(error);
         this.onStoreFailure({ bucketId, error: normalized.message });
-        if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
+        if (
+          consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length ||
+          (this.transportRetriesOutages &&
+            error instanceof AppRuntimeApiTransportError)
+        ) {
           throw new Error(
             `Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
+            { cause: error },
           );
         }
         await this.sleep(
@@ -1983,10 +1990,31 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     queueMicrotask(() => void this.flushReleases(key, pending));
   }
 
+  /**
+   * Background write: `permit.release()` has already returned, so a failure
+   * here has no caller, and the next acquire belongs to an unrelated row. An
+   * unreleased lease holds its slot until the store's TTL, which errs toward
+   * fewer provider calls, and the store still owns every grant. That is why a
+   * failed acquire fails closed and a failed release does not.
+   *
+   * A shed comes from a working store, and DELETE by lease id is idempotent, so
+   * it is retried inside a window. Anything else has spent the transport's
+   * retries already. A contract 4xx would only repeat: acquire passes the same
+   * auth and scope checks and fails closed on its own row, while a release-only
+   * bug surfaces as provider admission timeouts once slots pin. Report it and
+   * let the TTL reclaim the slot.
+   *
+   * Unreleased leases keep counting against the store's concurrency cap, so a
+   * shed lane cannot be granted more than that cap per TTL, and the window
+   * drops whatever the batch holds. The batch stays bounded, and this loop ends
+   * once the lane stops releasing.
+   */
   private async flushReleases(
     key: string,
     pending: PendingRelease,
   ): Promise<void> {
+    let shedStartedAtMs: number | null = null;
+    let consecutiveSheds = 0;
     while (pending.leaseIds.length > 0) {
       const leaseIds = pending.leaseIds.splice(0);
       try {
@@ -1997,15 +2025,37 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           leaseIds,
           schedulerSchema: this.schedulerSchema,
         });
+        shedStartedAtMs = null;
+        consecutiveSheds = 0;
         this.signalLeaseRelease(pending.bucketId);
       } catch (error) {
         const normalized = normalizeError(error);
-        this.pendingReleaseFailure = new Error(
-          `Rate-state release failed for ${pending.bucketId}: ${normalized.message}`,
-        );
+        const shedRetryAfterMs = capacityShedRetryAfterMs(error);
+        if (shedRetryAfterMs !== null) {
+          consecutiveSheds += 1;
+          shedStartedAtMs ??= this.now();
+          const delayMs = this.releaseShedDelayMs(
+            shedRetryAfterMs,
+            consecutiveSheds,
+          );
+          if (
+            this.now() - shedStartedAtMs + delayMs <
+            RELEASE_SHED_RETRY_WINDOW_MS
+          ) {
+            this.onStoreFailure({
+              bucketId: pending.bucketId,
+              error: `Rate-state release shed for ${pending.bucketId}; retrying in ${delayMs}ms: ${normalized.message}`,
+            });
+            pending.leaseIds.unshift(...leaseIds);
+            await this.sleep(delayMs);
+            continue;
+          }
+        }
+        shedStartedAtMs = null;
+        consecutiveSheds = 0;
         this.onStoreFailure({
           bucketId: pending.bucketId,
-          error: normalized.message,
+          error: `Rate-state release failed for ${pending.bucketId}; leases held until their TTL (${leaseIds.length}): ${normalized.message}`,
         });
       }
     }
@@ -2020,11 +2070,22 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     }
   }
 
-  private throwPendingReleaseFailure(): void {
-    if (!this.pendingReleaseFailure) return;
-    const error = this.pendingReleaseFailure;
-    this.pendingReleaseFailure = null;
-    throw error;
+  /**
+   * Capped doubling, never under the shed's Retry-After. Jitter only adds, so a
+   * wave of runners shed together spreads out above the deadline they were given.
+   */
+  private releaseShedDelayMs(
+    retryAfterMs: number,
+    consecutiveSheds: number,
+  ): number {
+    const floorMs = Math.max(
+      retryAfterMs,
+      Math.min(
+        RELEASE_SHED_BACKOFF_BASE_MS * 2 ** (consecutiveSheds - 1),
+        RELEASE_SHED_BACKOFF_MAX_MS,
+      ),
+    );
+    return floorMs + Math.round(this.jitter() * floorMs);
   }
 
   private signalLeaseRelease(bucketId: string): void {