deepline 0.1.19 → 0.1.20

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -156,6 +156,7 @@ interface CoordinatorEnv {
   DEEPLINE_API_BASE_URL: string;
   DEEPLINE_INTERNAL_TOKEN?: string;
   DEEPLINE_TAIL_LOG_TOKEN?: string;
+  DEEPLINE_COORDINATOR_DEPLOY_MARKER?: string;
   VERCEL_PROTECTION_BYPASS_TOKEN?: string;
   DEEPLINE_PLAY_PREVIEW_SLUG?: string;
   /**
@@ -369,6 +370,7 @@ const WORKFLOW_POOL_READY_TIMEOUT_MS = 1_500;
 const WORKFLOW_POOL_READY_POLL_MS = 250;
 const WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS = 2_500;
 const WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE = 1;
+const WORKFLOW_POOL_CONTROL_TIMEOUT_MS = 750;
 
 function buildDynamicWorkflowMetadata(
   params: PlayWorkflowParams,
@@ -469,26 +471,48 @@ function workflowPoolDurableObject(env: CoordinatorEnv): DurableObjectStub {
 async function callWorkflowPool<T>(
   env: CoordinatorEnv,
   path: string,
-  init?: RequestInit,
+  init?: RequestInit & { timeoutMs?: number },
 ): Promise<T> {
-  const response = await workflowPoolDurableObject(env).fetch(
-    `https://deepline.workflow-pool.internal${path}`,
-    {
-      ...init,
-      headers: {
-        'content-type': 'application/json',
-        ...(init?.headers ?? {}),
-      },
-    },
+  const timeoutMs = Math.max(
+    1,
+    Math.floor(init?.timeoutMs ?? WORKFLOW_POOL_CONTROL_TIMEOUT_MS),
   );
-  if (!response.ok) {
-    throw new Error(
-      `workflow pool ${path} failed ${response.status}: ${(
-        await response.text().catch(() => '')
-      ).slice(0, 400)}`,
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const fetchInit: RequestInit = { ...(init ?? {}) };
+    delete (fetchInit as { timeoutMs?: number }).timeoutMs;
+    delete fetchInit.signal;
+    const response = await workflowPoolDurableObject(env).fetch(
+      `https://deepline.workflow-pool.internal${path}`,
+      {
+        ...fetchInit,
+        signal: controller.signal,
+        headers: {
+          'content-type': 'application/json',
+          ...(init?.headers ?? {}),
+        },
+      },
     );
+    if (!response.ok) {
+      throw new Error(
+        `workflow pool ${path} failed ${response.status}: ${(
+          await response.text().catch(() => '')
+        ).slice(0, 400)}`,
+      );
+    }
+    return (await response.json()) as T;
+  } catch (error) {
+    if (
+      error instanceof Error &&
+      (error.name === 'AbortError' || error.message.includes('aborted'))
+    ) {
+      throw new Error(`workflow pool ${path} timed out after ${timeoutMs}ms`);
+    }
+    throw error;
+  } finally {
+    clearTimeout(timer);
   }
-  return (await response.json()) as T;
 }
 
 type WorkflowPoolCounts = {
@@ -903,7 +927,11 @@ async function submitViaPooledWorkflow(input: {
     return null;
   }
   const leaseStartedAt = Date.now();
-  let pooledInstanceId = await leaseWorkflowPoolId(input.env);
+  let leaseError: string | null = null;
+  let pooledInstanceId = await leaseWorkflowPoolId(input.env).catch((error) => {
+    leaseError = error instanceof Error ? error.message : String(error);
+    return null;
+  });
   const missCounts = pooledInstanceId
     ? null
     : await workflowPoolCount(input.env).catch(() => null);
@@ -913,6 +941,7 @@ async function submitViaPooledWorkflow(input: {
     graphHash: input.params.graphHash ?? null,
     extra: {
       pooled: Boolean(pooledInstanceId),
+      ...(leaseError ? { error: leaseError } : {}),
       ...(missCounts
         ? {
             availableAfterMiss: missCounts.available,
@@ -952,12 +981,19 @@ async function submitViaPooledWorkflow(input: {
     });
     if (refillResult?.available) {
       const retryStartedAt = Date.now();
-      pooledInstanceId = await leaseWorkflowPoolId(input.env);
+      let retryLeaseError: string | null = null;
+      pooledInstanceId = await leaseWorkflowPoolId(input.env).catch((error) => {
+        retryLeaseError = error instanceof Error ? error.message : String(error);
+        return null;
+      });
       input.recordSubmitTiming({
         phase: 'coordinator.workflow_pool_lease_retry',
         ms: Date.now() - retryStartedAt,
         graphHash: input.params.graphHash ?? null,
-        extra: { pooled: Boolean(pooledInstanceId) },
+        extra: {
+          pooled: Boolean(pooledInstanceId),
+          ...(retryLeaseError ? { error: retryLeaseError } : {}),
+        },
       });
     }
   }
@@ -967,38 +1003,26 @@ async function submitViaPooledWorkflow(input: {
   }
 
   const instance = await input.env.PLAY_WORKFLOW.get(pooledInstanceId);
+  const readyCheckStartedAt = Date.now();
+  const status = await instance.status().catch(() => null);
+  const statusName = workflowStatusName(status);
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_ready_check',
+    ms: Date.now() - readyCheckStartedAt,
+    graphHash: input.params.graphHash ?? null,
+    extra: { instanceId: pooledInstanceId, status: statusName },
+  });
+  if (!workflowPoolStatusIsReady(statusName)) {
+    await instance.terminate().catch(() => undefined);
+    disposeRpcStub(instance);
+    return null;
+  }
+  const sendStartedAt = Date.now();
   try {
-    const readyCheckStartedAt = Date.now();
-    const status = await instance.status().catch(() => null);
-    const statusName = workflowStatusName(status);
-    input.recordSubmitTiming({
-      phase: 'coordinator.workflow_pool_ready_check',
-      ms: Date.now() - readyCheckStartedAt,
-      graphHash: input.params.graphHash ?? null,
-      extra: { instanceId: pooledInstanceId, status: statusName },
-    });
-    if (!workflowPoolStatusIsReady(statusName)) {
-      await instance.terminate().catch(() => undefined);
-      disposeRpcStub(instance);
-      return null;
-    }
-    const sendStartedAt = Date.now();
     await instance.sendEvent({
       type: WORKFLOW_POOL_START_EVENT_TYPE,
       payload: buildDispatcherEnvelope(input.params),
     });
-    await mapRunToWorkflowInstance({
-      env: input.env,
-      runId: input.params.runId,
-      instanceId: pooledInstanceId,
-    });
-    input.recordSubmitTiming({
-      phase: 'coordinator.workflow_pool_send_event',
-      ms: Date.now() - sendStartedAt,
-      graphHash: input.params.graphHash ?? null,
-      extra: { instanceId: pooledInstanceId },
-    });
-    return instance;
   } catch (error) {
     disposeRpcStub(instance);
     console.warn('[coordinator.workflow_pool] sendEvent failed; falling back', {
@@ -1008,6 +1032,27 @@ async function submitViaPooledWorkflow(input: {
     });
     return null;
   }
+  try {
+    await mapRunToWorkflowInstance({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    });
+  } catch (error) {
+    disposeRpcStub(instance);
+    throw new Error(
+      `workflow pool mapRunToWorkflowInstance failed after pooled workflow start for ${input.params.runId}: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    );
+  }
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_send_event',
+    ms: Date.now() - sendStartedAt,
+    graphHash: input.params.graphHash ?? null,
+    extra: { instanceId: pooledInstanceId },
+  });
+  return instance;
 }
 
 function readWorkflowPayload(event: unknown): Record<string, unknown> | null {
@@ -2029,6 +2074,29 @@ const coordinatorEntrypoint = {
       if (authError) return authError;
       return await handleCoordinatorWarmup(request, env, ctx);
     }
+    if (url.pathname === '/tail-log-token/probe') {
+      const authError = authorizeCoordinatorControlRequest({ request, env });
+      if (authError) return authError;
+      const expectedTailLogToken = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
+      if (!expectedTailLogToken) {
+        return Response.json(
+          { ok: false, error: 'tail log token is not configured' },
+          { status: 503 },
+        );
+      }
+      const actualTailLogToken =
+        request.headers.get('x-deepline-tail-log-token')?.trim() ?? '';
+      if (actualTailLogToken !== expectedTailLogToken) {
+        return Response.json(
+          { ok: false, error: 'tail log token mismatch' },
+          { status: 401 },
+        );
+      }
+      return Response.json({
+        ok: true,
+        deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
+      });
+    }
     if (url.pathname === '/workflow-pool/refill') {
       const internalAuthError = authorizeCoordinatorControlRequest({
         request,
@@ -2798,7 +2866,7 @@ function loadDynamicPlayWorkerSync(
   }
   const artifactIdentity =
     metadata.artifactHash?.trim() || stableHash(artifactStorageKey);
-  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h4-runtime-api`;
+  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h5-runtime-api-coordinator`;
   const runIdForTrace = metadata.runId ?? graphHash;
   const loaderGetStartedAt = Date.now();
   const stub = env.LOADER.get(workerCacheKey, async () => {
@@ -2829,6 +2897,10 @@ function loadDynamicPlayWorkerSync(
         // tool execution, DB session, and artifact callbacks. This avoids a
         // public fetch hop when Cloudflare exposes the RuntimeApi export.
         ...makeRuntimeApiEnvBinding(),
+        // In-process coordinator control bridge used by ctx.runPlay and
+        // parent terminal signals. This keeps scalar child plays inline with
+        // the parent instead of round-tripping through nested Workflow waits.
+        ...makeCoordinatorControlBinding(),
         // NOTE: We intentionally do NOT pass `env.PLAYS_BUCKET` (an R2Bucket
         // binding) through to the per-play Worker's env. Including a raw
         // R2Bucket in the dynamically-loaded Worker's env makes Cloudflare
@@ -2880,7 +2952,7 @@ async function loadDynamicPlayWorker(
   }
   const artifactIdentity =
     metadata.artifactHash?.trim() || stableHash(artifactStorageKey);
-  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h4-runtime-api`;
+  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h5-runtime-api-coordinator`;
   const runIdForTrace = metadata.runId ?? graphHash;
   const loaderGetStartedAt = Date.now();
   const stub = env.LOADER.get(workerCacheKey, async () => {
@@ -2902,11 +2974,11 @@ async function loadDynamicPlayWorker(
         // Mirror of the sync loader (above) — see that copy for the
         // architectural rationale. The dynamic worker env is intentionally
         // minimal; runtime callbacks use RUNTIME_API, file reads go through
-        // HARNESS, and child workflow control uses the coordinator URL in the
-        // run request.
+        // HARNESS, and child workflow control uses the COORDINATOR binding.
         HARNESS: env.HARNESS,
         VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
         ...makeRuntimeApiEnvBinding(),
+        ...makeCoordinatorControlBinding(),
       },
     };
   });
@@ -3219,6 +3291,7 @@ async function handleCoordinatorWarmup(
  * worker uses its existing `fetch(req.baseUrl + path)` transport.
  */
 let loggedMissingRuntimeApiExport = false;
+let loggedMissingCoordinatorControlExport = false;
 
 function makeRuntimeApiEnvBinding():
   | { RUNTIME_API: { fetch(req: Request): Promise<Response> } }
@@ -3241,20 +3314,24 @@ function makeRuntimeApiEnvBinding():
   return { RUNTIME_API: ctor({ props: undefined }) };
 }
 
-function makeCoordinatorControlBinding(): {
-  submitChild(
-    parentRunId: string,
-    body: Record<string, unknown>,
-  ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
-  signal(
-    runId: string,
-    body: Record<string, unknown>,
-  ): Promise<Record<string, unknown>>;
-  recordPerfTrace(
-    runId: string,
-    payload: CoordinatorPerfTracePayload,
-  ): Promise<void>;
-} {
+function makeCoordinatorControlBinding():
+  | {
+      COORDINATOR: {
+        submitChild(
+          parentRunId: string,
+          body: Record<string, unknown>,
+        ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
+        signal(
+          runId: string,
+          body: Record<string, unknown>,
+        ): Promise<Record<string, unknown>>;
+        recordPerfTrace(
+          runId: string,
+          payload: CoordinatorPerfTracePayload,
+        ): Promise<void>;
+      };
+    }
+  | Record<string, never> {
   const exports = workersExports as unknown as {
     CoordinatorControl?: (init: { props: undefined }) => {
       submitChild(
@@ -3273,11 +3350,15 @@ function makeCoordinatorControlBinding(): {
   };
   const ctor = exports.CoordinatorControl;
   if (typeof ctor !== 'function') {
-    throw new Error(
-      'CoordinatorControl is not registered on cloudflare:workers exports.',
-    );
+    if (!loggedMissingCoordinatorControlExport) {
+      loggedMissingCoordinatorControlExport = true;
+      console.warn(
+        '[coordinator] CoordinatorControl is not registered on cloudflare:workers exports; using public coordinator transport.',
+      );
+    }
+    return {};
   }
-  return ctor({ props: undefined });
+  return { COORDINATOR: ctor({ props: undefined }) };
 }
 
 async function loadStoredPlayArtifactFromR2(

package/dist/repo/apps/play-runner-workers/src/dedup-do.ts

@@ -87,6 +87,7 @@ const COORDINATOR_TRACE_MAX_ENTRIES = 200;
 const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
 const WORKFLOW_POOL_DEFAULT_TTL_MS = 8 * 60 * 1000;
 const WORKFLOW_POOL_RUN_MAPPING_TTL_MS = 60 * 60 * 1000;
+const WORKFLOW_POOL_READY_MAX_AGE_MS = 20_000;
 
 interface DedupEnv {
   PLAY_DEDUP: DurableObjectNamespace;
@@ -248,7 +249,9 @@ export class PlayDedup implements DurableObject {
       }
     }
 
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handleAwait(req: Request): Promise<Response> {
@@ -313,7 +316,9 @@ export class PlayDedup implements DurableObject {
     // Schedule alarm to evict after a grace period (any straggler awaits
     // get a chance to resolve). Grace = 60s.
     await this.state.storage.setAlarm(Date.now() + FINISH_ALARM_DELAY_MS);
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handleDebug(_req: Request): Promise<Response> {
@@ -321,7 +326,11 @@ export class PlayDedup implements DurableObject {
     const dump: Record<string, DedupEntry> = {};
     for (const [k, v] of all) dump[k] = v as DedupEntry;
     return new Response(
-      JSON.stringify({ size: all.size, entries: dump, waiters: this.waiters.size }),
+      JSON.stringify({
+        size: all.size,
+        entries: dump,
+        waiters: this.waiters.size,
+      }),
       { headers: { 'content-type': 'application/json' } },
     );
   }
@@ -339,7 +348,8 @@ export class PlayDedup implements DurableObject {
       value.entry !== undefined &&
       value.entry.version === version &&
       value.entry.expiresAt > now &&
-      value.entry.readyAt !== null
+      value.entry.readyAt !== null &&
+      now - value.entry.readyAt <= WORKFLOW_POOL_READY_MAX_AGE_MS
     );
   }
 
@@ -360,6 +370,8 @@ export class PlayDedup implements DurableObject {
       if (
         !entry ||
         entry.expiresAt <= now ||
+        (entry.readyAt !== null &&
+          now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS) ||
         (version && entry.version !== version)
       ) {
         expiredKeys.push(key);
@@ -380,16 +392,15 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handlePoolAdd(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          ids?: unknown;
-          ttlMs?: unknown;
-          version?: unknown;
-          readyAt?: unknown;
-          ready?: unknown;
-        }
-      | null;
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const body = (await req.json().catch(() => null)) as {
+      ids?: unknown;
+      ttlMs?: unknown;
+      version?: unknown;
+      readyAt?: unknown;
+      ready?: unknown;
+    } | null;
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
@@ -445,7 +456,9 @@ export class PlayDedup implements DurableObject {
       const sorted = [...entries.entries()]
         .map(([key, entry]) => ({ key, entry }))
         .filter((entry) => this.isReadyWorkflowPoolEntry(entry, version, now))
-        .sort((a, b) => a.entry.readyAt - b.entry.readyAt);
+        // Lease the freshest ready instance. Older waiting Workflows are more
+        // likely to be hibernated, so "pooled" otherwise still pays a wake tax.
+        .sort((a, b) => b.entry.readyAt - a.entry.readyAt);
       const selected = sorted[0];
       if (!selected) return;
       leasedId = selected.entry.id;
@@ -461,7 +474,8 @@ export class PlayDedup implements DurableObject {
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
-    await this.gcWorkflowPool(Date.now(), version);
+    const now = Date.now();
+    await this.gcWorkflowPool(now, version);
     const entries = await this.state.storage.list<WorkflowPoolEntry>({
       prefix: WORKFLOW_POOL_KEY_PREFIX,
     });
@@ -469,8 +483,14 @@ export class PlayDedup implements DurableObject {
     let warming = 0;
     for (const entry of entries.values()) {
       if (entry.version !== version) continue;
-      if (entry.readyAt === null) warming += 1;
-      else available += 1;
+      if (
+        entry.readyAt === null ||
+        now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS
+      ) {
+        warming += 1;
+      } else {
+        available += 1;
+      }
     }
     return new Response(JSON.stringify({ available, warming }), {
       headers: { 'content-type': 'application/json' },
@@ -502,13 +522,12 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handlePoolPromote(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          ids?: unknown;
-          version?: unknown;
-        }
-      | null;
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const body = (await req.json().catch(() => null)) as {
+      ids?: unknown;
+      version?: unknown;
+    } | null;
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
@@ -536,19 +555,21 @@ export class PlayDedup implements DurableObject {
         await this.state.storage.put(writes);
       }
     });
-    return new Response(JSON.stringify({ promoted: Object.keys(writes).length }), {
-      headers: { 'content-type': 'application/json' },
-    });
+    return new Response(
+      JSON.stringify({ promoted: Object.keys(writes).length }),
+      {
+        headers: { 'content-type': 'application/json' },
+      },
+    );
   }
 
   private async handlePoolDelete(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          ids?: unknown;
-          version?: unknown;
-        }
-      | null;
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const body = (await req.json().catch(() => null)) as {
+      ids?: unknown;
+      version?: unknown;
+    } | null;
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
@@ -575,34 +596,32 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handlePoolMapRun(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          runId?: unknown;
-          instanceId?: unknown;
-          version?: unknown;
-        }
-      | null;
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+      instanceId?: unknown;
+      version?: unknown;
+    } | null;
     const runId = typeof body?.runId === 'string' ? body.runId : '';
     const instanceId =
       typeof body?.instanceId === 'string' ? body.instanceId : '';
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
     if (!runId || !instanceId || !version) {
       return new Response('runId, instanceId, and version are required', {
         status: 400,
       });
     }
     const now = Date.now();
-    await this.state.storage.put(
-      `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
-      {
-        runId,
-        instanceId,
-        version,
-        createdAt: now,
-        expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
-      } satisfies WorkflowRunMapping,
-    );
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+    await this.state.storage.put(`${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`, {
+      runId,
+      instanceId,
+      version,
+      createdAt: now,
+      expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
+    } satisfies WorkflowRunMapping);
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handlePoolResolveRun(req: Request): Promise<Response> {
@@ -620,7 +639,9 @@ export class PlayDedup implements DurableObject {
       `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
     );
     if (mapping && mapping.version !== version) {
-      await this.state.storage.delete(`${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`);
+      await this.state.storage.delete(
+        `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
+      );
       return new Response(JSON.stringify({ instanceId: null }), {
         headers: { 'content-type': 'application/json' },
       });
@@ -658,9 +679,9 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handleTraceAdd(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | Partial<CoordinatorTraceEntry>
-      | null;
+    const body = (await req
+      .json()
+      .catch(() => null)) as Partial<CoordinatorTraceEntry> | null;
     if (
       !body ||
       (body.source !== 'coordinator' && body.source !== 'dynamic_worker') ||
@@ -693,7 +714,9 @@ export class PlayDedup implements DurableObject {
         await this.state.storage.delete([...entries.keys()].slice(0, overflow));
       }
     });
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handleTraceList(): Promise<Response> {
@@ -702,7 +725,9 @@ export class PlayDedup implements DurableObject {
     });
     return new Response(
       JSON.stringify({
-        entries: [...entries.values()].sort((left, right) => left.ts - right.ts),
+        entries: [...entries.values()].sort(
+          (left, right) => left.ts - right.ts,
+        ),
       }),
       { headers: { 'content-type': 'application/json' } },
     );