dsh-lcx-codex 0.3.3 → 0.4.0-rc.13

package/lib/transport.js

@@ -1,396 +1,75 @@
-import { setTimeout as delay } from 'node:timers/promises'
-
-const MAX_RETRY_DELAY_MS = 30_000
-const combinedSignalStates = new WeakMap()
-
-export class LcxHttpError extends Error {
-  constructor(message, { code = 'LCX_HTTP_ERROR', status, retryable = false, cause } = {}) {
-    super(message, cause === undefined ? undefined : { cause })
-    this.name = 'LcxHttpError'
-    this.code = code
-    this.status = status
-    this.retryable = retryable
-  }
-}
+const DEFAULT_TIMEOUT_MS = 300_000
+const DEFAULT_MAX_BYTES = 8 * 1024 * 1024
 
 export function abortIfNeeded(signal) {
-  if (signal?.aborted) {
-    throw signal.reason instanceof Error ? signal.reason : new Error('request aborted')
-  }
-}
-
-function responseDetail(body) {
-  try {
-    const parsed = JSON.parse(body)
-    if (typeof parsed?.error === 'string') return parsed.error
-    if (typeof parsed?.error?.message === 'string') return parsed.error.message
-    if (typeof parsed?.message === 'string') return parsed.message
-  } catch {
-    // The status and a bounded body preview are enough when the gateway did not return JSON.
-  }
-  return String(body ?? '').replace(/\s+/gu, ' ').trim().slice(0, 500)
-}
-
-function retryableStatus(status) {
-  return status === 429 || (status >= 500 && status <= 599)
-}
-
-function combinedSignal(signal, timeoutMs) {
-  const controller = new AbortController()
-  let timer
-  const abort = (reason) => {
-    if (controller.signal.aborted) return
-    if (timer !== undefined) globalThis.clearTimeout(timer)
-    combinedSignalStates.delete(controller.signal)
-    signal?.removeEventListener('abort', onCallerAbort)
-    controller.abort(reason)
-  }
-  const onCallerAbort = () => abort(signal.reason)
-  if (Number.isFinite(timeoutMs)) {
-    timer = globalThis.setTimeout(() => {
-      abort(new DOMException(`The operation was aborted due to timeout`, 'TimeoutError'))
-    }, Math.max(0, timeoutMs))
-    timer.unref?.()
-  }
-  combinedSignalStates.set(controller.signal, { timer, callerSignal: signal, onCallerAbort })
-  if (signal?.aborted) abort(signal.reason)
-  else signal?.addEventListener('abort', onCallerAbort, { once: true })
-  return controller.signal
-}
-
-function holdSignal(signal) {
-  combinedSignalStates.get(signal)?.timer?.ref?.()
-}
-
-function releaseSignal(signal) {
-  const state = combinedSignalStates.get(signal)
-  if (!state) return
-  if (state.timer !== undefined) globalThis.clearTimeout(state.timer)
-  state.callerSignal?.removeEventListener('abort', state.onCallerAbort)
-  combinedSignalStates.delete(signal)
-}
-
-function retryAfterMilliseconds(response) {
-  const value = response?.headers?.get?.('retry-after')
-  if (!value) return undefined
-  const seconds = Number(value)
-  if (Number.isFinite(seconds) && seconds >= 0) return Math.min(MAX_RETRY_DELAY_MS, Math.ceil(seconds * 1000))
-  const date = Date.parse(value)
-  return Number.isFinite(date) ? Math.min(MAX_RETRY_DELAY_MS, Math.max(0, date - Date.now())) : undefined
-}
-
-function timeoutError(timeoutMs, cause) {
-  return new LcxHttpError(`LCX request timed out after ${timeoutMs} ms`, { code: 'LCX_TIMEOUT', retryable: true, cause })
-}
-
-function abortRequestIfNeeded(requestSignal, callerSignal, timeoutMs) {
-  abortIfNeeded(callerSignal)
-  if (!requestSignal?.aborted) return
-  if (requestSignal.reason?.name === 'TimeoutError') throw timeoutError(timeoutMs, requestSignal.reason)
-  throw requestSignal.reason instanceof Error ? requestSignal.reason : new Error('request aborted')
-}
-
-function retryWaitMilliseconds(error, baseDelayMs, attempt) {
-  const retryAfter = Number(error?.retryAfterMs)
-  if (Number.isFinite(retryAfter) && retryAfter >= 0) return Math.min(MAX_RETRY_DELAY_MS, retryAfter)
-  const exponential = baseDelayMs * (2 ** (attempt - 1))
-  return Number.isFinite(exponential) && exponential >= 0 ? Math.min(MAX_RETRY_DELAY_MS, exponential) : MAX_RETRY_DELAY_MS
-}
-
-async function readLimitedText(response, maxResponseBytes, signal, label) {
-  if (!response?.body) return ''
-  const reader = response.body.getReader()
-  const decoder = new TextDecoder()
-  const parts = []
-  let bytes = 0
-  const cancel = () => { reader.cancel(signal?.reason).catch(() => undefined) }
-  try {
-    signal?.addEventListener('abort', cancel, { once: true })
-    while (true) {
-      abortIfNeeded(signal)
-      const result = await reader.read()
-      abortIfNeeded(signal)
-      if (result.done) break
-      bytes += result.value.byteLength
-      if (bytes > maxResponseBytes) {
-        throw new LcxHttpError(`LCX ${label} response exceeds ${maxResponseBytes} bytes`, { code: 'LCX_RESPONSE_TOO_LARGE', status: response.status })
-      }
-      parts.push(decoder.decode(result.value, { stream: true }))
-    }
-    parts.push(decoder.decode())
-    return parts.join('')
-  } finally {
-    signal?.removeEventListener('abort', cancel)
-    await reader.cancel().catch(() => undefined)
-  }
-}
-
-function normalizeReadError(error, requestSignal, timeoutMs) {
-  if (requestSignal?.aborted && requestSignal.reason?.name === 'TimeoutError') return timeoutError(timeoutMs, error)
-  return error
-}
-
-function abortableResponse(response, requestSignal, callerSignal, timeoutMs, ownsRequestSignal = false) {
-  if (!response?.body || !requestSignal) return response
-  const source = response.body
-  const reader = source.getReader()
-  let abortError
-  let cleanedUp = false
-  let pendingReadReject
-  let onAbort
-  const cleanup = () => {
-    if (cleanedUp) return
-    cleanedUp = true
-    requestSignal.removeEventListener('abort', onAbort)
-    if (ownsRequestSignal) releaseSignal(requestSignal)
-  }
-  const abort = () => {
-    if (abortError) return
-    abortError = callerSignal?.aborted
-      ? (callerSignal.reason instanceof Error ? callerSignal.reason : new Error('request aborted'))
-      : requestSignal.reason?.name === 'TimeoutError'
-        ? timeoutError(timeoutMs, requestSignal.reason)
-        : (requestSignal.reason instanceof Error ? requestSignal.reason : new Error('request aborted'))
-    cleanup()
-    pendingReadReject?.(abortError)
-    reader.cancel(abortError).catch(() => undefined)
-  }
-  onAbort = abort
-  const stream = new ReadableStream({
-    start(controller) {
-      if (requestSignal.aborted) {
-        abort()
-        controller.error(abortError)
-      } else {
-        requestSignal.addEventListener('abort', onAbort, { once: true })
-      }
-    },
-    async pull(controller) {
-      if (abortError) return
-      try {
-        const result = await new Promise((resolve, reject) => {
-          pendingReadReject = reject
-          reader.read().then(resolve, reject)
-        })
-        pendingReadReject = undefined
-        if (abortError) return
-        if (result.done) {
-          cleanup()
-          controller.close()
-        } else {
-          controller.enqueue(result.value)
-        }
-      } catch (error) {
-        pendingReadReject = undefined
-        cleanup()
-        controller.error(abortError ?? error)
-      }
-    },
-    cancel(reason) {
-      cleanup()
-      reader.cancel(reason).catch(() => undefined)
-    },
-  })
-  return new Response(stream, {
-    status: response.status,
-    statusText: response.statusText,
-    headers: response.headers,
-  })
-}
-
-export async function fetchJson(url, body, headers, signal, timeoutMs, options = {}) {
-  abortIfNeeded(signal)
-  const maxResponseBytes = options.maxResponseBytes ?? 4 * 1024 * 1024
-  const requestSignal = options.requestSignal ?? combinedSignal(signal, timeoutMs)
-  const ownsRequestSignal = !options.requestSignal
-  if (ownsRequestSignal) holdSignal(requestSignal)
-  try {
-    abortRequestIfNeeded(requestSignal, signal, timeoutMs)
-    let serializedBody
-  try {
-    serializedBody = JSON.stringify(body)
-  } catch (error) {
-    throw new LcxHttpError(`LCX request body is not serializable: ${String(error)}`, { code: 'LCX_INVALID_REQUEST', cause: error })
-  }
-  let response
-  try {
-    response = await fetch(url, {
-      method: 'POST',
-      redirect: 'error',
-      headers: { accept: 'application/json', 'content-type': 'application/json', ...headers },
-      body: serializedBody,
-      signal: requestSignal,
-    })
-  } catch (error) {
-    abortIfNeeded(signal)
-    if (requestSignal?.aborted) abortRequestIfNeeded(requestSignal, signal, timeoutMs)
-    if (error?.name === 'TimeoutError') throw timeoutError(timeoutMs, error)
-    throw error
-  }
-
-  const contentLength = Number(response.headers.get('content-length') ?? 0)
-  if (Number.isFinite(contentLength) && contentLength > maxResponseBytes) {
-    throw new LcxHttpError(`LCX response exceeds ${maxResponseBytes} bytes`, { code: 'LCX_RESPONSE_TOO_LARGE', status: response.status })
-  }
-
-  let text
-  try {
-    text = await readLimitedText(response, maxResponseBytes, requestSignal, 'JSON')
-  } catch (error) {
-    throw normalizeReadError(error, requestSignal, timeoutMs)
-  }
-  if (!response.ok) {
-    const detail = responseDetail(text)
-    const error = new LcxHttpError(`LCX request failed (HTTP ${response.status})${detail ? `: ${detail}` : ''}`, {
-      code: retryableStatus(response.status) ? 'LCX_HTTP_RETRYABLE' : 'LCX_HTTP_ERROR',
-      status: response.status,
-      retryable: retryableStatus(response.status),
-    })
-    error.retryAfterMs = retryAfterMilliseconds(response)
-    throw error
-  }
-  try {
-    return JSON.parse(text)
-  } catch (error) {
-    throw new LcxHttpError(`LCX returned a non-JSON response: ${String(error)}`, { code: 'LCX_INVALID_JSON', status: response.status, cause: error })
-  }
-  }
-  finally {
-    if (ownsRequestSignal) releaseSignal(requestSignal)
-  }
-}
-
-export async function fetchSse(url, body, headers, signal, timeoutMs, options = {}) {
-  abortIfNeeded(signal)
-  const maxResponseBytes = options.maxResponseBytes ?? 8 * 1024 * 1024
-  const requestSignal = options.requestSignal ?? combinedSignal(signal, timeoutMs)
-  const ownsRequestSignal = !options.requestSignal
-  if (ownsRequestSignal) holdSignal(requestSignal)
-  let handedOff = false
-  try {
-    abortRequestIfNeeded(requestSignal, signal, timeoutMs)
-    let serializedBody
-    try {
-      serializedBody = JSON.stringify(body)
-  } catch (error) {
-    throw new LcxHttpError(`LCX request body is not serializable: ${String(error)}`, { code: 'LCX_INVALID_REQUEST', cause: error })
-  }
-  let response
-  try {
-    response = await fetch(url, {
-      method: 'POST',
-      redirect: 'error',
-      headers: { accept: 'text/event-stream', 'content-type': 'application/json', ...headers },
-      body: serializedBody,
-      signal: requestSignal,
-    })
-  } catch (error) {
+  if (signal?.aborted) throw signal.reason ?? Object.assign(new Error('request aborted'), { code: 'LCX_ABORTED' })
+}
+function makeError(message, code, extra = {}) { const error = new Error(message); error.code = code; Object.assign(error, extra); return error }
+function retryableStatus(status) { return status === 408 || status === 409 || status === 425 || status === 429 || (status >= 500 && status <= 599) }
+function delayFromHeaders(headers, attempt) {
+  const millis = Number(headers?.get?.('retry-after-ms')); if (Number.isFinite(millis) && millis >= 0) return Math.min(millis, 30_000)
+  const retryAfter = headers?.get?.('retry-after'); if (retryAfter) { const seconds = Number(retryAfter); if (Number.isFinite(seconds) && seconds >= 0) return Math.min(seconds * 1000, 30_000); const date = Date.parse(retryAfter); if (!Number.isNaN(date)) return Math.min(Math.max(0, date - Date.now()), 30_000) }
+  return Math.min(10_000, 500 * 2 ** Math.max(0, attempt - 1))
+}
+async function sleep(ms, signal) { if (ms <= 0) return; await new Promise((resolve, reject) => { if (signal?.aborted) return reject(signal.reason); const timer = setTimeout(resolve, ms); signal?.addEventListener('abort', () => { clearTimeout(timer); reject(signal.reason) }, { once: true }) }) }
+function combinedSignal(signal, timeoutMs) { if (!(Number.isFinite(timeoutMs) && timeoutMs > 0)) return signal; const timeout = AbortSignal.timeout(timeoutMs); return signal ? AbortSignal.any([signal, timeout]) : timeout }
+async function readLimited(response, maxBytes = DEFAULT_MAX_BYTES) { const buffer = new Uint8Array(await response.arrayBuffer()); if (buffer.byteLength > maxBytes) throw makeError(`response exceeds ${maxBytes} bytes`, 'LCX_RESPONSE_TOO_LARGE', { status: response.status }); return new TextDecoder().decode(buffer) }
+
+export async function fetchJsonWithRetry(url, body, headers = {}, signal, timeoutMs = DEFAULT_TIMEOUT_MS, options = {}) {
+  const maxAttempts = Math.max(1, Math.min(6, Number(options.maxAttempts ?? 3))); const maxResponseBytes = options.maxResponseBytes ?? DEFAULT_MAX_BYTES; let last
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     abortIfNeeded(signal)
-    if (requestSignal?.aborted) abortRequestIfNeeded(requestSignal, signal, timeoutMs)
-    if (error?.name === 'TimeoutError') throw timeoutError(timeoutMs, error)
-    throw error
-  }
-
-  const contentLength = Number(response.headers.get('content-length') ?? 0)
-  if (Number.isFinite(contentLength) && contentLength > maxResponseBytes) {
-    throw new LcxHttpError(`LCX SSE response exceeds ${maxResponseBytes} bytes`, { code: 'LCX_RESPONSE_TOO_LARGE', status: response.status })
-  }
-  if (!response.ok) {
-    let text
     try {
-      text = await readLimitedText(response, maxResponseBytes, requestSignal, 'SSE')
+      const requestSignal = combinedSignal(signal, timeoutMs)
+      const response = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json', ...headers }, body: JSON.stringify(body), signal: requestSignal, redirect: 'error' })
+      const text = await readLimited(response, maxResponseBytes)
+      if (!response.ok) {
+        const retryable = retryableStatus(response.status)
+        const error = makeError(`HTTP ${response.status}`, retryable ? 'LCX_HTTP_RETRYABLE' : 'LCX_HTTP_ERROR', { status: response.status, retryable, requestId: response.headers.get('x-request-id') ?? response.headers.get('request-id') ?? undefined })
+        if (!error.retryable || attempt >= maxAttempts) throw error; last = error; await sleep(delayFromHeaders(response.headers, attempt), signal); continue
+      }
+      let parsed
+      try { parsed = text ? JSON.parse(text) : {} } catch (cause) { throw makeError(`invalid JSON response from ${url}`, 'LCX_INVALID_JSON', { status: response.status, cause }) }
+      return parsed
     } catch (error) {
-      throw normalizeReadError(error, requestSignal, timeoutMs)
+      if (signal?.aborted) throw signal.reason ?? error
+      const retryable = error?.retryable === true || error?.code === 'LCX_HTTP_RETRYABLE' || error instanceof TypeError || ['ECONNRESET','ECONNREFUSED','ETIMEDOUT','EAI_AGAIN','UND_ERR_CONNECT_TIMEOUT'].includes(error?.code)
+      if (!retryable || attempt >= maxAttempts) { if (attempt >= maxAttempts && retryable) throw makeError(`request failed after ${maxAttempts} attempts`, 'LCX_RETRY_EXHAUSTED', { cause: error, retryable: true }); throw error }
+      last = error; await sleep(Math.min(10_000, 500 * 2 ** (attempt - 1)), signal)
     }
-    const detail = responseDetail(text)
-    const error = new LcxHttpError(`LCX request failed (HTTP ${response.status})${detail ? `: ${detail}` : ''}`, {
-      code: retryableStatus(response.status) ? 'LCX_HTTP_RETRYABLE' : 'LCX_HTTP_ERROR',
-      status: response.status,
-      retryable: retryableStatus(response.status),
-    })
-    error.retryAfterMs = retryAfterMilliseconds(response)
-    throw error
-  }
-  if (!response.body) {
-    throw new LcxHttpError('LCX SSE response has no body', { code: 'LCX_INVALID_SSE', status: response.status })
-  }
-    const wrapped = abortableResponse(response, requestSignal, signal, timeoutMs, ownsRequestSignal)
-    handedOff = ownsRequestSignal
-    return wrapped
-  } finally {
-    if (ownsRequestSignal && !handedOff) releaseSignal(requestSignal)
   }
+  throw last ?? makeError('request failed', 'LCX_REQUEST_FAILED')
 }
 
-async function waitForRetry(waitMs, operationSignal, callerSignal, timeoutMs) {
-  abortRequestIfNeeded(operationSignal, callerSignal, timeoutMs)
-  try {
-    await delay(waitMs, undefined, { signal: operationSignal })
-  } catch (error) {
-    abortRequestIfNeeded(operationSignal, callerSignal, timeoutMs)
-    throw error
-  }
-}
-
-function transientNetworkError(error) {
-  if (!error || error.name === 'AbortError' || error.name === 'TimeoutError') return false
-  if (error instanceof LcxHttpError) return error.code === 'LCX_HTTP_RETRYABLE' || error.code === 'LCX_TIMEOUT'
-  return error.retryable === true || ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT'].includes(error.code)
-}
-
-export async function fetchSseWithRetry(url, body, headers, signal, timeoutMs, options = {}) {
-  const maxAttempts = Math.max(1, Math.min(options.maxAttempts ?? 3, 6))
-  const baseDelayMs = Math.max(0, options.baseDelayMs ?? 250)
-  const consume = options.consume
-  if (typeof consume !== 'function') throw new TypeError('fetchSseWithRetry requires a consume callback')
-  const operationSignal = combinedSignal(signal, timeoutMs)
-  holdSignal(operationSignal)
-  let attempt = 0
-  try {
-    while (attempt < maxAttempts) {
-      attempt += 1
-      try {
-        abortRequestIfNeeded(operationSignal, signal, timeoutMs)
-        const response = await fetchSse(url, body, headers, signal, timeoutMs, { ...options, requestSignal: operationSignal })
-        try {
-          return await consume(response, { ...options, requestSignal: operationSignal })
-        } finally {
-          if (response.body) await response.body.cancel().catch(() => undefined)
-        }
-      } catch (error) {
-        abortIfNeeded(signal)
-        if (operationSignal.aborted) abortRequestIfNeeded(operationSignal, signal, timeoutMs)
-        if (!transientNetworkError(error) || attempt >= maxAttempts) throw error
-        await waitForRetry(retryWaitMilliseconds(error, baseDelayMs, attempt), operationSignal, signal, timeoutMs)
+export async function fetchSseWithRetry(url, body, headers = {}, signal, timeoutMs = DEFAULT_TIMEOUT_MS, options = {}) {
+  const maxAttempts = Math.max(1, Math.min(6, Number(options.maxAttempts ?? 3))); let last
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    abortIfNeeded(signal)
+    try {
+      const requestSignal = combinedSignal(signal, timeoutMs)
+      const response = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', accept: 'text/event-stream', ...headers }, body: JSON.stringify(body), signal: requestSignal, redirect: 'error' })
+      if (!response.ok) {
+        await readLimited(response, Math.min(options.maxResponseBytes ?? DEFAULT_MAX_BYTES, 512 * 1024))
+        const retryable = retryableStatus(response.status); const error = makeError(`HTTP ${response.status}`, retryable ? 'LCX_HTTP_RETRYABLE' : 'LCX_HTTP_ERROR', { status: response.status, retryable, requestId: response.headers.get('x-request-id') ?? response.headers.get('request-id') ?? undefined })
+        if (!retryable || attempt >= maxAttempts) throw error; last = error; await sleep(delayFromHeaders(response.headers, attempt), signal); continue
       }
+      if (typeof options.consume === 'function') return await options.consume(response, { requestSignal }); return response
+    } catch (error) {
+      if (signal?.aborted) throw signal.reason ?? error
+      const retryable = error?.retryable === true || error?.code === 'LCX_HTTP_RETRYABLE' || error instanceof TypeError || ['ECONNRESET','ECONNREFUSED','ETIMEDOUT','EAI_AGAIN','UND_ERR_CONNECT_TIMEOUT'].includes(error?.code)
+      if (!retryable || attempt >= maxAttempts) { if (attempt >= maxAttempts && retryable) throw makeError(`request failed after ${maxAttempts} attempts`, 'LCX_RETRY_EXHAUSTED', { cause: error, retryable: true }); throw error }
+      last = error; await sleep(Math.min(10_000, 500 * 2 ** (attempt - 1)), signal)
     }
-    throw new LcxHttpError('LCX SSE retry loop exhausted', { code: 'LCX_RETRY_EXHAUSTED' })
-  } finally {
-    releaseSignal(operationSignal)
   }
+  throw last ?? makeError('request failed', 'LCX_REQUEST_FAILED')
 }
 
-export async function fetchJsonWithRetry(url, body, headers, signal, timeoutMs, options = {}) {
-  const maxAttempts = Math.max(1, Math.min(options.maxAttempts ?? 3, 6))
-  const baseDelayMs = Math.max(0, options.baseDelayMs ?? 250)
-  const operationSignal = combinedSignal(signal, timeoutMs)
-  holdSignal(operationSignal)
-  let attempt = 0
+export async function consumeSse(response, onEvent, options = {}) {
+  if (!response?.body) throw makeError('SSE response has no body', 'LCX_INVALID_SSE')
+  const maxBytes = options.maxResponseBytes ?? DEFAULT_MAX_BYTES; const reader = response.body.getReader(); const decoder = new TextDecoder(); let pending = ''; let dataLines = []; let bytes = 0
+  const dispatch = () => { if (dataLines.length === 0) return; const data = dataLines.join('\n'); dataLines = []; if (data === '[DONE]') return; let event; try { event = JSON.parse(data) } catch (cause) { throw makeError('malformed SSE JSON', 'LCX_INVALID_SSE', { cause }) } onEvent(event) }
   try {
-    while (attempt < maxAttempts) {
-      attempt += 1
-      try {
-        abortRequestIfNeeded(operationSignal, signal, timeoutMs)
-        return await fetchJson(url, body, headers, signal, timeoutMs, { ...options, requestSignal: operationSignal })
-      } catch (error) {
-        abortIfNeeded(signal)
-        if (operationSignal.aborted) abortRequestIfNeeded(operationSignal, signal, timeoutMs)
-        if (!transientNetworkError(error) || attempt >= maxAttempts) throw error
-        await waitForRetry(retryWaitMilliseconds(error, baseDelayMs, attempt), operationSignal, signal, timeoutMs)
-      }
-    }
-    throw new LcxHttpError('LCX retry loop exhausted', { code: 'LCX_RETRY_EXHAUSTED' })
-  } finally {
-    releaseSignal(operationSignal)
-  }
+    while (true) { abortIfNeeded(options.signal); const { done, value } = await reader.read(); if (done) break; bytes += value.byteLength; if (bytes > maxBytes) throw makeError(`SSE response exceeds ${maxBytes} bytes`, 'LCX_RESPONSE_TOO_LARGE'); pending += decoder.decode(value, { stream: true }); let newline; while ((newline = pending.indexOf('\n')) >= 0) { const line = pending.slice(0, newline).replace(/\r$/u, ''); pending = pending.slice(newline + 1); if (line === '') dispatch(); else if (!line.startsWith(':') && line.startsWith('data:')) dataLines.push(line.slice(5).replace(/^ /u, '')) } }
+    pending += decoder.decode(); if (pending.startsWith('data:')) dataLines.push(pending.slice(5).replace(/^ /u, '')); dispatch()
+  } finally { await reader.cancel().catch(() => undefined) }
 }

package/lib/web-run-output.js

@@ -3,155 +3,16 @@ const CITATION_PATTERN = /cite([^]+)/gu
 const PAGE_LINE_PATTERN = /^L(\d+)(?:@P(\d+)(?:-(\d+))?)?:\s?(.*)$/u
 const EMBEDDED_PAGE_LINE_PATTERN = / (?=L\d+(?:@P\d+(?:-\d+)?)?:)/gu
 const TITLE_URL_PATTERN = /^(.*?)\s*\((https?:\/\/[^)]+)\)\s*$/u
-
-function pushUnique(target, value) {
-  if (!target.includes(value)) target.push(value)
-}
-
-function pushLink(target, payload) {
-  const [idValue, labelValue, domainValue] = payload.split('†')
-  if (!/^\d+$/u.test(idValue ?? '') || !labelValue) return false
-  const id = Number(idValue)
-  if (!Number.isSafeInteger(id)) return false
-  const label = labelValue.trim().slice(0, 500)
-  const domain = domainValue?.trim().slice(0, 253)
-  if (!label) return false
-  const existing = target.find((link) => link.id === id)
-  if (!existing) target.push({ id, label, ...(domain ? { domain } : {}) })
-  else if (!existing.domain && domain) existing.domain = domain
-  return true
-}
-
-function cleanCitations(value, references, links) {
-  return String(value ?? '').replace(CITATION_PATTERN, (_match, payload) => {
-    if (/^turn[\w-]+$/u.test(payload)) {
-      pushUnique(references, payload)
-      return ''
-    }
-    if (pushLink(links, payload)) return payload.split('†')[1]
-    const separator = payload.indexOf('†')
-    return separator < 0 ? payload : payload.slice(separator + 1)
-  }).trim()
-}
-
-function metadataParts(value) {
-  return value
-    .replace(/^\[wordlim:\s*(\d+)\]\s*/u, '$1-word excerpt; ')
-    .split(/;\s*/u)
-    .map((part) => part.trim())
-    .filter(Boolean)
-    .map((part) => {
-      const contentType = part.match(/^Content type:\s*(.+)$/u)
-      if (contentType) return contentType[1] === 'text/html' ? 'HTML' : contentType[1] === 'application/pdf' ? 'PDF' : contentType[1]
-      const totalLines = part.match(/^Total lines:\s*(\d+)$/u)
-      if (totalLines) return `${totalLines[1]} lines`
-      const pages = part.match(/^Number of pages:\s*(\d+)$/u)
-      if (pages) return `${pages[1]} pages`
-      return part
-    })
-}
-
-function isMetadata(value) {
-  return value.startsWith('[wordlim:') || /^(?:Published|Crawled|Content type|Source|Total lines|Number of pages):/u.test(value)
-}
-
-function parseLine(value, references, links) {
-  const clean = cleanCitations(value, references, links)
-  const pageLine = clean.match(PAGE_LINE_PATTERN)
-  if (pageLine) {
-    const text = pageLine[4] ?? ''
-    const heading = text.match(/^(#{1,6})\s+(.+)$/u)
-    return {
-      line: Number(pageLine[1]),
-      ...(pageLine[2] === undefined ? {} : { page: Number(pageLine[2]) }),
-      ...(pageLine[3] === undefined ? {} : { pageEnd: Number(pageLine[3]) }),
-      text: heading?.[2] ?? text,
-      ...(heading ? { heading: heading[1].length } : {}),
-    }
-  }
-  const heading = clean.match(/^(#{1,6})\s+(.+)$/u)
-  return heading ? { text: heading[2], heading: heading[1].length } : { text: clean }
-}
-
-function parseBlock(value) {
-  const references = []
-  const links = []
-  const rawLines = String(value ?? '')
-    .replace(EMBEDDED_PAGE_LINE_PATTERN, '\n')
-    .split(/\r?\n/u)
-    .map((line) => line.trimEnd())
-  while (rawLines[0]?.trim() === '') rawLines.shift()
-  while (rawLines.at(-1)?.trim() === '') rawLines.pop()
-  if (rawLines.length === 0) return undefined
-
-  const firstLine = cleanCitations(rawLines[0], references, links)
-  const header = firstLine.match(TITLE_URL_PATTERN)
-  const titleOnly = firstLine.match(/^(.+?)\s*\(\)\s*$/u)
-  let title = header?.[1]?.trim() || titleOnly?.[1]?.trim() || undefined
-  const url = header?.[2]
-  let bodyStart = header || titleOnly ? 1 : 0
-  if (!header && /^\s*\([^)]*\)\s*$/u.test(firstLine)) bodyStart = 1
-
-  const metadata = []
-  while (bodyStart < rawLines.length) {
-    const clean = cleanCitations(rawLines[bodyStart], references, links)
-    if (!isMetadata(clean)) break
-    metadata.push(...metadataParts(clean))
-    bodyStart += 1
-  }
-  const lines = rawLines.slice(bodyStart).map((line) => parseLine(line, references, links))
-  if (!title && url) {
-    try { title = new URL(url).hostname } catch { title = url }
-  }
-  return {
-    ...(title ? { title } : {}),
-    ...(url ? { url } : {}),
-    references,
-    links,
-    metadata,
-    lines,
-  }
-}
-
-export function parseWebRunOutput(output) {
-  if (typeof output !== 'string' || output.length === 0) return []
-  return output.split(RESULT_SEPARATOR_PATTERN).map(parseBlock).filter(Boolean)
-}
-
-export function outputLineRange(blocks) {
-  const numbers = (blocks ?? []).flatMap((block) => (block.lines ?? []).flatMap((line) => line.line === undefined ? [] : [line.line]))
-  if (numbers.length === 0) return undefined
-  return { first: Math.min(...numbers), last: Math.max(...numbers) }
-}
-
-export function outputDomains(blocks) {
-  const domains = []
-  for (const block of blocks ?? []) {
-    if (!block.url) continue
-    try { pushUnique(domains, new URL(block.url).hostname) } catch { /* ignore malformed source URLs */ }
-  }
-  return domains
-}
-
-export function outputLinks(blocks) {
-  const links = []
-  for (const block of blocks ?? []) {
-    for (const link of block.links ?? []) {
-      if (!links.some((value) => value.id === link.id)) links.push({ ...link })
-    }
-  }
-  return links
-}
-
-export function outputPdfRefs(blocks) {
-  const refs = []
-  for (const block of blocks ?? []) {
-    if (!(block.metadata ?? []).includes('PDF')) continue
-    for (const ref of block.references ?? []) pushUnique(refs, ref)
-  }
-  return refs
-}
-
-export function blockPlainText(block) {
-  return (block?.lines ?? []).map((line) => line.text).filter(Boolean).join(' ')
-}
+function pushUnique(target, value) { if (!target.includes(value)) target.push(value) }
+function pushLink(target, payload) { const [idValue, labelValue, domainValue] = payload.split('†'); if (!/^\d+$/u.test(idValue ?? '') || !labelValue) return false; const id = Number(idValue); if (!Number.isSafeInteger(id)) return false; const label = labelValue.trim().slice(0, 500); const domain = domainValue?.trim().slice(0, 253); if (!label) return false; const existing = target.find((link) => link.id === id); if (!existing) target.push({ id, label, ...(domain ? { domain } : {}) }); else if (!existing.domain && domain) existing.domain = domain; return true }
+function cleanCitations(value, references, links) { return String(value ?? '').replace(CITATION_PATTERN, (_match, payload) => { if (/^turn[\w-]+$/u.test(payload)) { pushUnique(references, payload); return '' } if (pushLink(links, payload)) return payload.split('†')[1]; const separator = payload.indexOf('†'); return separator < 0 ? payload : payload.slice(separator + 1) }).trim() }
+function metadataParts(value) { return value.replace(/^\[wordlim:\s*(\d+)\]\s*/u, '$1-word excerpt; ').split(/;\s*/u).map((p) => p.trim()).filter(Boolean).map((part) => { const contentType = part.match(/^Content type:\s*(.+)$/u); if (contentType) return contentType[1] === 'text/html' ? 'HTML' : contentType[1] === 'application/pdf' ? 'PDF' : contentType[1]; const totalLines = part.match(/^Total lines:\s*(\d+)$/u); if (totalLines) return `${totalLines[1]} lines`; const pages = part.match(/^Number of pages:\s*(\d+)$/u); if (pages) return `${pages[1]} pages`; return part }) }
+function isMetadata(value) { return value.startsWith('[wordlim:') || /^(?:Published|Crawled|Content type|Source|Total lines|Number of pages):/u.test(value) }
+function parseLine(value, references, links) { const clean = cleanCitations(value, references, links); const pageLine = clean.match(PAGE_LINE_PATTERN); if (pageLine) { const text = pageLine[4] ?? ''; const heading = text.match(/^(#{1,6})\s+(.+)$/u); return { line: Number(pageLine[1]), ...(pageLine[2] === undefined ? {} : { page: Number(pageLine[2]) }), ...(pageLine[3] === undefined ? {} : { pageEnd: Number(pageLine[3]) }), text: heading?.[2] ?? text, ...(heading ? { heading: heading[1].length } : {}) } } const heading = clean.match(/^(#{1,6})\s+(.+)$/u); return heading ? { text: heading[2], heading: heading[1].length } : { text: clean } }
+function parseBlock(value) { const references = [], links = []; const rawLines = String(value ?? '').replace(EMBEDDED_PAGE_LINE_PATTERN, '\n').split(/\r?\n/u).map((line) => line.trimEnd()); while (rawLines[0]?.trim() === '') rawLines.shift(); while (rawLines.at(-1)?.trim() === '') rawLines.pop(); if (!rawLines.length) return undefined; const firstLine = cleanCitations(rawLines[0], references, links); const header = firstLine.match(TITLE_URL_PATTERN); const titleOnly = firstLine.match(/^(.+?)\s*\(\)\s*$/u); let title = header?.[1]?.trim() || titleOnly?.[1]?.trim() || undefined; const url = header?.[2]; let bodyStart = header || titleOnly ? 1 : 0; if (!header && /^\s*\([^)]*\)\s*$/u.test(firstLine)) bodyStart = 1; const metadata = []; while (bodyStart < rawLines.length) { const clean = cleanCitations(rawLines[bodyStart], references, links); if (!isMetadata(clean)) break; metadata.push(...metadataParts(clean)); bodyStart += 1 } const lines = rawLines.slice(bodyStart).map((line) => parseLine(line, references, links)); if (!title && url) { try { title = new URL(url).hostname } catch { title = url } } return { ...(title ? { title } : {}), ...(url ? { url } : {}), references, links, metadata, lines } }
+export function parseWebRunOutput(output) { return typeof output === 'string' && output ? output.split(RESULT_SEPARATOR_PATTERN).map(parseBlock).filter(Boolean) : [] }
+export function outputLineRange(blocks) { const numbers = (blocks ?? []).flatMap((block) => (block.lines ?? []).flatMap((line) => line.line === undefined ? [] : [line.line])); return numbers.length ? { first: Math.min(...numbers), last: Math.max(...numbers) } : undefined }
+export function outputDomains(blocks) { const domains = []; for (const block of blocks ?? []) if (block.url) { try { pushUnique(domains, new URL(block.url).hostname) } catch {} } return domains }
+export function outputLinks(blocks) { const links = []; for (const block of blocks ?? []) for (const link of block.links ?? []) if (!links.some((v) => v.id === link.id)) links.push({ ...link }); return links }
+export function outputPdfRefs(blocks) { const refs = []; for (const block of blocks ?? []) { if (!(block.metadata ?? []).includes('PDF')) continue; for (const ref of block.references ?? []) pushUnique(refs, ref) } return refs }
+export function blockPlainText(block) { return (block?.lines ?? []).map((line) => line.text).filter(Boolean).join(' ') }