@neurocode-ai/app 1.18.8

package/e2e/utils/sse-transport.ts

@@ -0,0 +1,312 @@
+import type { Page } from "@playwright/test"
+
+export type SseConnectionRecord = {
+  id: number
+  url: string
+  path: "/global/event" | "/event" | "/api/event"
+  headers: Record<string, string>
+  openedAt: number
+  endedAt?: number
+  endedBy?: "close" | "disconnect" | "error" | "abort"
+  error?: string
+}
+
+export type SseDeliveryAcknowledgement = {
+  deliveryID: number
+  connectionID: number
+  bytes: number
+  chunkCount: number
+  deliveredAt: number
+  eventID?: string
+}
+
+export type SseEventOptions = {
+  id?: string
+  event?: string
+  retry?: number
+  marker?: string
+}
+
+export type SseTransport<T> = {
+  server: string
+  waitForConnection(options?: { after?: number; timeout?: number }): Promise<SseConnectionRecord>
+  send(payload: T, options?: SseEventOptions): Promise<SseDeliveryAcknowledgement>
+  burst(payloads: readonly T[], options?: readonly SseEventOptions[]): Promise<SseDeliveryAcknowledgement[]>
+  split(payload: T, cuts: readonly number[], options?: SseEventOptions): Promise<SseDeliveryAcknowledgement>
+  heartbeat(options?: SseEventOptions): Promise<SseDeliveryAcknowledgement>
+  writeRaw(value: string | Uint8Array, cuts?: readonly number[], marker?: string): Promise<SseDeliveryAcknowledgement>
+  close(): Promise<void>
+  disconnect(message?: string): Promise<void>
+  error(message?: string): Promise<void>
+  connections(): Promise<SseConnectionRecord[]>
+  acknowledgements(): Promise<SseDeliveryAcknowledgement[]>
+}
+
+type BrowserCommand<T> =
+  | { type: "send"; deliveries: { payload: T; options?: SseEventOptions }[]; burst: boolean; cuts?: number[] }
+  | { type: "raw"; bytes: number[]; cuts?: number[]; marker?: string }
+  | { type: "end"; mode: "close" | "disconnect" | "error"; message?: string }
+  | { type: "connections" }
+  | { type: "acknowledgements" }
+
+type BrowserTransport = Window & {
+  __testSseTransport?: {
+    command: (command: BrowserCommand<unknown>) => unknown
+  }
+}
+
+export async function installSseTransport<T>(
+  page: Page,
+  options: { server: string; retry?: number },
+): Promise<SseTransport<T>> {
+  const server = new URL(options.server).origin
+  await page.addInitScript(
+    ({ server, retry }) => {
+      type Connection = SseConnectionRecord & { controller: ReadableStreamDefaultController<Uint8Array> }
+      type ProbeWindow = Window & {
+        __visualStabilityProbe?: { startedAt: number; markers: { at: number; label: string }[] }
+      }
+      const originalFetch = window.fetch.bind(window)
+      const connections: Connection[] = []
+      const acknowledgements: SseDeliveryAcknowledgement[] = []
+      const encoder = new TextEncoder()
+      let nextConnectionID = 0
+      let nextDeliveryID = 0
+
+      const current = () => connections.findLast((connection) => connection.endedAt === undefined)
+      const chunks = (bytes: Uint8Array, cuts?: readonly number[]) => {
+        const boundaries = [...new Set(cuts ?? [])]
+          .filter((cut) => Number.isInteger(cut) && cut > 0 && cut < bytes.byteLength)
+          .sort((a, b) => a - b)
+        return [0, ...boundaries].map((start, index) => bytes.slice(start, boundaries[index] ?? bytes.byteLength))
+      }
+      const marker = (label?: string) => {
+        if (!label) return
+        const probe = (window as ProbeWindow).__visualStabilityProbe
+        if (!probe) return
+        probe.markers.push({ at: performance.now() - probe.startedAt, label })
+      }
+      const frame = (payload: unknown, eventOptions: SseEventOptions = {}) =>
+        [
+          eventOptions.event === undefined ? "" : `event: ${eventOptions.event}\n`,
+          eventOptions.id === undefined ? "" : `id: ${eventOptions.id}\n`,
+          eventOptions.retry === undefined ? "" : `retry: ${eventOptions.retry}\n`,
+          `data: ${JSON.stringify(payload)}\n\n`,
+        ].join("")
+      const currentEvent = (input: unknown) => {
+        if (!input || typeof input !== "object" || !("payload" in input)) return input
+        const envelope = input as { directory?: string; payload?: unknown }
+        if (!envelope.payload || typeof envelope.payload !== "object") return input
+        const payload = envelope.payload as { id?: string; type?: string; properties?: unknown }
+        if (!payload.type) return input
+        return {
+          id: payload.id ?? `evt_mock_${Date.now()}`,
+          created: Date.now(),
+          type: payload.type,
+          data: payload.properties ?? {},
+          location:
+            envelope.directory && envelope.directory !== "global" ? { directory: envelope.directory } : undefined,
+        }
+      }
+      const acknowledge = (
+        connection: Connection,
+        bytes: number,
+        chunkCount: number,
+        eventID?: string,
+      ): SseDeliveryAcknowledgement => {
+        const acknowledgement = {
+          deliveryID: ++nextDeliveryID,
+          connectionID: connection.id,
+          bytes,
+          chunkCount,
+          deliveredAt: performance.now(),
+          ...(eventID === undefined ? {} : { eventID }),
+        }
+        acknowledgements.push(acknowledgement)
+        return acknowledgement
+      }
+      const end = (mode: "close" | "disconnect" | "error", message?: string) => {
+        const connection = current()
+        if (!connection) throw new Error("SSE transport has no active connection")
+        connection.endedAt = performance.now()
+        connection.endedBy = mode
+        if (message) connection.error = message
+        if (mode === "close") {
+          connection.controller.close()
+          return
+        }
+        const error = new DOMException(
+          message ?? "SSE connection disconnected",
+          mode === "error" ? "Error" : "NetworkError",
+        )
+        connection.controller.error(error)
+      }
+
+      const command = (input: BrowserCommand<unknown>) => {
+        if (input.type === "connections")
+          return connections.map(({ controller: _controller, ...connection }) => connection)
+        if (input.type === "acknowledgements") return acknowledgements
+        if (input.type === "end") return end(input.mode, input.message)
+        const connection = current()
+        if (!connection) throw new Error("SSE transport has no active connection")
+        if (input.type === "raw") {
+          marker(input.marker)
+          const output = chunks(new Uint8Array(input.bytes), input.cuts)
+          output.forEach((chunk) => connection.controller.enqueue(chunk))
+          return acknowledge(connection, input.bytes.length, output.length)
+        }
+        const encoded = input.deliveries.map((delivery) => {
+          const payload = connection.path === "/api/event" ? currentEvent(delivery.payload) : delivery.payload
+          return { delivery, payload, bytes: encoder.encode(frame(payload, delivery.options)) }
+        })
+        encoded.forEach((item) => marker(item.delivery.options?.marker))
+        if (input.burst) {
+          const bytes = encoder.encode(encoded.map((item) => frame(item.payload, item.delivery.options)).join(""))
+          connection.controller.enqueue(bytes)
+          return encoded.map((item) => acknowledge(connection, item.bytes.byteLength, 1, item.delivery.options?.id))
+        }
+        const output = chunks(encoded[0]!.bytes, input.cuts)
+        output.forEach((chunk) => connection.controller.enqueue(chunk))
+        return acknowledge(connection, encoded[0]!.bytes.byteLength, output.length, encoded[0]!.delivery.options?.id)
+      }
+
+      ;(window as BrowserTransport).__testSseTransport = { command }
+      const fetch = (input: RequestInfo | URL, init?: RequestInit) => {
+        const request = new Request(input, init)
+        const url = new URL(request.url)
+        if (
+          url.origin !== server ||
+          (url.pathname !== "/global/event" && url.pathname !== "/event" && url.pathname !== "/api/event")
+        )
+          return originalFetch(request)
+
+        const id = ++nextConnectionID
+        const record = {
+          id,
+          url: url.href,
+          path: url.pathname,
+          headers: Object.fromEntries(request.headers.entries()),
+          openedAt: performance.now(),
+        } as Connection
+        const stream = new ReadableStream<Uint8Array>({
+          start(controller) {
+            record.controller = controller
+            connections.push(record)
+            if (retry !== undefined) controller.enqueue(encoder.encode(`retry: ${retry}\n\n`))
+            if (url.pathname === "/api/event")
+              controller.enqueue(
+                encoder.encode(frame({ id: `evt_mock_connected_${id}`, type: "server.connected", data: {} })),
+              )
+            if (url.pathname === "/global/event")
+              controller.enqueue(
+                encoder.encode(
+                  frame({
+                    payload: { id: `evt_mock_connected_${id}`, type: "server.connected", properties: {} },
+                  }),
+                ),
+              )
+            request.signal.addEventListener(
+              "abort",
+              () => {
+                if (record.endedAt !== undefined) return
+                record.endedAt = performance.now()
+                record.endedBy = "abort"
+                controller.error(request.signal.reason ?? new DOMException("The operation was aborted", "AbortError"))
+              },
+              { once: true },
+            )
+          },
+          cancel() {
+            if (record.endedAt !== undefined) return
+            record.endedAt = performance.now()
+            record.endedBy = "disconnect"
+          },
+        })
+        return Promise.resolve(
+          new Response(stream, {
+            status: 200,
+            headers: {
+              "cache-control": "no-cache",
+              "content-type": "text/event-stream",
+            },
+          }),
+        )
+      }
+      Object.defineProperty(window, "fetch", { configurable: true, writable: true, value: fetch })
+    },
+    { server, retry: options.retry },
+  )
+
+  const command = <Result>(input: BrowserCommand<T>) =>
+    page.evaluate((input) => {
+      const transport = (window as BrowserTransport).__testSseTransport
+      if (!transport) throw new Error("SSE transport was not installed before page load")
+      return transport.command(input as BrowserCommand<unknown>)
+    }, input) as Promise<Result>
+
+  return {
+    server,
+    async waitForConnection(input = {}) {
+      await page.waitForFunction(
+        (after) => {
+          const transport = (window as BrowserTransport).__testSseTransport
+          const connections = transport?.command({ type: "connections" }) as SseConnectionRecord[] | undefined
+          return connections?.some((connection) => connection.id > after)
+        },
+        input.after ?? 0,
+        { timeout: input.timeout },
+      )
+      return (await command<SseConnectionRecord[]>({ type: "connections" })).findLast(
+        (connection) => connection.id > (input.after ?? 0),
+      )!
+    },
+    send(payload, eventOptions) {
+      return command({ type: "send", deliveries: [{ payload, options: eventOptions }], burst: false })
+    },
+    burst(payloads, eventOptions = []) {
+      return command({
+        type: "send",
+        deliveries: payloads.map((payload, index) => ({ payload, options: eventOptions[index] })),
+        burst: true,
+      })
+    },
+    split(payload, cuts, eventOptions) {
+      return command({ type: "send", deliveries: [{ payload, options: eventOptions }], burst: false, cuts: [...cuts] })
+    },
+    heartbeat(eventOptions) {
+      return command({
+        type: "send",
+        deliveries: [
+          {
+            payload: { directory: "global", payload: { type: "server.heartbeat", properties: {} } } as T,
+            options: eventOptions,
+          },
+        ],
+        burst: false,
+      })
+    },
+    writeRaw(value, cuts, marker) {
+      return command({
+        type: "raw",
+        bytes: Array.from(typeof value === "string" ? new TextEncoder().encode(value) : value),
+        cuts: cuts ? [...cuts] : undefined,
+        marker,
+      })
+    },
+    close() {
+      return command({ type: "end", mode: "close" })
+    },
+    disconnect(message) {
+      return command({ type: "end", mode: "disconnect", message })
+    },
+    error(message) {
+      return command({ type: "end", mode: "error", message })
+    },
+    connections() {
+      return command({ type: "connections" })
+    },
+    acknowledgements() {
+      return command({ type: "acknowledgements" })
+    },
+  }
+}

package/e2e/utils/visual-stability/analyzer.ts

@@ -0,0 +1,209 @@
+import type { VisualInvariant, VisualPlan } from "./invariant"
+import type { VisualObservation, VisualStabilityTrace } from "./model"
+
+export function analyzeVisualObservations<RegionName extends string>(
+  observations: readonly VisualObservation<RegionName>[],
+  plan: VisualPlan<RegionName>,
+) {
+  const issues: string[] = []
+  const invariants = plan.invariants
+  const names = [...new Set(observations.flatMap((sample) => Object.keys(sample.regions) as RegionName[]))]
+  const required = regions(invariants, "required")
+  const continuousAny = invariants.filter(
+    (invariant): invariant is Extract<VisualInvariant<RegionName>, { type: "continuous-any" }> =>
+      invariant.type === "continuous-any",
+  )
+  const unique = new Set(regions(invariants, "unique"))
+  const stable = new Set(regions(invariants, "stable"))
+  const fixed = invariants.filter(
+    (invariant): invariant is Extract<VisualInvariant<RegionName>, { type: "fixed" }> => invariant.type === "fixed",
+  )
+  const opacity = invariants.filter(
+    (invariant): invariant is Extract<VisualInvariant<RegionName>, { type: "opacity" }> => invariant.type === "opacity",
+  )
+  const continuity = invariants.filter(
+    (invariant): invariant is Extract<VisualInvariant<RegionName>, { type: "continuity" }> =>
+      invariant.type === "continuity",
+  )
+  const motion = invariants.filter(
+    (invariant): invariant is Extract<VisualInvariant<RegionName>, { type: "motion" }> => invariant.type === "motion",
+  )
+  const labelStability = invariants.filter(
+    (invariant): invariant is Extract<VisualInvariant<RegionName>, { type: "label-stability" }> =>
+      invariant.type === "label-stability",
+  )
+
+  for (const name of new Set(required)) {
+    if (!observations.some((sample) => sample.regions[name]?.visible)) issues.push(`${name} never rendered`)
+  }
+  for (const invariant of continuousAny) {
+    if (!invariant.regions.some((name) => observations.some((sample) => sample.regions[name]?.visible)))
+      issues.push(`${invariant.regions.join(" | ")} never rendered`)
+  }
+
+  for (const name of names) {
+    const samples = observations.flatMap((observation) => {
+      const region = observation.regions[name]
+      if (!region) return []
+      const clipped =
+        observation.viewport && (region.bottom <= observation.viewport.top || region.top >= observation.viewport.bottom)
+      return [{ at: observation.at, ...region, visible: region.visible && !clipped }]
+    })
+    const visible = samples.filter((sample) => sample.visible)
+    if (visible.length === 0) continue
+    if (unique.has(name)) {
+      const duplicate = samples.find((sample) => sample.count > 1)
+      if (duplicate) issues.push(`${name} appeared ${duplicate.count} times at ${Math.round(duplicate.at)}ms`)
+    }
+    if (stable.has(name)) {
+      const identities = [...new Set(visible.map((sample) => sample.node).filter((node) => node > 0))]
+      if (identities.length > 1) issues.push(`${name} remounted ${identities.length - 1} times`)
+    }
+    for (const invariant of fixed.filter((invariant) => includes(invariant.regions, name))) {
+      const origin = visible[0]
+      const movement = origin ? Math.max(0, ...visible.map((sample) => Math.abs(sample.top - origin.top))) : 0
+      if (movement > (invariant.tolerance ?? 1))
+        issues.push(`${name} moved ${Math.round(movement * 10) / 10}px in the viewport`)
+    }
+    for (const invariant of opacity.filter((invariant) => includes(invariant.regions, name))) {
+      for (const sample of visible) {
+        if (sample.opacity < (invariant.floor ?? 0.65))
+          issues.push(`${name} opacity fell to ${sample.opacity} at ${Math.round(sample.at)}ms`)
+      }
+    }
+    if (continuity.some((invariant) => includes(invariant.regions, name))) {
+      const firstPresent = samples.findIndex((sample) => sample.present)
+      const lastPresent = samples.findLastIndex((sample) => sample.present)
+      if (samples.slice(firstPresent, lastPresent + 1).some((sample) => !sample.present))
+        issues.push(`${name} disappeared between present frames`)
+      const firstVisible = samples.findIndex((sample) => sample.visible)
+      const lastVisible = samples.findLastIndex((sample) => sample.visible)
+      if (
+        firstVisible >= 0 &&
+        samples.slice(firstVisible, lastVisible + 1).some((sample) => !sample.visible && sample.inViewport)
+      )
+        issues.push(`${name} blanked between visible frames`)
+    }
+    for (const invariant of motion.filter((invariant) => includes(invariant.regions, name))) {
+      for (const metric of ["top", "bottom", "width", "height"] as const) {
+        const directions = visible
+          .slice(1)
+          .map((sample, index) => sample[metric] - visible[index]![metric])
+          .filter((delta) => Math.abs(delta) > (invariant.tolerance ?? 1))
+          .map(Math.sign)
+        const reversals = directions.slice(1).filter((direction, index) => direction !== directions[index]).length
+        const allowed =
+          metric === "top" || metric === "bottom"
+            ? (invariant.maxPositionReversals ?? invariant.maxReversals ?? 1)
+            : (invariant.maxReversals ?? 1)
+        if (reversals > allowed) issues.push(`${name} ${metric} reversed ${reversals} times`)
+      }
+    }
+    if (labelStability.some((invariant) => includes(invariant.regions, name))) {
+      const labels = samples
+        .map((sample) => sample.label)
+        .filter((label) => label.length > 0)
+        .filter((label, index, all) => label !== all[index - 1])
+      if (labels.some((label, index) => labels.indexOf(label) !== index))
+        issues.push(`${name} label reverted: ${labels.join(" -> ")}`)
+    }
+  }
+
+  if (invariants.some((invariant) => invariant.type === "preserve-bottom-anchor")) {
+    const viewports = observations.flatMap((sample) => (sample.viewport ? [sample.viewport] : []))
+    if (viewports[0] && viewports[0].distanceFromBottom <= 4) {
+      const lost = viewports.find((viewport) => viewport.distanceFromBottom > 4)
+      if (lost) issues.push(`bottom anchor moved to ${lost.distanceFromBottom}px`)
+    }
+  }
+  if (invariants.some((invariant) => invariant.type === "acquire-bottom-anchor")) {
+    const final = observations.findLast((sample) => sample.viewport)?.viewport
+    if (!final || final.distanceFromBottom > 4)
+      issues.push(`did not acquire bottom anchor${final ? ` (${final.distanceFromBottom}px away)` : ""}`)
+  }
+
+  for (const invariant of continuousAny) {
+    const active = observations.map((sample) => invariant.regions.some((name) => sample.regions[name]?.visible))
+    const first = active.indexOf(true)
+    const last = active.lastIndexOf(true)
+    if (first >= 0 && active.slice(first, last + 1).some((value) => !value))
+      issues.push(`${invariant.regions.join(" | ")} blanked between visible frames`)
+  }
+
+  for (const invariant of invariants.filter(
+    (item): item is Extract<VisualInvariant<RegionName>, { type: "flow" }> => item.type === "flow",
+  )) {
+    for (const [before, after] of invariant.regions
+      .slice(1)
+      .map((after, index) => [invariant.regions[index]!, after])) {
+      let maximum: { overlap: number; at: number } | undefined
+      let inverted: { at: number } | undefined
+      for (const sample of observations) {
+        const first = sample.regions[before]
+        const second = sample.regions[after]
+        if (!first?.visible || !second?.visible) continue
+        if (
+          sample.viewport &&
+          (first.bottom <= sample.viewport.top ||
+            first.top >= sample.viewport.bottom ||
+            second.bottom <= sample.viewport.top ||
+            second.top >= sample.viewport.bottom)
+        )
+          continue
+        const overlap = first.bottom - second.top
+        if (first.top > second.top && !inverted) inverted = { at: sample.at }
+        if (overlap > (invariant.overlapTolerance ?? 0.5) && (!maximum || overlap > maximum.overlap))
+          maximum = { overlap, at: sample.at }
+      }
+      if (inverted) issues.push(`${before} rendered after ${after} at ${Math.round(inverted.at)}ms`)
+      if (maximum)
+        issues.push(
+          `${before} overlapped ${after} by ${Math.round(maximum.overlap * 10) / 10}px at ${Math.round(maximum.at)}ms`,
+        )
+    }
+  }
+  return [...new Set(issues)]
+}
+
+export function analyzeVisualTraceByMarker<RegionName extends string>(
+  trace: VisualStabilityTrace<RegionName>,
+  plan: VisualPlan<RegionName>,
+) {
+  if (trace.markers.length === 0) return analyzeVisualObservations(trace.samples, plan)
+  const required = [...new Set(plan.markerRequired ?? regions(plan.invariants, "required"))].flatMap((name) =>
+    trace.samples.some((sample) => sample.regions[name]?.visible) ? [] : [`${name} never rendered`],
+  )
+  const withoutRequired = plan.invariants.filter((invariant) => invariant.type !== "required")
+  const windows = trace.markers.flatMap((marker, index) => {
+    const end = trace.markers[index + 1]?.at ?? Infinity
+    const before = trace.samples.findLast((sample) => sample.at < marker.at)
+    const samples = [
+      ...(before ? [before] : []),
+      ...trace.samples.filter((sample) => sample.at >= marker.at && sample.at < end),
+    ]
+    if (samples.length < 2) return []
+    return analyzeVisualObservations(samples, { ...plan, perMarker: false, invariants: withoutRequired }).map(
+      (issue) => `${marker.label}: ${issue}`,
+    )
+  })
+  const aggregateMotion =
+    plan.aggregateMotion === false
+      ? []
+      : analyzeVisualObservations(trace.samples, {
+          invariants: plan.invariants.filter((invariant) => invariant.type === "motion"),
+        }).filter((issue) => / (?:top|bottom|width|height) reversed \d+ times$/.test(issue))
+  return [...new Set([...required, ...aggregateMotion, ...windows])]
+}
+
+function regions<RegionName extends string, Type extends VisualInvariant<RegionName>["type"]>(
+  invariants: readonly VisualInvariant<RegionName>[],
+  type: Type,
+) {
+  return invariants.flatMap((invariant) =>
+    invariant.type === type && "regions" in invariant && invariant.regions !== "all" ? [...invariant.regions] : [],
+  ) as RegionName[]
+}
+
+function includes<RegionName extends string>(regions: readonly RegionName[] | "all", name: RegionName) {
+  return regions === "all" || regions.includes(name)
+}

package/e2e/utils/visual-stability/capture.ts

@@ -0,0 +1,51 @@
+import type { CDPSession, Page } from "@playwright/test"
+import type { CapturedFrame } from "./model"
+
+export type VisualCapture = {
+  session: CDPSession
+  frames: CapturedFrame[]
+  startedAtEpoch: number
+  running: boolean
+  capture: Promise<void>
+}
+
+export async function startVisualCapture(page: Page, startedAtEpoch: number) {
+  if (process.env.OPENCODE_STABILITY_CAPTURE !== "1") return
+  const session = await page.context().newCDPSession(page)
+  await session.send("Page.enable")
+  const recording: VisualCapture = {
+    session,
+    frames: [],
+    startedAtEpoch,
+    running: true,
+    capture: Promise.resolve(),
+  }
+  recording.capture = (async () => {
+    try {
+      while (recording.running && recording.frames.length < 900) {
+        const frame = await session.send("Page.captureScreenshot", {
+          format: "jpeg",
+          quality: 80,
+          captureBeyondViewport: false,
+          optimizeForSpeed: true,
+        })
+        recording.frames.push({ at: Date.now() - recording.startedAtEpoch, data: frame.data })
+        await new Promise((resolve) => setTimeout(resolve, 50))
+      }
+    } catch {
+      recording.running = false
+    }
+  })()
+  return recording
+}
+
+export async function stopVisualCapture(recording: VisualCapture | undefined) {
+  if (!recording) return []
+  recording.running = false
+  try {
+    await recording.capture
+  } finally {
+    await recording.session.detach().catch(() => undefined)
+  }
+  return recording.frames
+}

package/e2e/utils/visual-stability/index.ts

@@ -0,0 +1,8 @@
+export * from "./analyzer"
+export * from "./capture"
+export * from "./invariant"
+export * from "./model"
+export * from "./probe"
+export * from "./regions"
+export * from "./reporter"
+export * from "./scenario"