@c9up/ream 0.2.25 → 0.2.27

package/src/profiler/Profiler.ts

@@ -0,0 +1,182 @@
+/**
+ * The profiler: one profile per request, and the token that finds it again.
+ *
+ * The hard part is attribution. A SQL query is reported by a listener the
+ * database package calls, with no idea which request caused it — and in a
+ * server answering three requests at once, "the current request" is not a
+ * variable anything can hold. `AsyncLocalStorage` is what makes it one: the
+ * request runs inside a store, and whatever a collector does during it, however
+ * deep in the await chain, lands in the right bag.
+ *
+ * Dev only, and hard-gated: a profiler that kept every request in production
+ * would be a memory leak wearing a feature's name.
+ */
+
+import { AsyncLocalStorage } from 'node:async_hooks'
+import { randomBytes } from 'node:crypto'
+import type { CollectionScope, Profile, ProfilerCollector, ProfileStore } from './types.js'
+
+/** What is live while one request runs. */
+interface ActiveProfile {
+  scope: CollectionScope
+  /** Each collector's bag, keyed by collector name. */
+  bags: Map<string, unknown>
+}
+
+/** Where the toolbar and the profiler page are served. */
+export const PROFILER_PATH = '/__ream/profiler'
+
+/** The header carrying the token back, so a client can ask for the profile. */
+export const PROFILER_TOKEN_HEADER = 'x-debug-token'
+
+export class Profiler {
+  readonly #storage = new AsyncLocalStorage<ActiveProfile>()
+  readonly #collectors: readonly ProfilerCollector[]
+  readonly #store: ProfileStore
+
+  constructor(collectors: readonly ProfilerCollector[], store: ProfileStore) {
+    this.#collectors = collectors
+    this.#store = store
+  }
+
+  get store(): ProfileStore {
+    return this.#store
+  }
+
+  get collectors(): readonly ProfilerCollector[] {
+    return this.#collectors
+  }
+
+  /**
+   * Run one request inside a profile.
+   *
+   * `work` is handed the scope so it can stamp the token on the response
+   * before anything else runs: a request that throws unwinds past its caller,
+   * and a token written after the await would be missing on exactly the
+   * request worth looking at.
+   *
+   * `describe` is called AFTER the work, because the status is only known
+   * then — asking for it up front would record the request as it arrived
+   * rather than as it was answered. It is given whatever was thrown, since a
+   * handler that threw has not had its status written yet.
+   */
+  async profile<T>(
+    request: { method: string; url: string },
+    work: (scope: CollectionScope) => Promise<T>,
+    describe: (error?: unknown) => { status: number },
+  ): Promise<{ result: T; token: string }> {
+    const token = newToken()
+    const startedAt = Date.now()
+    const started = performance.now()
+    const active: ActiveProfile = { scope: { token, startedAt }, bags: new Map() }
+
+    for (const collector of this.#collectors) {
+      // A collector that throws while starting must not fail the request it was
+      // only meant to watch.
+      try {
+        active.bags.set(collector.name, collector.start(active.scope))
+      } catch {
+        active.bags.set(collector.name, undefined)
+      }
+    }
+
+    // A request that threw is the profile you most want to look at, so it is
+    // recorded like any other and the error is re-thrown afterwards. Losing it
+    // here would mean the profiler goes quiet exactly when something broke.
+    let result: T | undefined
+    let thrown: unknown
+    let failed = false
+    try {
+      result = await this.#storage.run(active, () => work(active.scope))
+    } catch (error) {
+      thrown = error
+      failed = true
+      recordException(active, error)
+    }
+
+    const profile: Profile = {
+      token,
+      time: startedAt,
+      method: request.method,
+      url: request.url,
+      status: safeStatus(describe, thrown),
+      duration: performance.now() - started,
+      data: {},
+    }
+    for (const collector of this.#collectors) {
+      try {
+        profile.data[collector.name] = collector.finish(
+          active.bags.get(collector.name),
+          active.scope,
+        )
+      } catch {
+        // Same rule on the way out: one broken panel, not a broken response.
+        profile.data[collector.name] = {}
+      }
+    }
+    this.#store.save(profile)
+    if (failed) throw thrown
+    // Only reachable when the work resolved, which is when `result` is set.
+    return { result: result as T, token }
+  }
+
+  /**
+   * The bag of the collector named, for the request running right now.
+   *
+   * `undefined` outside a request — a query from a console command, a boot-time
+   * warm-up — and a collector that writes into it must accept that.
+   */
+  current<Bag>(collector: string): Bag | undefined {
+    const active = this.#storage.getStore()
+    if (active === undefined) return undefined
+    const bag = active.bags.get(collector)
+    return bag === undefined ? undefined : (bag as Bag)
+  }
+
+  /** Whether a request is being profiled right now. */
+  get active(): boolean {
+    return this.#storage.getStore() !== undefined
+  }
+}
+
+/**
+ * Put the error where the exception panel reads it.
+ *
+ * The collector's own bag, by name: the framework ships that collector, and a
+ * profiler that could not say what broke would miss the reason most people
+ * open it.
+ */
+function recordException(active: ActiveProfile, error: unknown): void {
+  const bag = active.bags.get('exception')
+  if (typeof bag !== 'object' || bag === null) return
+  const target: Record<string, unknown> = Object.assign(bag)
+  if (error instanceof Error) {
+    target.name = error.name
+    target.message = error.message
+    if (error.stack !== undefined) target.stack = error.stack
+    const code = Reflect.get(error, 'code')
+    if (typeof code === 'string') target.code = code
+  } else {
+    target.name = 'Error'
+    target.message = String(error)
+  }
+}
+
+/**
+ * A token, short enough to read aloud and random enough not to collide.
+ *
+ * Six bytes of hex, as Symfony's is six characters: it identifies a request
+ * within one dev session, not across the internet.
+ */
+function newToken(): string {
+  return randomBytes(6).toString('hex')
+}
+
+function safeStatus(describe: (error?: unknown) => { status: number }, error: unknown): number {
+  try {
+    const { status } = describe(error)
+    return Number.isInteger(status) ? status : 0
+  } catch {
+    return 0
+  }
+}

package/src/profiler/collectors.ts

@@ -0,0 +1,230 @@
+/**
+ * What the profiler watches.
+ *
+ * Each one is independent and may be replaced: the database collector is the
+ * interesting case, because the data does not come from the request at all. It
+ * comes from `db:query` events the data layer emits with no idea which request
+ * caused them, and the profiler's async store is what puts each one back where
+ * it belongs.
+ *
+ * NO import of atlas. The collector listens on the application's emitter for
+ * `db:query`, which is the shape atlas documents for bridging its own registry
+ * (`onDbQuery((event) => emitter.emit('db:query', event))`). A project on
+ * another data layer emits the same event and gets the same panel.
+ */
+
+import type { CollectedData, ProfilerCollector } from './types.js'
+
+/** One SQL statement, as the panel shows it. */
+export interface RecordedQuery {
+  sql: string
+  bindings: readonly unknown[]
+  /** Milliseconds. */
+  duration: number
+  connection?: string
+  model?: string
+  method?: string
+  inTransaction?: boolean
+  /** The message, when the statement threw. */
+  error?: string
+}
+
+/** What the emitter must give this collector. Structural: no package imported. */
+export interface QueryEventSource {
+  on(event: string, listener: (payload: unknown) => void): unknown
+}
+
+/**
+ * How many statements one request may record.
+ *
+ * A runaway loop issuing ten thousand queries is exactly the bug this panel
+ * exists to show, and keeping all ten thousand would take the dev server down
+ * with it. The count stays true past the cap; the list stops.
+ */
+const QUERY_CAP = 500
+
+export function databaseCollector(
+  events: QueryEventSource,
+  read: () => QueryBag | undefined,
+): ProfilerCollector<QueryBag> {
+  // Subscribed once, at construction: a listener added per request would leak
+  // one per request, and removing it on the way out would miss a query still
+  // in flight.
+  events.on('db:query', (payload) => {
+    const bag = read()
+    if (bag === undefined) return
+    bag.total += 1
+    bag.duration += durationOf(payload)
+    if (bag.queries.length < QUERY_CAP) bag.queries.push(toQuery(payload))
+  })
+
+  return {
+    name: 'database',
+    badge: (data) => {
+      const total = typeof data.total === 'number' ? data.total : 0
+      const duration = typeof data.duration === 'number' ? data.duration : 0
+      return `${total} ${total === 1 ? 'query' : 'queries'} · ${round(duration)} ms`
+    },
+    start: () => ({ total: 0, duration: 0, queries: [] }),
+    finish: (bag) =>
+      bag === undefined
+        ? { total: 0, duration: 0, queries: [], capped: false }
+        : {
+            total: bag.total,
+            duration: round(bag.duration),
+            queries: bag.queries,
+            capped: bag.total > bag.queries.length,
+          },
+  }
+}
+
+export interface QueryBag {
+  total: number
+  duration: number
+  queries: RecordedQuery[]
+}
+
+/** The request and what it answered. */
+export function requestCollector(): ProfilerCollector<RequestBag> {
+  return {
+    name: 'request',
+    badge: (data) => (typeof data.route === 'string' ? data.route : undefined),
+    start: () => ({}),
+    finish: (bag) => ({ ...(bag ?? {}) }),
+  }
+}
+
+export interface RequestBag {
+  route?: string
+  controller?: string
+  userId?: string
+}
+
+/**
+ * What went wrong, if anything.
+ *
+ * The one panel worth opening by itself: an exception that reached the handler
+ * is reported here so it shows up in the page rather than only in the terminal
+ * someone may not be looking at.
+ */
+export function exceptionCollector(): ProfilerCollector<ExceptionBag> {
+  return {
+    name: 'exception',
+    badge: (data) => (typeof data.message === 'string' ? 'error' : undefined),
+    start: () => ({}),
+    finish: (bag) => ({ ...(bag ?? {}) }),
+  }
+}
+
+export interface ExceptionBag {
+  name?: string
+  message?: string
+  stack?: string
+  /** Where it was reported from, when the framework knows. */
+  code?: string
+}
+
+/**
+ * Lines logged while the request ran.
+ *
+ * Same arrangement as the database panel, and for the same reason: a logger
+ * writes a line with no idea which request caused it. It comes off the
+ * application's emitter as `log:line`, which is what `@c9up/spectrum` bridges
+ * its own registry to (`onLog((entry) => emitter.emit('log:line', entry))`),
+ * so a project on another logger emits the same event and gets the same panel.
+ */
+export function logCollector(events: LogEventSource, read: () => LogBag | undefined) {
+  // Subscribed once, at construction, for the same reason as the queries: a
+  // listener per request leaks one per request.
+  events.on('log:line', (payload) => {
+    const bag = read()
+    if (bag === undefined) return
+    if (bag.lines.length < LINE_CAP) bag.lines.push(toLine(payload))
+    bag.total += 1
+  })
+
+  return {
+    name: 'logs',
+    badge: (data) => {
+      const total = typeof data.total === 'number' ? data.total : 0
+      return total === 0 ? undefined : `${total} ${total === 1 ? 'log' : 'logs'}`
+    },
+    start: () => ({ total: 0, lines: [] }),
+    // Nothing logged is nothing to show: an empty bag renders no panel at all,
+    // rather than a heading over an empty list.
+    finish: (bag) =>
+      bag === undefined || bag.total === 0 ? {} : { total: bag.total, lines: bag.lines },
+  } satisfies ProfilerCollector<LogBag>
+}
+
+/** What the emitter must give the log collector. Structural, as above. */
+export interface LogEventSource {
+  on(event: string, listener: (payload: unknown) => void): unknown
+}
+
+/** As many lines as are worth scrolling; the count stays true past it. */
+const LINE_CAP = 200
+
+export interface LogBag {
+  total: number
+  lines: Array<{ level: string; message: string; time?: string }>
+}
+
+function toLine(payload: unknown): LogBag['lines'][number] {
+  const entry = asRecord(payload)
+  return {
+    level: typeof entry.level === 'string' ? entry.level : 'info',
+    message: typeof entry.message === 'string' ? entry.message : String(entry.message ?? ''),
+    ...(typeof entry.timestamp === 'string' ? { time: entry.timestamp } : {}),
+  }
+}
+
+/** Peak heap while the request ran, which is what a leak shows up in. */
+export function memoryCollector(): ProfilerCollector<{ start: number }> {
+  return {
+    name: 'memory',
+    badge: (data) => (typeof data.usedMb === 'number' ? `${data.usedMb.toFixed(1)} MB` : undefined),
+    start: () => ({ start: process.memoryUsage().heapUsed }),
+    finish: (bag) => {
+      const used = process.memoryUsage().heapUsed
+      return {
+        usedMb: used / 1024 / 1024,
+        deltaMb: bag === undefined ? 0 : (used - bag.start) / 1024 / 1024,
+      }
+    },
+  }
+}
+
+function toQuery(payload: unknown): RecordedQuery {
+  const event = asRecord(payload)
+  return {
+    sql: typeof event.sql === 'string' ? event.sql : String(event.sql ?? ''),
+    bindings: Array.isArray(event.bindings) ? event.bindings : [],
+    duration: durationOf(payload),
+    ...pickString(event, 'connection'),
+    ...pickString(event, 'model'),
+    ...pickString(event, 'method'),
+    ...(event.inTransaction === true ? { inTransaction: true } : {}),
+    ...(event.error instanceof Error ? { error: event.error.message } : {}),
+  }
+}
+
+function durationOf(payload: unknown): number {
+  const value = asRecord(payload).duration
+  return typeof value === 'number' && Number.isFinite(value) ? value : 0
+}
+
+function pickString(event: Record<string, unknown>, key: string): CollectedData {
+  const value = event[key]
+  return typeof value === 'string' && value !== '' ? { [key]: value } : {}
+}
+
+function asRecord(value: unknown): Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+    ? Object.fromEntries(Object.entries(value))
+    : {}
+}
+
+function round(value: number): number {
+  return Math.round(value * 100) / 100
+}

package/src/profiler/endpoint.ts

@@ -0,0 +1,98 @@
+/**
+ * Where the toolbar and the profiler read from.
+ *
+ * Three routes, all under `/__ream/profiler` and all development-only:
+ *
+ *     /__ream/profiler                 the history, newest first
+ *     /__ream/profiler/<token>         one request, every panel
+ *     /__ream/profiler/<token>.json    the same, for the bar in the page
+ *
+ * Server middleware rather than routes, like the hot-reload endpoint: it
+ * answers before the application's own middleware, so a profiler request does
+ * not run the authentication stack and end up in its own history.
+ */
+
+import type { Profiler } from './Profiler.js'
+import { PROFILER_PATH } from './Profiler.js'
+import { renderProfileList, renderProfilePage } from './render.js'
+
+/** The slice of the context this endpoint uses. */
+export interface ProfilerContext {
+  request: { url: () => string }
+  response: {
+    header: (name: string, value: string) => unknown
+    send: (body: unknown) => unknown
+    status: (code: number) => unknown
+  }
+}
+
+export function profilerEndpoint(
+  profiler: Profiler,
+): (ctx: ProfilerContext, next: () => Promise<void>) => Promise<void> {
+  return async (ctx, next) => {
+    const path = ctx.request.url().split('?')[0] ?? ''
+    if (path !== PROFILER_PATH && !path.startsWith(`${PROFILER_PATH}/`)) {
+      await next()
+      return
+    }
+
+    if (path === PROFILER_PATH) {
+      ctx.response.header('content-type', 'text/html; charset=utf-8')
+      ctx.response.send(renderProfileList(profiler.store.recent(50)))
+      return
+    }
+
+    const rest = path.slice(`${PROFILER_PATH}/`.length)
+    const asJson = rest.endsWith('.json')
+    const token = asJson ? rest.slice(0, -'.json'.length) : rest
+    const profile = profiler.store.find(token)
+
+    if (profile === undefined) {
+      // 404 rather than an empty page: a token the store has dropped is a real
+      // answer, and the bar reads the status to stop asking.
+      ctx.response.status(404)
+      if (asJson) {
+        ctx.response.header('content-type', 'application/json')
+        ctx.response.send({ error: 'no profile for this token' })
+      } else {
+        ctx.response.header('content-type', 'text/html; charset=utf-8')
+        ctx.response.send(renderProfileList(profiler.store.recent(50), token))
+      }
+      return
+    }
+
+    if (asJson) {
+      ctx.response.header('content-type', 'application/json')
+      // No cache: the token is single-use in practice, and a cached profile
+      // would show the previous request's numbers after a reload.
+      ctx.response.header('cache-control', 'no-store')
+      ctx.response.send({
+        ...profile,
+        badges: badgesFor(profiler, profile.data),
+      })
+      return
+    }
+
+    ctx.response.header('content-type', 'text/html; charset=utf-8')
+    ctx.response.send(renderProfilePage(profile, badgesFor(profiler, profile.data)))
+  }
+}
+
+/** Each collector's headline, asked of the collector rather than guessed. */
+function badgesFor(
+  profiler: Profiler,
+  data: Record<string, Record<string, unknown>>,
+): Record<string, string> {
+  const badges: Record<string, string> = {}
+  for (const collector of profiler.collectors) {
+    const panel = data[collector.name]
+    if (panel === undefined || collector.badge === undefined) continue
+    try {
+      const badge = collector.badge(panel)
+      if (badge !== undefined) badges[collector.name] = badge
+    } catch {
+      // A badge that throws costs its own label, nothing more.
+    }
+  }
+  return badges
+}

package/src/profiler/index.ts

@@ -0,0 +1,56 @@
+/**
+ * The profiler, assembled.
+ *
+ * One place that knows which collectors ship, so the Ignitor wires a profiler
+ * rather than a list — and an application that wants its own panel adds a
+ * collector here instead of rebuilding the set.
+ */
+
+import {
+  databaseCollector,
+  exceptionCollector,
+  type LogBag,
+  logCollector,
+  memoryCollector,
+  type QueryBag,
+  type QueryEventSource,
+  requestCollector,
+} from './collectors.js'
+import { MemoryProfileStore } from './MemoryProfileStore.js'
+import { Profiler } from './Profiler.js'
+import type { ProfilerCollector } from './types.js'
+
+export * from './collectors.js'
+export { profilerEndpoint } from './endpoint.js'
+export { MemoryProfileStore } from './MemoryProfileStore.js'
+export { profileRequest } from './middleware.js'
+export { PROFILER_PATH, PROFILER_TOKEN_HEADER, Profiler } from './Profiler.js'
+export { toolbarScript } from './toolbar.js'
+export * from './types.js'
+
+/**
+ * The profiler a dev server runs with.
+ *
+ * `events` is the application's emitter. The database panel comes from
+ * `db:query` on it, which is how a data layer reports a statement without the
+ * profiler importing one.
+ */
+export function createProfiler(
+  events: QueryEventSource,
+  extra: readonly ProfilerCollector[] = [],
+): Profiler {
+  // Built before the profiler so the database collector can read the bag of
+  // whichever request is running — the reference is filled in a line later,
+  // and nothing reads it until a request arrives.
+  let profiler: Profiler
+  const collectors: ProfilerCollector[] = [
+    requestCollector(),
+    databaseCollector(events, () => profiler.current<QueryBag>('database')),
+    exceptionCollector(),
+    logCollector(events, () => profiler.current<LogBag>('logs')),
+    memoryCollector(),
+    ...extra,
+  ]
+  profiler = new Profiler(collectors, new MemoryProfileStore(collectors))
+  return profiler
+}

package/src/profiler/middleware.ts

@@ -0,0 +1,83 @@
+/**
+ * The middleware that records a request.
+ *
+ * It wraps everything after it, so what it measures is the whole answer — the
+ * application's own middleware included — and it stamps the token on the
+ * response first thing. That header is how a client finds the profile again;
+ * the injected bar carries the same token inline, because a script in a page
+ * cannot read the headers of the navigation that delivered it.
+ *
+ * Registered AFTER the profiler's own endpoint and the hot-reload stream, so
+ * neither ends up in its own history — and so a reload stream, which is open
+ * for as long as the tab is, never sits in the middle of an unfinished profile.
+ */
+
+import type { HttpContext } from '../http/HttpContext.js'
+import type { MiddlewareFunction } from '../middleware/Pipeline.js'
+import type { RequestBag } from './collectors.js'
+import { PROFILER_TOKEN_HEADER, type Profiler } from './Profiler.js'
+
+export function profileRequest(profiler: Profiler): MiddlewareFunction {
+  return async (ctx: HttpContext, next: () => Promise<void>) => {
+    await profiler.profile(
+      { method: ctx.request.method(), url: ctx.request.url() },
+      async (scope) => {
+        // Before the work, not after: a handler that throws unwinds past this
+        // line, and the request that broke is the one whose profile you most
+        // want to open.
+        ctx.response.header(PROFILER_TOKEN_HEADER, scope.token)
+        try {
+          await next()
+        } finally {
+          // After routing, because this runs ahead of it: the pattern and the
+          // controller are only on the context once a route has matched. In a
+          // `finally`, because a request that threw still matched one.
+          describeRoute(profiler.current<RequestBag>('request'), ctx)
+        }
+      },
+      (error) => ({ status: statusOf(ctx, error) }),
+    )
+  }
+}
+
+/**
+ * The status the response will carry.
+ *
+ * When the work threw, the exception handler has not run yet — the error is
+ * still travelling outwards — so the response still holds whatever it held
+ * before. Read the error the way the handler will instead, and fall back to
+ * 500, which is what an error carrying no status becomes.
+ */
+function statusOf(ctx: HttpContext, error: unknown): number {
+  if (error === undefined) return ctx.response.getStatus()
+  if (typeof error === 'object' && error !== null) {
+    const status = Reflect.get(error, 'status') ?? Reflect.get(error, 'statusCode')
+    if (typeof status === 'number' && Number.isInteger(status)) return status
+  }
+  return 500
+}
+
+/**
+ * What matched, as the request panel shows it.
+ *
+ * Read off the context rather than passed in, and every field optional: a 404
+ * matched nothing, and an inline handler has no controller to name.
+ */
+function describeRoute(bag: RequestBag | undefined, ctx: HttpContext): void {
+  if (bag === undefined) return
+  const route = ctx.route
+  if (route === undefined) return
+  bag.route = route.name ?? route.pattern
+  const controller = route.controller
+  if (controller !== undefined && route.action !== undefined) {
+    // `route.controller` is the PROTOTYPE, not the class — that is what the
+    // decorator-reading middleware needs — so the name is on its constructor.
+    const name = Reflect.get(Reflect.get(controller, 'constructor') ?? {}, 'name')
+    bag.controller = `${typeof name === 'string' ? name : 'Controller'}.${route.action}`
+  }
+  const user = ctx.auth?.user
+  if (typeof user === 'object' && user !== null) {
+    const id = Reflect.get(user, 'id')
+    if (typeof id === 'string' || typeof id === 'number') bag.userId = String(id)
+  }
+}