@oceanmetrics/engine 0.0.0-stage → 0.1.0

package/src/cells.ts

@@ -0,0 +1,190 @@
+// the masked grid, as map geometry: one GeoJSON square per mask cell, sized from the dataset's own
+// axis spacing (lon/lat +- half a step), carrying the cell's value and area weight.
+//
+// antimeridian: the mask cells of a place like PMNM arrive on both sides of +-180. drawn naively in
+// the (-180, 180] frame, the two halves sit at opposite edges of the map and any square straddling
+// the line wraps the whole globe. so the whole set is put in one continuous frame first: when the
+// cells span more than 180 degrees of longitude, they are expressed in [0, 360) ("centred at 180")
+// and the squares keep going past 180 (179.95 -> 180.05) instead of wrapping. MapLibre accepts
+// longitudes outside [-180, 180] and draws them across the seam.
+import type { Feature, FeatureCollection, Point, Polygon } from 'geojson'
+
+/** a mask cell with the value of one time step (null when the grid has no data there). */
+export interface ValueCell { lon: number; lat: number; weight: number; value?: number | null }
+/** what each square carries into the map (hover shows `value` and `weight`). */
+export interface CellProps { lon: number; lat: number; weight: number; value: number | null }
+
+export type LonCenter = 0 | 180
+export interface CellSquaresOptions {
+  /** cell width in degrees; inferred from the distinct cell longitudes when omitted. */
+  lonSpacing?: number
+  /** cell height in degrees; inferred from the distinct cell latitudes when omitted. */
+  latSpacing?: number
+  /** longitude frame: (-180, 180] for 0, [0, 360) for 180, or 'auto' by the cells' own span. */
+  center?: LonCenter | 'auto'
+}
+export interface CellSquares {
+  geojson   : FeatureCollection<Polygon, CellProps>
+  lonSpacing: number
+  latSpacing: number
+  center    : LonCenter
+  /** [west, south, east, north] of the squares, in the chosen frame (east may exceed 180). */
+  bbox      : [number, number, number, number]
+  /** value range over the cells that have one, for the colour ramp ([0, 0] when there are none). */
+  range     : [number, number]
+}
+
+/** a longitude in the frame `center` names: (-180, 180] for 0, [0, 360) for 180. */
+export function normalizeLon(lon: number, center: LonCenter = 0): number {
+  let x = ((((lon + 180) % 360) + 360) % 360) - 180   // (-180, 180]
+  if (x === -180) x = 180
+  return center === 180 && x < 0 ? x + 360 : x
+}
+
+/**
+ * The step of an axis given only the values that were used: the **median** positive gap between
+ * distinct sorted values, so gaps left by cells outside the mask (2x, 3x the step) cannot inflate
+ * it. Returns 0 when there is nothing to measure (fewer than two distinct values).
+ */
+export function axisSpacing(values: number[]): number {
+  const v = [...new Set(values.filter((x) => Number.isFinite(x)))].sort((a, b) => a - b)
+  if (v.length < 2) return 0
+  const gaps: number[] = []
+  for (let i = 1; i < v.length; i++) { const d = v[i] - v[i - 1]; if (d > 0) gaps.push(d) }
+  if (!gaps.length) return 0
+  gaps.sort((a, b) => a - b)
+  const m = gaps.length >> 1
+  return gaps.length % 2 ? gaps[m] : (gaps[m - 1] + gaps[m]) / 2
+}
+
+/** the frame the cells fit in without wrapping: 180 when they span more than half the globe. */
+export function pickCenter(lons: number[]): LonCenter {
+  if (!lons.length) return 0
+  const x = lons.map((l) => normalizeLon(l, 0))
+  return Math.max(...x) - Math.min(...x) > 180 ? 180 : 0
+}
+
+/**
+ * One square per mask cell: `lon +- lonSpacing/2`, `lat +- latSpacing/2` (latitude clamped to the
+ * poles), in a single continuous longitude frame, with `value`, `weight`, `lon` and `lat` as
+ * properties. Cells whose value is missing are kept (drawn as a gap) with `value: null`.
+ */
+export function cellSquares(cells: ValueCell[], opts: CellSquaresOptions = {}): CellSquares {
+  const center = opts.center === undefined || opts.center === 'auto'
+    ? pickCenter(cells.map((c) => c.lon))
+    : opts.center
+  const lons = cells.map((c) => normalizeLon(c.lon, center))
+  let dx     = opts.lonSpacing ?? axisSpacing(lons)
+  let dy     = opts.latSpacing ?? axisSpacing(cells.map((c) => c.lat))
+  // a single row or column gives that axis no gap to measure: borrow the other axis's step
+  if (!dx) dx = dy
+  if (!dy) dy = dx
+  const hx   = dx / 2, hy = dy / 2
+
+  let w = Infinity, s = Infinity, e = -Infinity, n = -Infinity
+  let vmin = Infinity, vmax = -Infinity
+  const features: Feature<Polygon, CellProps>[] = cells.map((c, i) => {
+    const x = lons[i], y = c.lat
+    const x0 = x - hx, x1 = x + hx
+    const y0 = Math.max(-90, y - hy), y1 = Math.min(90, y + hy)
+    const value = typeof c.value === 'number' && Number.isFinite(c.value) ? c.value : null
+    if (value !== null) { if (value < vmin) vmin = value; if (value > vmax) vmax = value }
+    if (x0 < w) w = x0; if (x1 > e) e = x1
+    if (y0 < s) s = y0; if (y1 > n) n = y1
+    return {
+      type      : 'Feature',
+      id        : i,
+      properties: { lon: x, lat: y, weight: c.weight, value },
+      geometry  : { type: 'Polygon', coordinates: [[[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]] },
+    }
+  })
+
+  return {
+    geojson   : { type: 'FeatureCollection', features },
+    lonSpacing: dx, latSpacing: dy, center,
+    bbox      : features.length ? [w, s, e, n] : [0, 0, 0, 0],
+    range     : Number.isFinite(vmin) ? [vmin, vmax] : [0, 0],
+  }
+}
+
+// ── map bounds for a place ────────────────────────────────────────────────────
+/**
+ * Map bounds for a place, antimeridian-aware.
+ *
+ * The gazetteer splits geometry at +-180, so a place that straddles the line (PMNM) has the useless
+ * stored bbox `[-180, .., 180, ..]`. Only in that case — width > 180 — is the geometry walked, in
+ * the [0, 360) frame, giving bounds like `[177.8, 19.2, 210.0, 31.8]` that MapLibre fits across the
+ * seam. Every other place returns its stored bbox untouched, so selecting a place still does no
+ * geometry work (the FKNMS/TBNMS vertex counts are the reason that matters).
+ */
+export function placeMapBounds(
+  place: { bbox: [number, number, number, number]; geometry?: unknown },
+): [number, number, number, number] {
+  const [x0, y0, x1, y1] = place.bbox
+  if (x1 - x0 <= 180 || !place.geometry) return [x0, y0, x1, y1]
+  let w = Infinity, s = Infinity, e = -Infinity, n = -Infinity
+  const walk = (c: any) => {
+    if (typeof c?.[0] === 'number') {
+      const x = normalizeLon(c[0], 180), y = c[1]
+      if (x < w) w = x; if (x > e) e = x
+      if (y < s) s = y; if (y > n) n = y
+    } else if (Array.isArray(c)) for (const k of c) walk(k)
+  }
+  const g: any = place.geometry
+  if (g.type === 'GeometryCollection') for (const sub of g.geometries) walk(sub.coordinates)
+  else walk(g.coordinates)
+  return Number.isFinite(w) ? [w, s, e, n] : [x0, y0, x1, y1]
+}
+
+/**
+ * The box to fit for a place of the gazetteer index. The index unwraps a place cut at the antimeridian
+ * (NMS:PMNM is 177.8..199.0, `centroid_lon` 188), so its bbox fits directly. A few rows still read -180..180
+ * (the geometry is split but the bbox is not unwrapped): those need the geometry, and give null until it is
+ * loaded (the caller fits when it arrives).
+ */
+export function indexBounds(
+  bbox: [number, number, number, number],
+  geometry?: unknown,
+): [number, number, number, number] | null {
+  if (bbox[2] - bbox[0] <= 180) return bbox
+  return geometry ? placeMapBounds({ bbox, geometry }) : null
+}
+
+// ── points (tabledap) ─────────────────────────────────────────────────────────
+export interface CellPoints {
+  geojson: FeatureCollection<Point, CellProps>
+  center : LonCenter
+  bbox   : [number, number, number, number]
+  range  : [number, number]
+}
+
+/**
+ * One point per sample position, for a **tabledap** run: there is no cell to draw, so the map gets
+ * circles instead of squares. Same longitude framing as `cellSquares()`, so an antimeridian place
+ * stays in one piece, and the same properties, so the hover popup and the colour ramp are shared.
+ */
+export function cellPoints(cells: ValueCell[], opts: { center?: LonCenter | 'auto' } = {}): CellPoints {
+  const center = opts.center === undefined || opts.center === 'auto'
+    ? pickCenter(cells.map((c) => c.lon))
+    : opts.center
+  let w = Infinity, s = Infinity, e = -Infinity, n = -Infinity
+  let vmin = Infinity, vmax = -Infinity
+  const features: Feature<Point, CellProps>[] = cells.map((c, i) => {
+    const x = normalizeLon(c.lon, center), y = c.lat
+    const value = typeof c.value === 'number' && Number.isFinite(c.value) ? c.value : null
+    if (value !== null) { if (value < vmin) vmin = value; if (value > vmax) vmax = value }
+    if (x < w) w = x; if (x > e) e = x
+    if (y < s) s = y; if (y > n) n = y
+    return {
+      type: 'Feature', id: i,
+      properties: { lon: x, lat: y, weight: c.weight, value },
+      geometry  : { type: 'Point', coordinates: [x, y] },
+    }
+  })
+  return {
+    geojson: { type: 'FeatureCollection', features },
+    center,
+    bbox   : features.length ? [w, s, e, n] : [0, 0, 0, 0],
+    range  : Number.isFinite(vmin) ? [vmin, vmax] : [0, 0],
+  }
+}

package/src/chunks.ts

@@ -0,0 +1,58 @@
+// the time chunking of a long window: a window is cut into requests of at most 90 time steps and 100,000 values, so a
+// server is asked many light questions instead of a few heavy ones. moved from erddap-places `precompute/src/stats.ts`
+// (`window`, `chunks`, `chunkDaysFor`) with the two constants of `precompute/src/targets.ts`; the arithmetic is
+// unchanged, and the two limits are now options (defaulting to the precompute's values) so the browser preview runs
+// the same rule.
+import { addDays, day, daysBetween, type TimeExtent } from './extent.js'
+
+/** never ask ERDDAP for more than this many time steps in one request (90 days of a daily grid). */
+export const MAX_CHUNK_DAYS = 90
+/**
+ * and shrink the chunk further so a single request (one lobe) stays under this many values: the
+ * `n_max_vals_per_req = 100000` of marinebon/extractr `ed_extract()`, which keeps an ERDDAP server
+ * responsive with many light requests instead of a few heavy ones.
+ */
+export const MAX_REQUEST_VALUES = 100_000
+
+export interface ChunkLimits {
+  /** values per request (one lobe): time steps x axis cells */
+  maxValues?: number
+  /** time steps per request */
+  maxDays?: number
+}
+
+/** the last `days` days of the dataset, or its full extent when that is shorter (or `days` is 'all'). */
+export function window(ext: TimeExtent, days: number | 'all' = 'all'): { start: string; end: string } {
+  const end = day(ext.end)
+  if (days === 'all' || !Number.isFinite(days)) return { start: day(ext.start), end }
+  let start = addDays(end, -(days - 1))
+  if (daysBetween(day(ext.start), start) < 0) start = day(ext.start)
+  return { start, end }
+}
+
+/** split [start, end] into consecutive, non-overlapping slices of at most `chunkDays` days. */
+export function chunks(start: string, end: string, chunkDays: number): Array<[string, string]> {
+  const out: Array<[string, string]> = []
+  let s = start
+  while (daysBetween(s, end) >= 0) {
+    const e = addDays(s, chunkDays - 1)
+    out.push([s, daysBetween(e, end) < 0 ? end : e])
+    s = addDays(e, 1)
+  }
+  return out
+}
+
+/**
+ * days per request: `maxDays` time steps (90 days of a daily grid; a coarser grid's steps are
+ * counted in its own days, so a monthly product asks for up to 90 months at once), shrunk so one
+ * request stays under `maxValues` values (extractr's n_max_vals_per_req). `cellsPerStep` is the
+ * largest lobe's axis cells: each lobe is its own request. Never below one step.
+ */
+export function chunkDaysFor(cellsPerStep: number, stepDays = 1,
+                             { maxValues = MAX_REQUEST_VALUES, maxDays = MAX_CHUNK_DAYS }: ChunkLimits = {}): number {
+  const sd  = Math.max(1, stepDays)
+  const cap = Math.floor(maxDays * sd)
+  if (!(cellsPerStep > 0)) return cap
+  const steps = Math.max(1, Math.floor(maxValues / cellsPerStep))
+  return Math.max(1, Math.min(cap, Math.floor(steps * sd)))
+}

package/src/dataset.ts

@@ -0,0 +1,160 @@
+// the STAC side of a dataset: one erddap/* Collection -> the fields a run needs (the griddap URL, the value expression,
+// the template). moved from erddap-places `app/src/lib/catalog.ts`: the pure functions, unchanged; `loadDatasets` (it
+// reads the gazetteer's catalog.json) and `linkMismatch` (the app's wording) stay in the app.
+import type { Format } from './erddap.js'
+import { pickFormat } from './erddap.js'
+export interface CubeVariable {
+  name       : string
+  unit       : string | null
+  description: string
+  categorical: boolean
+  /** class value -> label, for categorical variables (from `erddap-places:classes`). */
+  classes   ?: Record<string, string>
+}
+
+/** the long-format columns of a tabledap table: `measurement_type` / `measurement_value`. */
+export interface LongFormat { typeColumn: string; valueColumn: string }
+
+export interface Dataset {
+  id           : string           // erddap/dhw_5km
+  title        : string
+  baseUrl      : string
+  datasetId    : string
+  protocol     : 'griddap' | 'tabledap'
+  /** set when the table keeps its variables in a column (tabledap, long format) */
+  longFormat  ?: LongFormat
+  version     ?: string
+  cors         : boolean
+  formats      : string[]
+  lonRange     : [number, number]
+  latDescending: boolean
+  variables    : CubeVariable[]
+  timeStep    ?: string           // e.g. P1D, P8D
+  timeExtent  ?: [string | null, string | null]
+  /** `erddap-places:depth`: the depth level (m) to slice on a grid with a depth axis (the surface) */
+  depth       ?: number
+  /** `erddap-places:status`: 'pending' when the server does not serve it yet (list it, greyed out) */
+  status      ?: string
+  format       : Format           // best format this server can serve us
+  collection   : any              // the raw STAC Collection
+}
+
+const KELVIN = /^(k|kelvin|degree_?k(elvin)?)$/i
+
+/**
+ * The SQL expression for a variable's value, converting Kelvin to °C.
+ *
+ * For a long-format table the variable is not a column: the rows were already filtered to
+ * `measurement_type = '<name>'` by the request, so the expression is the value column.
+ */
+export function valueExpr(v: CubeVariable, alias = 's', long?: LongFormat): string {
+  const col = long ? `${alias}."${long.valueColumn}"` : `${alias}."${v.name}"`
+  return KELVIN.test(v.unit ?? '') ? `(${col} - 273.15)` : col
+}
+/** the axis label for a variable, after any unit conversion. */
+export function valueLabel(v: CubeVariable): string {
+  if (KELVIN.test(v.unit ?? '')) return '°C'
+  const u = (v.unit ?? '').trim()
+  if (!u || u === '1' || u.toLowerCase() === 'none') return v.name
+  if (/^(celsius|degree_?c(elsius)?)$/i.test(u)) return '°C'
+  return u
+}
+
+/** the `erddap-places:categorical` flag as published in the catalog (boolean, or the string). */
+export function isCategorical(v: any): boolean {
+  const f = v?.['erddap-places:categorical']
+  return f === true || f === 'true'
+}
+
+/** `erddap-places:long_format` -> the two column names, or undefined. */
+export function toLongFormat(f: any): LongFormat | undefined {
+  const t = f?.type_column, v = f?.value_column
+  return t && v ? { typeColumn: String(t), valueColumn: String(v) } : undefined
+}
+
+/**
+ * The SQL template a run is summarised with: point (tabledap) data rolls up by month, a flagged
+ * categorical grid gets per-class fractions, everything else is the daily grid statistics.
+ */
+export function statsTemplate(
+  v: Pick<CubeVariable, 'categorical'> | null | undefined,
+  protocol: 'griddap' | 'tabledap' = 'griddap',
+): 'stats_daily' | 'stats_categorical' | 'stats_tabledap' {
+  if (protocol === 'tabledap') return 'stats_tabledap'
+  return v?.categorical === true ? 'stats_categorical' : 'stats_daily'
+}
+
+function toVariable(name: string, v: any): CubeVariable {
+  return {
+    name,
+    unit       : v?.unit ?? null,
+    description: v?.description ?? name,
+    categorical: isCategorical(v),
+    classes    : v?.['erddap-places:classes'] ?? undefined,
+  }
+}
+
+/** one STAC Collection -> the fields the app uses. */
+export function toDataset(c: any): Dataset {
+  const vars = Object.entries(c['cube:variables'] ?? {})
+    .filter(([, v]: [string, any]) => (v?.type ?? 'data') === 'data')
+    .map(([k, v]) => toVariable(k, v))
+  const time = c['cube:dimensions']?.time ?? {}
+  return {
+    id           : c.id,
+    title        : c.title ?? c.id,
+    baseUrl      : c['erddap:base_url'],
+    datasetId    : c['erddap:dataset_id'],
+    protocol     : c['erddap:protocol'] === 'tabledap' ? 'tabledap' : 'griddap',
+    longFormat   : toLongFormat(c['erddap-places:long_format']),
+    version      : c['erddap:version'],
+    cors         : c['erddap:cors'] !== false,
+    formats      : c['erddap:formats'] ?? [],
+    lonRange     : c['erddap:lon_range'] ?? [-180, 180],
+    latDescending: c['erddap:lat_descending'] === true,
+    variables    : vars,
+    timeStep     : time.step,
+    timeExtent   : time.extent,
+    depth        : Number.isFinite(Number(c['erddap-places:depth'])) && c['erddap-places:depth'] != null
+                     ? Number(c['erddap-places:depth']) : undefined,
+    status       : c['erddap-places:status'] ?? undefined,
+    format       : pickFormat({ erddap: { cors: c['erddap:cors'], formats: c['erddap:formats'] } }),
+    collection   : c,
+  }
+}
+
+/**
+ * The dataset a link names. The app writes the STAC collection id (`erddap/dhw_5km`), but links written elsewhere name
+ * ERDDAP's own dataset id (`dhw_5km`, `IMERG_monthly_global_precip`), and one collection id differs from its dataset id
+ * in case (`erddap/imerg_monthly_global_precip`). Tried in turn, each ignoring case: the collection id, the collection
+ * id with `erddap/` before it, ERDDAP's dataset id. A pending dataset is never picked; null when nothing matches.
+ */
+export function resolveDataset<D extends Pick<Dataset, 'id' | 'datasetId' | 'status'>>(datasets: D[], wanted: string): D | null {
+  const w = String(wanted ?? '').trim().toLowerCase()
+  if (!w) return null
+  const live = datasets.filter((d) => d.status !== 'pending')
+  const eq = (a: string | undefined, b: string) => (a ?? '').toLowerCase() === b
+  return live.find((d) => d.id === wanted)
+    ?? live.find((d) => eq(d.id, w))
+    ?? live.find((d) => eq(d.id, `erddap/${w}`))
+    ?? live.find((d) => eq(d.datasetId, w.replace(/^erddap\//, '')))
+    ?? null
+}
+
+/** ERDDAP wants a lon in the dataset's own range: [-180, 180] or [0, 360]. */
+export function toDatasetLon(lon: number, range: [number, number]): number {
+  if (range[1] > 180 && lon < 0)  return lon + 360
+  if (range[0] < 0   && lon > 180) return lon - 360
+  return lon
+}
+
+/**
+ * A lobe's longitude span in the dataset's own frame, [min, max], clamped to `erddap:lon_range`:
+ * a lobe that runs to 180 on a grid ending at 179.75 (CMEMS 0.25°) would otherwise ask for a
+ * longitude past the axis maximum, which ERDDAP refuses (404).
+ */
+export function lobeLonSpan(bbox: [number, number, number, number] | number[], range: [number, number]): [number, number] {
+  const lo = toDatasetLon(bbox[0], range), hi = toDatasetLon(bbox[2], range)
+  const clamp = (x: number) => Math.min(Math.max(x, range[0]), range[1])
+  return [clamp(Math.min(lo, hi)), clamp(Math.max(lo, hi))]
+}

package/src/engine.ts

@@ -0,0 +1,190 @@
+// DuckDB-WASM as the statistics engine: the slab goes in as a registered Parquet buffer, the mask as an Arrow table,
+// and every statistic is a sql/ template rendered with {{named}} params. moved from erddap-places
+// `app/src/lib/engine.ts`; the methods are unchanged, but the package never boots DuckDB itself: the app owns the
+// bundles and the Web Worker (Vite `?url` imports, a CDN, or the node-blocking bundle in tests) and hands the Engine a
+// `connect()` that resolves to `{ db, conn }`. `connectBrowser(duckdb, bundles)` is that boot for a browser app, with the
+// duckdb-wasm module passed in (a peer dependency, never imported at run time here).
+import { DataType, tableFromArrays, tableToIPC } from 'apache-arrow'
+import type * as duckdbWasm from '@duckdb/duckdb-wasm'
+import type { MaskCell } from './mask.js'
+import { render } from './sql.js'
+import type { Params } from './sql.js'
+
+// ---- the database it runs on ----
+
+/** the slice of a DuckDB-WASM database the engine uses: `AsyncDuckDB` in a browser, the node-blocking bindings in tests */
+export interface DuckDbLike {
+  registerFileBuffer(name: string, buf: Uint8Array): unknown
+  copyFileToBuffer(name: string): Uint8Array | Promise<Uint8Array>
+  dropFile(name: string): unknown
+}
+/** the slice of a connection the engine uses: `AsyncDuckDBConnection`, or the blocking `DuckDBConnection` */
+export interface DuckConnLike {
+  /** an Arrow table (or a promise of one): iterable rows, `schema.fields`, `numRows` */
+  query(sql: string): any
+  insertArrowFromIPCStream(buf: Uint8Array, opts: { name: string; create?: boolean }): unknown
+}
+export interface Duck { db: DuckDbLike; conn: DuckConnLike }
+
+/**
+ * The browser boot erddap-places' engine did itself: pick the bundle (eh or mvp), start its Web Worker, instantiate,
+ * connect. `duckdb` is the app's `import * as duckdb from '@duckdb/duckdb-wasm'`, `bundles` the module and worker
+ * URLs it serves (see the README, "Worker setup").
+ */
+export async function connectBrowser(duckdb: typeof duckdbWasm, bundles: duckdbWasm.DuckDBBundles,
+                                     logger: duckdbWasm.Logger = new duckdb.VoidLogger()): Promise<Duck> {
+  const bundle = await duckdb.selectBundle(bundles)
+  const worker = new Worker(bundle.mainWorker!)
+  const db = new duckdb.AsyncDuckDB(logger, worker)
+  await db.instantiate(bundle.mainModule, bundle.pthreadWorker)
+  return { db, conn: await db.connect() }
+}
+
+// ---- rows ----
+export type Row = Record<string, any>
+/**
+ * an Arrow result as plain rows: BIGINT as a number, and DECIMAL scaled. DuckDB-WASM hands a DECIMAL back as its
+ * unscaled integer (`1.0 AS weight` in points_tabledap.sql arrived as 10, a `VALUES (20.05, ...)` latitude as 2005);
+ * erddap-places' toRows passed it through, which no ERDDAP slab (float axes, float values) ever exercised.
+ */
+export function toRows(t: any): Row[] {
+  const cols: string[] = t.schema.fields.map((f: any) => f.name)
+  const scale: number[] = t.schema.fields.map((f: any) => (DataType.isDecimal(f.type) ? 10 ** f.type.scale : 0))
+  const out: Row[] = []
+  for (const r of t) {
+    const o: Row = {}
+    cols.forEach((c, i) => {
+      const v = r[c]
+      o[c] = typeof v === 'bigint' ? Number(v) : scale[i] && v !== null && v !== undefined ? Number(v) / scale[i] : v
+    })
+    out.push(o)
+  }
+  return out
+}
+
+export interface EngineEvent { name: string; ms: number; note?: string }
+
+const now = () => (globalThis.performance ?? Date).now()
+
+export class Engine {
+  db!: DuckDbLike
+  conn!: DuckConnLike
+  ready: Promise<void>
+  marks: EngineEvent[] = []
+  lastSql = ''
+  private q: Promise<any> = Promise.resolve() // one connection: serialize everything
+
+  /** `connect` runs at once (construct the engine when the app first needs it) */
+  constructor(connect: () => Promise<Duck> | Duck) { this.ready = this.init(connect) }
+  private mark(name: string, ms: number, note?: string) { this.marks.push({ name, ms: Math.round(ms), note }) }
+
+  private async init(connect: () => Promise<Duck> | Duck) {
+    const t = now()
+    const { db, conn } = await connect()
+    this.db = db
+    this.conn = conn
+    this.mark('connect', now() - t)
+  }
+
+  /** register an already-fetched slab (parquet bytes) under a virtual file name. */
+  async registerBuffer(name: string, buf: Uint8Array) {
+    const run = async () => {
+      await this.ready
+      const t = now()
+      await this.db.registerFileBuffer(name, buf)
+      this.mark(`register:${name}`, now() - t, `${(buf.byteLength / 1e3).toFixed(0)} kB`)
+    }
+    this.q = this.q.then(run, run)
+    return this.q
+  }
+
+  /** forget a registered file (a slab whose statistics are done). */
+  async dropFile(name: string) {
+    const run = async () => {
+      await this.ready
+      await this.db.dropFile(name)
+    }
+    this.q = this.q.then(run, run)
+    return this.q
+  }
+
+  /** the gridMask cells as a DuckDB table (via Arrow IPC, so no giant VALUES list). */
+  async insertMask(cells: MaskCell[], table = 'mask') {
+    const run = async () => {
+      await this.ready
+      const t = now()
+      const arrow = tableFromArrays({
+        latitude : Float64Array.from(cells.map((c) => c.lat)),
+        longitude: Float64Array.from(cells.map((c) => c.lon)),
+        weight   : Float64Array.from(cells.map((c) => c.weight)),
+      })
+      await this.conn.query(`DROP TABLE IF EXISTS ${table}`)
+      await this.conn.insertArrowFromIPCStream(tableToIPC(arrow, 'stream'), { name: table, create: true })
+      this.mark(`mask:${table}`, now() - t, `${cells.length} cells`)
+    }
+    this.q = this.q.then(run, run)
+    return this.q
+  }
+
+  /** rows from a non-Parquet slab (.csvp / .jsonp) as a table, so the same SQL works on them. */
+  async insertRows(rows: Record<string, unknown>[], table: string) {
+    const run = async () => {
+      await this.ready
+      const t = now()
+      const cols = Object.keys(rows[0] ?? {})
+      const arrays: Record<string, any> = {}
+      for (const c of cols) {
+        const vals = rows.map((r) => r[c])
+        arrays[c] = vals.every((v) => typeof v === 'number' || v === null)
+          ? Float64Array.from(vals.map((v) => (v === null ? NaN : (v as number))))
+          : vals.map((v) => (v === null || v === undefined ? null : String(v)))
+      }
+      await this.conn.query(`DROP TABLE IF EXISTS ${table}`)
+      await this.conn.insertArrowFromIPCStream(tableToIPC(tableFromArrays(arrays), 'stream'), { name: table, create: true })
+      this.mark(`rows:${table}`, now() - t, `${rows.length} rows`)
+    }
+    this.q = this.q.then(run, run)
+    return this.q
+  }
+
+  exec(sql: string, label = 'exec'): Promise<Row[]> {
+    const run = async () => {
+      await this.ready
+      const t = now()
+      this.lastSql = sql
+      const res = await this.conn.query(sql)
+      this.mark(`query:${label}`, now() - t, `${res.numRows} rows`)
+      return toRows(res)
+    }
+    this.q = this.q.then(run, run)
+    return this.q
+  }
+
+  /**
+   * The result table as Parquet bytes, written by DuckDB itself.
+   *
+   * The rows go back into a temporary table (Arrow IPC, as the mask does), `COPY … TO` writes a
+   * Parquet file into the WASM filesystem and `copyFileToBuffer` hands it back. Going through the
+   * rows rather than re-running the statistics keeps the download identical to what is on screen,
+   * whatever has happened to the registered slab since.
+   */
+  async toParquet(rows: Record<string, unknown>[], table = '_export', file = 'export.parquet'): Promise<Uint8Array> {
+    if (!rows.length) throw new Error('nothing to export')
+    await this.insertRows(rows, table)
+    const run = async () => {
+      await this.ready
+      const t = now()
+      await this.conn.query(`COPY (SELECT * FROM ${table}) TO '${file}' (FORMAT PARQUET)`)
+      const buf = await this.db.copyFileToBuffer(file)
+      await Promise.resolve(this.db.dropFile(file)).catch(() => {})
+      await this.conn.query(`DROP TABLE IF EXISTS ${table}`)
+      this.mark(`parquet:${file}`, now() - t, `${rows.length} rows, ${(buf.byteLength / 1e3).toFixed(0)} kB`)
+      return buf
+    }
+    this.q = this.q.then(run, run)
+    return this.q
+  }
+
+  /** render a `sql/<name>.sql` template with {{params}} and run it. */
+  runTemplate(name: string, params: Params): Promise<Row[]> { return this.exec(render(name, params), name) }
+}