@bakery-framework/plugin-db-explorer 2.0.0-alpha.9 → 2.0.0-rc.1

package/src/endpoints/import.ts

@@ -1,5 +1,5 @@
 /**
- * CSV import — the rows, not the file.
+ * CSV import: the rows, not the file.
  *
  * The parse happens in the browser, in `shared/csv.ts`, and stays there. This
  * endpoint takes rows that are already records, so the mapping the user
@@ -8,7 +8,7 @@
  *
  * Two things separate it from `POST /api/_db/rows`, and both are about scale:
  * the bound is a spreadsheet's worth of rows rather than an edit's, and a bad
- * row does not have to end the whole import — `onBadRow: 'skip'` reports it and
+ * row does not have to end the whole import: `onBadRow: 'skip'` reports it and
  * carries on, which is what a 50,000-row file with three malformed lines needs.
  */
 
@@ -19,7 +19,7 @@ import { DB } from '@bakery-framework/orm/orm'
 import { overLimit } from '../policy'
 import { isRollbackSignal, previewRollback } from '../preview'
 import { type FieldError, validateInsertRow } from '../validate'
-import { beginWrite, invalid } from './common'
+import { beginWrite, invalid, refuse } from './common'
 
 export type OnBadRow = 'stop' | 'skip'
 
@@ -56,7 +56,7 @@ export async function handleImport(
     records.push(validated.values)
   })
 
-  // `stop` refuses the whole file before a statement runs — the same "413 with
+  // `stop` refuses the whole file before a statement runs: the same "413 with
   // nothing executed" promise, for a 400.
   if (onBadRow === 'stop' && errors.length) return invalid(errors)
 
@@ -83,7 +83,7 @@ export async function handleImport(
       if (isRollbackSignal(error)) {
         return response.json.success(error.message, error.report, error.status)
       }
-      return response.json.error(400, error?.message ?? 'Import failed')
+      return refuse('import', error)
     },
   )
 }

package/src/endpoints/read.ts

@@ -3,7 +3,7 @@
  *
  * `/api/_db/schema` now answers with more than the schema: the caller's own
  * access level, and per table whether it is writable and why not. The client
- * needs its posture *before* it renders — a grid that draws edit affordances
+ * needs its posture *before* it renders: a grid that draws edit affordances
  * and then discovers on save that the table has no primary key has already
  * wasted the user's work.
  */
@@ -15,10 +15,11 @@ import { connection } from '@bakery-framework/orm/connection'
 import { currentAccess, currentCanWrite } from '../access'
 import { type Identity, introspect } from '../identity'
 import { parseFilters } from '../shared/filters'
+import { refuse } from './common'
 
 export interface SchemaColumn {
   name: string
-  /** The database's own type string, unchanged — what the grid shows. */
+  /** The database's own type string, unchanged: what the grid shows. */
   type: string
   notnull: boolean
   pk: boolean
@@ -46,8 +47,8 @@ export interface SchemaTable {
   /**
    * Declared indexes.
    *
-   * `introspect()` has always computed these — it walks them to find a usable
-   * unique key when there is no primary key — and used to throw them away here.
+   * `introspect()` has always computed these (it walks them to find a usable
+   * unique key when there is no primary key), and used to throw them away here.
    * The Structure view is the first thing that shows them, and there is no
    * other endpoint that knows them.
    */
@@ -67,7 +68,7 @@ export interface SchemaReport {
 export async function handleSchema(): Promise<JsonResponseData<unknown>> {
   return await Try.return(
     async () => {
-      const tables = await introspect()
+      const tables = await introspect({ rowCounts: true })
       const access = currentAccess()
       const canWrite = currentCanWrite()
 
@@ -82,7 +83,9 @@ export async function handleSchema(): Promise<JsonResponseData<unknown>> {
             : table.identity.reason
           return {
             name: table.name,
-            rowCount: table.rowCount,
+            // The listing asked for counts, so null cannot occur here; the
+            // fallback keeps the payload type honest rather than asserting.
+            rowCount: table.rowCount ?? 0,
             columns: table.columns.map(column => ({
               name: column.name,
               type: column.sqlType,
@@ -110,6 +113,32 @@ export async function handleSchema(): Promise<JsonResponseData<unknown>> {
   )
 }
 
+/**
+ * A total the client counted on an earlier page, or `undefined`.
+ *
+ * The `COUNT(*)` is 97% of what a page costs - 51.3 ms against 1.6 ms for the
+ * rows on a filtered page of a 200,000-row table - and page 2 of a listing is
+ * asking the same question page 1 already answered.
+ *
+ * **Only from page 2 onwards.** The first page of any listing, and any request
+ * that arrives without a page, counts for real. That is what makes the value
+ * self-correcting: a client whose total has gone stale sees it fixed as soon
+ * as it returns to the first page or changes its filters, because changing
+ * filters restarts at page 1.
+ *
+ * The client asserts it counted with *these* filters; nothing here can check
+ * that, and nothing needs to. The total decides the row readout and the page
+ * count, never which rows are returned, so a wrong one is a stale number on
+ * screen rather than wrong data.
+ */
+function readKnownTotal(url: URL, page: number): number | undefined {
+  if (page <= 1) return undefined
+  const raw = url.searchParams.get('knownTotal')
+  if (raw === null) return undefined
+  const parsed = Number(raw)
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined
+}
+
 /** Table names the way the ORM writes them: identifier characters only. */
 const RX_TABLE_NAME = /^[a-zA-Z0-9_]+$/
 
@@ -121,7 +150,7 @@ const RX_TABLE_NAME = /^[a-zA-Z0-9_]+$/
  * filter set: `JSON.parse` throwing on a mangled parameter, and `parseFilters`
  * rejecting an operator the ORM would otherwise drop. A dropped filter *widens*
  * the result, and the explorer's Delete acts on a selection made from this
- * view — see the header of `shared/filters.ts`.
+ * view. See the header of `shared/filters.ts`.
  */
 function readFilters(
   url: URL,
@@ -159,15 +188,46 @@ export async function handleTableData(
 
   return await Try.return(
     async () => {
+      const page = readBounded(url, 'page', 1, 1, Number.MAX_SAFE_INTEGER)
       const data = await connection.getData(tableName, {
-        page: Number.parseInt(url.searchParams.get('page') || '1', 10),
-        pageSize: Number.parseInt(url.searchParams.get('pageSize') || '50', 10),
+        page,
+        pageSize: readBounded(url, 'pageSize', 50, 1, MAX_PAGE_SIZE),
         sortBy: url.searchParams.get('sortBy'),
         sortOrder: url.searchParams.get('sortOrder') || 'ASC',
         filters: filters.filters,
+        knownTotal: readKnownTotal(url, page),
       })
       return response.json.success('success', data)
     },
-    (error: any) => response.json.error(400, error.message),
+    error => refuse('table-data', error),
   )
 }
+
+/** The largest page the read endpoint will assemble. */
+const MAX_PAGE_SIZE = 500
+
+/**
+ * A positive integer from the query string, clamped.
+ *
+ * The read side was the only unbounded surface left: every write goes through
+ * `policy.ts`, and this took `page` and `pageSize` as whatever
+ * `Number.parseInt` returned. `pageSize=1000000` assembled 50,000 rows and
+ * 5.25 MB in one response, `pageSize=-1` returned the whole table with a
+ * negative `totalPages`, and `page=0` silently served page 1. None needs a
+ * credential beyond `read`.
+ *
+ * `NaN` falls back rather than clamping: `page=abc` is a malformed request,
+ * and answering it with page 1 is friendlier than a 400 for a value that
+ * changes nothing about what the caller may see.
+ */
+function readBounded(
+  url: URL,
+  name: string,
+  fallback: number,
+  min: number,
+  max: number,
+): number {
+  const raw = Number.parseInt(url.searchParams.get(name) || '', 10)
+  if (!Number.isFinite(raw)) return fallback
+  return Math.min(max, Math.max(min, raw))
+}

package/src/endpoints/rows.ts

@@ -4,11 +4,11 @@
  * **Every statement carries an explicit identity predicate**, built from the
  * key the caller sent and checked against the table's declared identity first.
  * The adapter's `update(table, rowid, row)` / `remove(table, rowid)` triple is
- * never used — see the header of `identity.ts` for the three ways it is wrong.
+ * never used. See the header of `identity.ts` for the three ways it is wrong.
  *
- * **Optimistic concurrency** is the same predicate with `expect` appended:
+ * **Optimiztic concurrency** is the same predicate with `expect` appended:
  * identity ∧ expect. `changes === 0` therefore means one of two things, and the
- * dialects will not tell them apart — *the row moved on* or *the update was a
+ * dialects will not tell them apart: *the row moved on* or *the update was a
  * no-op*. MySQL reports 0 changed rows when an UPDATE sets a column to the
  * value it already held, so a zero has to be probed rather than trusted, and
  * the probe runs inside the same transaction as the UPDATE or it is answering
@@ -32,7 +32,7 @@ import {
   validateKey,
   validatePartial,
 } from '../validate'
-import { beginWrite, type Envelope, invalid } from './common'
+import { beginWrite, type Envelope, invalid, refuse } from './common'
 
 /**
  * A conflict, as the caller sees it: which edit, which row, and what the row
@@ -46,7 +46,7 @@ export interface Conflict {
 }
 
 /**
- * One row by its identity, read through the active connection — which inside
+ * One row by its identity, read through the active connection, which inside
  * `DB.transaction` is the transaction's own handle, not the pooled one.
  *
  * Built with `qId` and bound parameters (convention 8). `IS NULL` rather than
@@ -80,7 +80,7 @@ async function selectRow(
  * `where(a).and(b).and(c)…` over a predicate of any width.
  *
  * `any` for the value, because the builder's `WhereValue` is a union that also
- * carries column references and subqueries — narrowing to it here would mean
+ * carries column references and subqueries: narrowing to it here would mean
  * asserting a coerced database value is not one of those, which is true but
  * unprovable at this end. The values themselves are already bound parameters.
  */
@@ -99,7 +99,7 @@ function chain<E extends { and(column: any, value?: any): E }>(
  * Turn a `RollbackSignal` back into a response, and rethrow anything else.
  *
  * The rethrow is the load-bearing half. A `catch` that answered 200 for every
- * throw would report a failed write as a successful dry run — see `preview.ts`.
+ * throw would report a failed write as a successful dry run. See `preview.ts`.
  */
 function fromRollback(error: any): Envelope {
   if (isRollbackSignal(error)) {
@@ -107,7 +107,10 @@ function fromRollback(error: any): Envelope {
       ? response.json.error(error.status, error.message, error.report)
       : response.json.success(error.message, error.report, error.status)
   }
-  return response.json.error(400, error?.message ?? 'The write failed')
+  // A rollback signal carries a message this plugin wrote, so it is safe to
+  // pass on. Anything else reaching here is the driver's, and the driver's
+  // text is not the caller's business. See `refuse`.
+  return refuse('write', error)
 }
 
 // ---------------------------------------------------------------- POST /rows
@@ -145,7 +148,7 @@ export async function handleInsertRows(
     async () => {
       // `DB.Insert` already batches under the adapter's parameter ceiling and
       // wraps multiple batches in one transaction, so there is nothing to add
-      // here — which is exactly why the insert goes through it rather than
+      // here, which is exactly why the insert goes through it rather than
       // through the adapter's own `insert()`.
       const insert = DB.Insert.into(table.name).values(records)
       if (!returning) {
@@ -154,7 +157,7 @@ export async function handleInsertRows(
           inserted: Number(result.changes ?? 0),
         })
       }
-      // `RETURNING` is SQLite and Postgres only — MySQL has no such clause and
+      // `RETURNING` is SQLite and Postgres only: MySQL has no such clause and
       // answers with its own syntax error, which is loud and correct. It is not
       // emulated: a re-SELECT would have to guess the generated keys, and
       // guessing which rows were just written is the class of bug this whole
@@ -167,7 +170,7 @@ export async function handleInsertRows(
         rows: written,
       })
     },
-    (error: any) => response.json.error(400, error?.message ?? 'Insert failed'),
+    error => refuse('insert', error),
   )
 }
 
@@ -187,7 +190,7 @@ export async function handleUpdateRow(
     // "I accept that".
     return response.json.error(
       400,
-      'expect is required — send {} to update without a concurrency check',
+      'expect is required: send {} to update without a concurrency check',
     )
   }
 
@@ -256,6 +259,202 @@ export async function handleUpdateRow(
   )
 }
 
+type PreparedEdit = {
+  key: Record<string, unknown>
+  set: Record<string, unknown>
+  expect: Record<string, unknown>
+}
+
+type CollapsedGroup = {
+  /** The one identity column every member addresses. */
+  column: string
+  set: Record<string, unknown>
+  expect: Record<string, unknown>
+  /**
+   * The original edit index alongside its key value, so a conflict still names
+   * the edit the caller sent rather than a position in a regrouped list.
+   */
+  members: { index: number; value: unknown }[]
+}
+
+/**
+ * Split a bulk edit into runs that can share one statement and those that
+ * cannot.
+ *
+ * A bulk edit issued one `UPDATE` per row, so 1,000 edits were 1,000
+ * statements plus the transaction and the conflict probes on top. The rows
+ * almost always share their `set` and their `expect` - that is what makes it a
+ * *bulk* edit, one action applied to a selection - so grouping on those two
+ * plus the identity column collapses the common case to one statement per
+ * group.
+ *
+ * Three things disqualify a member, and each falls back rather than failing:
+ *
+ *   - a composite identity, because `IN` addresses one column;
+ *   - a `null` key value, because `IN` never matches NULL and `IS NULL` is a
+ *     different clause. The single-row `where` already handles it correctly,
+ *     so falling back is not a compromise;
+ *   - being alone in its group, where one statement is one statement either
+ *     way and the collapsed path would only add a probe.
+ *
+ * Exported as a test seam (convention 9). Two of the three disqualifications -
+ * a composite identity and a `null` key value - are hard to reach through the
+ * endpoint, because a NOT NULL primary key cannot carry a null and validation
+ * refuses it before this runs. The guard still earns its place: identity can
+ * be a unique index over a nullable column, and `IN` would silently match
+ * nothing there. Asserting it directly is the only honest way to pin it.
+ */
+export function groupBulkEdits(prepared: (PreparedEdit | null)[]): {
+  collapsible: CollapsedGroup[]
+  single: number[]
+} {
+  const groups = new Map<string, CollapsedGroup>()
+  const single: number[] = []
+
+  for (let index = 0; index < prepared.length; index++) {
+    const edit = prepared[index]!
+    const columns = Object.keys(edit.key)
+    const column = columns[0]
+    const value = column === undefined ? undefined : edit.key[column]
+
+    if (columns.length !== 1 || column === undefined || value === null) {
+      single.push(index)
+      continue
+    }
+
+    const signature = JSON.stringify([column, edit.set, edit.expect])
+    const group = groups.get(signature)
+    if (group) {
+      group.members.push({ index, value })
+    } else {
+      groups.set(signature, {
+        column,
+        set: edit.set,
+        expect: edit.expect,
+        members: [{ index, value }],
+      })
+    }
+  }
+
+  const collapsible: CollapsedGroup[] = []
+  for (const group of groups.values()) {
+    if (group.members.length > 1) collapsible.push(group)
+    else single.push(group.members[0]!.index)
+  }
+  // The leftovers came out of a Map and have lost their order; they are
+  // applied in sequence below, so put them back in the order they were sent.
+  single.sort((a, b) => a - b)
+
+  return { collapsible, single }
+}
+
+/**
+ * Apply one collapsed group, and report any member that cannot be applied.
+ *
+ * **The probe runs before the write, not after it**, and that ordering is the
+ * whole design. A collapsed `UPDATE` reports a row count, not identities, so a
+ * short count says *some* member did not match without saying which. Finding
+ * out afterwards is not possible: the update has already changed the columns
+ * `expect` refers to, and re-running the members one at a time inside the same
+ * transaction would apply them twice.
+ *
+ * So: ask which keys satisfy identity AND expect, treat the rest as conflicts,
+ * and update only when there are none. That answers the MySQL no-op case for
+ * free - a row that matches the probe but reports zero changed rows is a
+ * no-op, decided exactly as the single-row path decides it, and the probe has
+ * already established the row is there.
+ *
+ * Any conflict rolls the entire bulk edit back, so a group that has one skips
+ * its update rather than writing something about to be discarded.
+ */
+async function applyCollapsedGroup(
+  table: TableFacts,
+  group: CollapsedGroup,
+  conflicts: Conflict[],
+): Promise<number> {
+  let changed = 0
+
+  // One bound parameter per key, plus the `set` and `expect` values, has to
+  // fit inside the dialect's statement limit. `maxQueryParams` is the
+  // adapter's own number; SQLite's 32,766 is the smallest of the three.
+  const fixed = Object.keys(group.set).length + Object.keys(group.expect).length
+  const limit = Math.max(1, getActiveDb().maxQueryParams - fixed - 1)
+
+  for (let start = 0; start < group.members.length; start += limit) {
+    const chunk = group.members.slice(start, start + limit)
+
+    const matched = await selectKeyValues(
+      table,
+      group.column,
+      chunk.map(m => m.value),
+      group.expect,
+    )
+
+    const missing = chunk.filter(m => !matched.has(m.value))
+    if (missing.length) {
+      for (const member of missing) {
+        const key = { [group.column]: member.value }
+        conflicts.push({
+          index: member.index,
+          key,
+          reason: 'the row changed since it was read',
+          row: await selectRow(table, key),
+        })
+      }
+      continue
+    }
+
+    let statement = DB.Update.table(table.name)
+      .set(group.set)
+      .where(group.column, DB.inList(chunk.map(m => m.value)))
+    for (const [column, value] of Object.entries(group.expect)) {
+      // `expect` has already been through `validatePartial`, which is what
+      // decides a value is comparable at all; the builder's parameter type is
+      // narrower than "whatever survived that" and the cast says so once here
+      // rather than loosening the validator.
+      statement = statement.and(column, value as never)
+    }
+
+    const result = await statement.run()
+    changed += Number(result.changes ?? 0)
+  }
+
+  return changed
+}
+
+/**
+ * Which of `values` name a row that also satisfies `expect`.
+ *
+ * A Set of the key values seen, so a key matching more than one row counts
+ * once: the question is whether the edit addresses anything, not how much.
+ */
+async function selectKeyValues(
+  table: TableFacts,
+  column: string,
+  values: unknown[],
+  expect: Record<string, unknown>,
+): Promise<Set<unknown>> {
+  const params: unknown[] = [...values]
+  const clauses = [`${qId(column)} IN (${values.map(() => '?').join(', ')})`]
+
+  for (const [name, value] of Object.entries(expect)) {
+    if (value === null) {
+      clauses.push(`${qId(name)} IS NULL`)
+    } else {
+      params.push(value)
+      clauses.push(`${qId(name)} = ?`)
+    }
+  }
+
+  const rows = (await getActiveDb()
+    .query(
+      `SELECT ${qId(column)} FROM ${qId(table.name)} WHERE ${clauses.join(' AND ')}`,
+    )
+    .all(...params)) as Record<string, unknown>[]
+
+  return new Set(rows.map(row => row[column]))
+}
+
 // ---------------------------------------------------------- POST /rows/bulk
 
 export async function handleBulkEdit(
@@ -313,7 +512,13 @@ export async function handleBulkEdit(
         const conflicts: Conflict[] = []
         let changed = 0
 
-        for (let index = 0; index < prepared.length; index++) {
+        const { collapsible, single } = groupBulkEdits(prepared)
+
+        for (const group of collapsible) {
+          changed += await applyCollapsedGroup(table, group, conflicts)
+        }
+
+        for (const index of single) {
           const edit = prepared[index]!
           const predicate = { ...edit.key, ...edit.expect }
           const result = await chain(
@@ -339,7 +544,7 @@ export async function handleBulkEdit(
 
         // **All or nothing.** A bulk edit is one action from the user's side,
         // and a partial apply leaves them with no way to know which half
-        // landed — the retry then double-applies whatever succeeded. Any
+        // landed: the retry then double-applies whatever succeeded. Any
         // conflict rolls the whole transaction back.
         if (conflicts.length) conflictRollback({ changed: 0, conflicts })
         if (dryRun) previewRollback({ changed, conflicts })