@hienlh/ppm 0.23.12 → 0.23.13

package/src/server/routes/database-grid-export.ts

@@ -0,0 +1,212 @@
+/**
+ * The download half of DBGate's Export ▸. `POST /connections/:id/grid/export` (database-grid.ts)
+ * starts the read and answers with a ticket; `GET /api/db/grid-export/:ticket` is the file, written
+ * as the rows come off the database and sent as fast as the browser takes it — the read waits
+ * while it does not. The export is audited once its download has ended, however it ended.
+ */
+import { Hono } from "hono";
+import type { GridExportFormat } from "../../shared/db-grid-export.ts";
+import { toJsonValue } from "../../services/database/db-values.ts";
+import { exportFile, type ExportTarget } from "../../services/database/grid-export.ts";
+import { claimExportTicket, issueExportTicket } from "../../services/database/grid-export-tickets.ts";
+import type { GridExport } from "../../services/database/grid.service.ts";
+import { EDGE_ROWS, MAX_RESULT_BYTES } from "../../services/query-audit/result-truncate.ts";
+import { attachmentDisposition } from "../../services/fs-ops/fs-content-disposition.ts";
+import { err } from "../../types/api.ts";
+import { logQueryAs, type AuditCaller, type AuditFields } from "./query-audit-hook.ts";
+import { holdRequestOpen } from "./database-route-helpers.ts";
+
+/** Seconds the download may go without sending a byte: the database can take that long to find the next rows. */
+const DOWNLOAD_IDLE_SECONDS = 300;
+
+/** Bytes the response holds ready before it stops asking the read for more. */
+const RESPONSE_BUFFER_BYTES = 1024 * 1024;
+
+export const CONTENT_TYPES: Record<GridExportFormat, string> = {
+  json: "application/json; charset=utf-8",
+  jsonl: "application/x-ndjson; charset=utf-8",
+  sql: "application/sql; charset=utf-8",
+  csv: "text/csv; charset=utf-8",
+  csvSemicolon: "text/csv; charset=utf-8",
+  csvExcel: "text/csv; charset=utf-16le",
+  tsv: "text/tab-separated-values; charset=utf-8",
+  xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  xml: "application/xml; charset=utf-8",
+};
+
+/**
+ * Exports open at once — waiting for their download or downloading. Each holds a connection to the
+ * database of its own and a batch of rows, so a script starting exports in a loop cannot use up the
+ * database's connections or PPM's memory.
+ */
+export const MAX_OPEN_EXPORTS = 8;
+
+let openExports = 0;
+
+export class TooManyExportsError extends Error {
+  constructor() {
+    super(`${MAX_OPEN_EXPORTS} exports are already running. Export again once one has finished.`);
+  }
+}
+
+/** Take one of the export slots, or throw `TooManyExportsError`; the function returned gives it back, once. */
+export function reserveExportSlot(): () => void {
+  if (openExports >= MAX_OPEN_EXPORTS) throw new TooManyExportsError();
+  openExports++;
+  let released = false;
+  return () => {
+    if (released) return;
+    released = true;
+    openExports--;
+  };
+}
+
+/** A value as the audit log stores it. A string longer than the whole sample may be is cut, losing nothing stored. */
+function sampleValue(value: unknown): unknown {
+  const json = toJsonValue(value);
+  return typeof json === "string" && json.length > MAX_RESULT_BYTES ? json.slice(0, MAX_RESULT_BYTES) : json;
+}
+
+/** The first and last few rows of an export, which its audit entry keeps as every audited read does. */
+export class RowSample {
+  private head: Record<string, unknown>[] = [];
+  private tail: unknown[][] = [];
+
+  constructor(private readonly columns: readonly string[]) {}
+
+  add(batch: readonly unknown[][]): void {
+    for (const row of batch) {
+      // One past the edge, so the stored sample says rows were left out between head and tail.
+      if (this.head.length <= EDGE_ROWS) this.head.push(this.record(row));
+      else {
+        this.tail.push(row);
+        if (this.tail.length > EDGE_ROWS) this.tail.shift();
+      }
+    }
+  }
+
+  records(): Record<string, unknown>[] {
+    return [...this.head, ...this.tail.map((row) => this.record(row))];
+  }
+
+  private record(row: readonly unknown[]): Record<string, unknown> {
+    return Object.fromEntries(this.columns.map((c, i) => [c, sampleValue(row[i])]));
+  }
+}
+
+export interface ExportAudit {
+  caller: AuditCaller;
+  fields: Omit<AuditFields, "sql" | "status">;
+  startedAt: number;
+}
+
+/**
+ * One value read whole — Save cell to file's — ticketed for its download, a slice at a time. Its
+ * slot is given back once the bytes are sent, the download stopped, or nobody came for them.
+ */
+export function ticketCellDownload(bytes: Uint8Array, fileName: string, release: () => void): string {
+  return issueExportTicket({
+    fileName,
+    contentType: "application/octet-stream",
+    abandon: release,
+    open: () => {
+      let at = 0;
+      return new ReadableStream<Uint8Array>({
+        pull(controller) {
+          if (at >= bytes.length) {
+            controller.close();
+            release();
+            return;
+          }
+          controller.enqueue(bytes.subarray(at, at + RESPONSE_BUFFER_BYTES));
+          at += RESPONSE_BUFFER_BYTES;
+        },
+        cancel: release,
+      });
+    },
+  });
+}
+
+/**
+ * The file of an opened export, ticketed for its download. `release` is called once the export has
+ * ended — downloaded, failed, stopped, or never fetched.
+ */
+export function ticketGridExport(
+  opened: GridExport, target: ExportTarget, fileName: string, audit: ExportAudit, release: () => void,
+): string {
+  let rows = 0;
+  let ended = false;
+  const sample = new RowSample(opened.columns.map((c) => c.name));
+  const end = (error: string | null): void => {
+    if (ended) return;
+    ended = true;
+    release();
+    logQueryAs(audit.caller, {
+      ...audit.fields,
+      sql: opened.built.displaySql,
+      status: error === null ? "ok" : "error",
+      error,
+      rows: sample.records(),
+      rowCount: rows,
+      durationMs: Date.now() - audit.startedAt,
+    });
+  };
+
+  async function* counted(): AsyncGenerator<unknown[][]> {
+    for await (const batch of opened.batches) {
+      rows += batch.length;
+      sample.add(batch);
+      yield batch;
+    }
+  }
+
+  return issueExportTicket({
+    fileName,
+    contentType: CONTENT_TYPES[target.format],
+    abandon() {
+      end("The download never started");
+      opened.close().catch(() => {});
+    },
+    open() {
+      const file = exportFile(target, opened.columns, counted());
+      return new ReadableStream<Uint8Array>({
+        async pull(controller) {
+          try {
+            const next = await file.next();
+            if (next.done) {
+              controller.close();
+              end(null);
+            } else controller.enqueue(next.value);
+          } catch (e) {
+            end((e as Error).message);
+            controller.error(e);
+          }
+        },
+        async cancel() {
+          end(`The download was stopped after ${rows.toLocaleString("en-US")} rows`);
+          await file.return(undefined);
+        },
+      }, { highWaterMark: RESPONSE_BUFFER_BYTES, size: (chunk) => chunk?.byteLength ?? 0 });
+    },
+  });
+}
+
+export const gridExportDownloadRoutes = new Hono();
+
+/** GET /api/db/grid-export/:ticket — the file, once. The ticket stands in for the Authorization header a download cannot send. */
+gridExportDownloadRoutes.get("/:ticket", (c) => {
+  // Hono answers HEAD with the GET handler and throws its body away: that would spend the ticket
+  // and leave the read open with nothing to take its rows.
+  if (c.req.method !== "GET") return c.json(err("Download the export with GET"), 405, { Allow: "GET" });
+  const download = claimExportTicket(c.req.param("ticket"));
+  if (!download) return c.json(err("This export has been downloaded already, or waited too long. Export again."), 404);
+  holdRequestOpen(c, DOWNLOAD_IDLE_SECONDS);
+  return new Response(download.open(), {
+    headers: {
+      "Content-Type": download.contentType,
+      "Content-Disposition": attachmentDisposition(download.fileName),
+      "Cache-Control": "no-store",
+      "X-Content-Type-Options": "nosniff",
+    },
+  });
+});

package/src/server/routes/database-grid.ts

@@ -0,0 +1,317 @@
+/**
+ * The data grid's reads: `POST /connections/:id/grid` for one page,
+ * `POST /connections/:id/grid/count` for the total,
+ * `POST /connections/:id/grid/values` for one column's distinct values,
+ * `POST /connections/:id/grid/export` for every row as a file and
+ * `POST /connections/:id/grid/cell` for one cell's value whole. The browser
+ * sends a structured request; the SQL is built here, for the connection's
+ * dialect.
+ */
+import { Hono, type Context } from "hono";
+import type { ConnectionRow } from "../../services/db.service.ts";
+import { isReadonlyRefusal, readonlyRefusalMessage } from "../../services/database/db-errors.ts";
+import { dialectFor } from "../../services/database/dialects.ts";
+import { isReadOnlyQuery } from "../../services/database/readonly-check.ts";
+import {
+  GridRequestError, parseGridCountRequest, parseGridExportRequest, parseGridRequest, parseGridValuesRequest,
+  rawSqlConditions, type GridScope,
+} from "../../services/database/grid-query-builder.ts";
+import {
+  GRID_COUNT_TIMEOUT_MS, GRID_EXACT_COUNT_TIMEOUT_MS, GridTableNotFoundError, countGridRows, defaultSchemaFor, fetchGridPage,
+  fetchGridValues, openGridExport, type GridExport,
+} from "../../services/database/grid.service.ts";
+import { ChangesetRequestError } from "../../services/database/changeset.ts";
+import { GridCellConflictError, GridCellGoneError, parseGridCellRequest, readGridCell } from "../../services/database/grid-cell.ts";
+import { rowsToRecords } from "../../shared/db-grid.ts";
+import { gridExportFileName, type GridExportTicket } from "../../shared/db-grid-export.ts";
+import { ok, err } from "../../types/api.ts";
+import { auditCaller, logQuery } from "./query-audit-hook.ts";
+import { reserveExportSlot, ticketCellDownload, ticketGridExport, TooManyExportsError } from "./database-grid-export.ts";
+import { connAudit, connTarget, databaseParam, holdRequestOpen, requestDatabase, resolveTargetConn } from "./database-route-helpers.ts";
+
+export const databaseGridRoutes = new Hono();
+
+const READONLY_RAW_SQL = "Connection is readonly — this SQL condition is not a plain read. Change this in PPM web UI.";
+
+/** Seconds a page read may take: a sort over a big table has nothing to send until it is done. */
+const GRID_READ_SECONDS = 300;
+
+/** Seconds past a count's own limit that the request is kept open, for the table lookup and the estimate. */
+const COUNT_SLACK_SECONDS = 30;
+
+/** Parse the body, answering 400 with the reason when it cannot become SQL. */
+async function readRequest<T>(
+  c: Context,
+  conn: ConnectionRow,
+  parse: (body: unknown, defaultSchema: string | null) => T,
+): Promise<T | Response> {
+  let body: unknown;
+  try { body = await c.req.json(); } catch { return c.json(err("Request body must be JSON"), 400); }
+  try {
+    return parse(body, defaultSchemaFor(conn.type));
+  } catch (e) {
+    if (e instanceof GridRequestError) return c.json(err(e.message), 400);
+    throw e;
+  }
+}
+
+/**
+ * A readonly connection still runs the SQL of a `{$$ …}` condition, so it gets
+ * the same first check as typed SQL: nothing that looks like a write. Only a
+ * column's own filter can hold one; the parser refuses it in `anyColumn`.
+ */
+function blocksRawSql(conn: ConnectionRow, req: GridScope): boolean {
+  const dialect = dialectFor(conn.type).name;
+  return !!conn.readonly && rawSqlConditions(req.filters).some((sql) => !isReadOnlyQuery(`SELECT 1 WHERE ${sql}`, dialect));
+}
+
+/** The statement a blocked request would have run, for the audit log. */
+function blockedSql(conn: ConnectionRow, req: GridScope): string {
+  const attempted = rawSqlConditions(req.filters).map((sql) => `(${sql})`).join(" AND ");
+  return `SELECT * FROM ${dialectFor(conn.type).qualify(req.table, req.schema)} WHERE ${attempted}`;
+}
+
+/** Status for an error the grid service raised on purpose, or the database's readonly refusal; 500 for anything else. */
+function errorStatus(e: unknown): 400 | 403 | 404 | 409 | 500 {
+  if (e instanceof GridRequestError || e instanceof ChangesetRequestError) return 400;
+  if (e instanceof GridTableNotFoundError || e instanceof GridCellGoneError) return 404;
+  if (e instanceof GridCellConflictError) return 409;
+  if (isReadonlyRefusal(e)) return 403;
+  return 500;
+}
+
+/** POST /connections/:id/grid — one page of a table, filtered and sorted by the server. */
+databaseGridRoutes.post("/:id/grid", async (c) => {
+  const startedAt = Date.now();
+  const conn = resolveTargetConn(c);
+  if (!conn) return c.json(err("Connection not found"), 404);
+  const req = await readRequest(c, conn, parseGridRequest);
+  if (req instanceof Response) return req;
+  holdRequestOpen(c, GRID_READ_SECONDS);
+
+  // Browsing a table is not audited, as it never was; filtering is, because it
+  // used to run through /query and the log should not lose those entries.
+  const audited = req.filters.length > 0 || req.anyColumn.length > 0;
+  const d = dialectFor(conn.type);
+  const auditBase = {
+    ...connAudit(conn),
+    source: "filter" as const,
+    operation: "select" as const,
+    params: {
+      ...databaseParam(c), table: req.table, schema: req.schema, filters: req.filters, anyColumn: req.anyColumn,
+      sort: req.sort, offset: req.offset, limit: req.limit,
+    },
+  };
+
+  if (blocksRawSql(conn, req)) {
+    logQuery(c, { ...auditBase, sql: blockedSql(conn, req), status: "blocked", error: READONLY_RAW_SQL, durationMs: Date.now() - startedAt });
+    return c.json(err(READONLY_RAW_SQL), 403);
+  }
+
+  try {
+    const page = await fetchGridPage(connTarget(conn, requestDatabase(c)), req);
+    if (audited) {
+      logQuery(c, {
+        ...auditBase,
+        sql: `${page.built.displaySql}\n${d.limitOffset(String(req.limit + 1), String(req.offset))}`,
+        status: "ok",
+        rows: rowsToRecords(page.response.columns, page.response.rows).records,
+        rowCount: page.response.rows.length,
+        durationMs: Date.now() - startedAt,
+      });
+    }
+    return c.json(ok(page.response));
+  } catch (e) {
+    const status = errorStatus(e);
+    const message = status === 403 ? readonlyRefusalMessage(e) : (e as Error).message;
+    if (audited && (status === 500 || status === 403)) {
+      // The statement text is not known when the failure came before it was built; params hold the request.
+      logQuery(c, {
+        ...auditBase,
+        sql: `SELECT * FROM ${d.qualify(req.table, req.schema)} WHERE …`,
+        status: status === 403 ? "blocked" : "error",
+        error: message,
+        durationMs: Date.now() - startedAt,
+      });
+    }
+    return c.json(err(message), status);
+  }
+});
+
+/**
+ * POST /connections/:id/grid/count — exact row count under the same filters,
+ * plus the database's estimate. Separate from /grid so opening a large table
+ * never waits on a COUNT(*); a count that runs too long answers `timedOut`.
+ * `exact` is a count the user asked for, which may run far longer.
+ */
+databaseGridRoutes.post("/:id/grid/count", async (c) => {
+  const conn = resolveTargetConn(c);
+  if (!conn) return c.json(err("Connection not found"), 404);
+  const req = await readRequest(c, conn, parseGridCountRequest);
+  if (req instanceof Response) return req;
+  if (blocksRawSql(conn, req)) return c.json(err(READONLY_RAW_SQL), 403);
+  const timeoutMs = req.exact ? GRID_EXACT_COUNT_TIMEOUT_MS : GRID_COUNT_TIMEOUT_MS;
+  holdRequestOpen(c, timeoutMs / 1000 + COUNT_SLACK_SECONDS);
+
+  try {
+    return c.json(ok(await countGridRows(connTarget(conn, requestDatabase(c)), req, timeoutMs)));
+  } catch (e) {
+    const status = errorStatus(e);
+    return c.json(err(status === 403 ? readonlyRefusalMessage(e) : (e as Error).message), status);
+  }
+});
+
+/**
+ * POST /connections/:id/grid/values — the distinct values of one column under
+ * the filters given, for "Choose value". Audited and vetted like a filtered
+ * page: the filters it carries are the user's, and so is a search.
+ */
+databaseGridRoutes.post("/:id/grid/values", async (c) => {
+  const startedAt = Date.now();
+  const conn = resolveTargetConn(c);
+  if (!conn) return c.json(err("Connection not found"), 404);
+  const req = await readRequest(c, conn, parseGridValuesRequest);
+  if (req instanceof Response) return req;
+
+  const audited = req.filters.length > 0 || req.anyColumn.length > 0 || req.search !== "";
+  const auditBase = {
+    ...connAudit(conn),
+    source: "filter" as const,
+    operation: "select" as const,
+    params: {
+      ...databaseParam(c), table: req.table, schema: req.schema, column: req.column, search: req.search,
+      filters: req.filters, anyColumn: req.anyColumn,
+    },
+  };
+
+  if (blocksRawSql(conn, req)) {
+    logQuery(c, { ...auditBase, sql: blockedSql(conn, req), status: "blocked", error: READONLY_RAW_SQL, durationMs: Date.now() - startedAt });
+    return c.json(err(READONLY_RAW_SQL), 403);
+  }
+
+  try {
+    const page = await fetchGridValues(connTarget(conn, requestDatabase(c)), req);
+    if (audited) {
+      logQuery(c, {
+        ...auditBase,
+        sql: page.built.displaySql,
+        status: "ok",
+        rowCount: page.response.values.length,
+        durationMs: Date.now() - startedAt,
+      });
+    }
+    return c.json(ok(page.response));
+  } catch (e) {
+    const status = errorStatus(e);
+    const message = status === 403 ? readonlyRefusalMessage(e) : (e as Error).message;
+    if (audited && (status === 500 || status === 403)) {
+      logQuery(c, {
+        ...auditBase,
+        sql: `SELECT DISTINCT … FROM ${dialectFor(conn.type).qualify(req.table, req.schema)} WHERE …`,
+        status: status === 403 ? "blocked" : "error",
+        error: message,
+        durationMs: Date.now() - startedAt,
+      });
+    }
+    return c.json(err(message), status);
+  }
+});
+
+/**
+ * POST /connections/:id/grid/export — DBGate's Export ▸: every row the grid's filters leave, in its
+ * sort, the columns it shows in its order, as one file. The read starts here and its first rows are
+ * read before answering, so a statement the database refuses is an error here and not a broken
+ * download; the answer is the ticket the file is then fetched with (database-grid-export.ts).
+ * Always audited — an export takes the table away whole — once its download has ended.
+ */
+databaseGridRoutes.post("/:id/grid/export", async (c) => {
+  const startedAt = Date.now();
+  const conn = resolveTargetConn(c);
+  if (!conn) return c.json(err("Connection not found"), 404);
+  const req = await readRequest(c, conn, parseGridExportRequest);
+  if (req instanceof Response) return req;
+  holdRequestOpen(c, GRID_READ_SECONDS);
+
+  const d = dialectFor(conn.type);
+  const auditBase = {
+    ...connAudit(conn),
+    source: "export" as const,
+    operation: "select" as const,
+    params: {
+      ...databaseParam(c), table: req.table, schema: req.schema, filters: req.filters, anyColumn: req.anyColumn,
+      sort: req.sort, columns: req.columns, format: req.format,
+    },
+  };
+
+  if (blocksRawSql(conn, req)) {
+    logQuery(c, { ...auditBase, sql: blockedSql(conn, req), status: "blocked", error: READONLY_RAW_SQL, durationMs: Date.now() - startedAt });
+    return c.json(err(READONLY_RAW_SQL), 403);
+  }
+
+  let release: () => void;
+  try {
+    release = reserveExportSlot();
+  } catch (e) {
+    if (e instanceof TooManyExportsError) return c.json(err(e.message), 429);
+    throw e;
+  }
+
+  let opened: GridExport;
+  try {
+    opened = await openGridExport(connTarget(conn, requestDatabase(c)), req);
+  } catch (e) {
+    release();
+    const status = errorStatus(e);
+    const message = status === 403 ? readonlyRefusalMessage(e) : (e as Error).message;
+    if (status === 500 || status === 403) {
+      logQuery(c, {
+        ...auditBase,
+        sql: `SELECT … FROM ${d.qualify(req.table, req.schema)}`,
+        status: status === 403 ? "blocked" : "error",
+        error: message,
+        durationMs: Date.now() - startedAt,
+      });
+    }
+    return c.json(err(message), status);
+  }
+
+  const fileName = gridExportFileName(req.table, req.format);
+  const ticket = ticketGridExport(
+    opened,
+    { format: req.format, dialect: d, table: req.table },
+    fileName,
+    { caller: auditCaller(c), fields: auditBase, startedAt },
+    release,
+  );
+  return c.json(ok<GridExportTicket>({ ticket, fileName }));
+});
+
+/**
+ * POST /connections/:id/grid/cell — Save cell to file for bytes the grid has only the start of: the
+ * row found again by its key, the value read whole, and the file downloaded by ticket as an
+ * export's is, taking one of the same slots while it waits. Not audited, as browsing is not: the
+ * row was on screen already.
+ */
+databaseGridRoutes.post("/:id/grid/cell", async (c) => {
+  const conn = resolveTargetConn(c);
+  if (!conn) return c.json(err("Connection not found"), 404);
+  const req = await readRequest(c, conn, parseGridCellRequest);
+  if (req instanceof Response) return req;
+  holdRequestOpen(c, GRID_READ_SECONDS);
+
+  let release: () => void;
+  try {
+    release = reserveExportSlot();
+  } catch (e) {
+    if (e instanceof TooManyExportsError) return c.json(err(e.message), 429);
+    throw e;
+  }
+  try {
+    const bytes = await readGridCell(connTarget(conn, requestDatabase(c)), req);
+    return c.json(ok<GridExportTicket>({ ticket: ticketCellDownload(bytes, req.fileName, release), fileName: req.fileName }));
+  } catch (e) {
+    release();
+    const status = errorStatus(e);
+    return c.json(err(status === 403 ? readonlyRefusalMessage(e) : (e as Error).message), status);
+  }
+});