@protobase/server 0.17.0 → 0.18.0

package/dist/admin-options.d.ts

@@ -13,10 +13,9 @@ export type AdminOptions = {
   basePath?: string;
   /** Applied to every request's transaction with SET LOCAL. Default 15000. */
   statementTimeoutMs?: number;
-  /** List shapes that would sequentially scan a table of more than `seqScanRows` rows (default 10000). Default mode `reject`. */
+  /** Reads that filter, sort or count values by a column no index serves. Default mode `reject`. */
   scanGuard?: {
     mode?: ScanGuardMode;
-    seqScanRows?: number;
   };
   /** Runs inside the write transaction after each create, update and delete. */
   writeHooks?: PipelineHook[];

package/dist/aggregate-service.js

@@ -5,7 +5,7 @@ import { readScope, requestAccess, requireOperation } from "./request-access.js"
 import { readTransaction } from "./transactions.js";
 import { parseParams } from "./params.js";
 import { labelRequests, relatedLabels } from "./related-labels.js";
-import { facetCounts, histogram, series } from "@protobase/query";
+import { facetCounts, histogram, series, withoutField } from "@protobase/query";
 import { sql } from "kysely";
 import { z } from "zod";
 //#region src/aggregate-service.ts
@@ -66,25 +66,36 @@ const facets = async (deps, entry, session, input) => {
 	const params = parseParams(facetParams, input);
 	const { model, filter: checked, scope } = await scoped(deps, entry, session, params.filter);
 	const labels = params.label === void 0 ? [] : await labelRequests(deps, session, model, [params.field], `${params.field}.${params.label}`);
-	return readTransaction(deps.db, deps.statementTimeoutMs, async (trx) => {
-		const counts = await facetCounts(trx, model, params.field, checked, {
-			statementTimeoutMs: deps.statementTimeoutMs,
-			...params.limit !== void 0 && { limit: params.limit },
-			scope
-		});
-		const names = (await relatedLabels(trx, labels, counts.map((facet) => ({ [params.field]: facet.value }))))[params.field] ?? {};
-		const named = counts.map((facet) => {
-			const label = facet.value === null ? void 0 : names[String(facet.value)];
-			return label === void 0 ? facet : {
-				...facet,
-				label
-			};
-		});
-		return {
-			field: params.field,
-			facets: named
-		};
+	const others = withoutField(checked, params.field);
+	const warning = await deps.guard(entry, {
+		...others && { filter: others },
+		groupBy: params.field
+	}, {
+		model,
+		scope
 	});
+	return {
+		body: await readTransaction(deps.db, deps.statementTimeoutMs, async (trx) => {
+			const counts = await facetCounts(trx, model, params.field, checked, {
+				statementTimeoutMs: deps.statementTimeoutMs,
+				...params.limit !== void 0 && { limit: params.limit },
+				scope
+			});
+			const names = (await relatedLabels(trx, labels, counts.map((facet) => ({ [params.field]: facet.value }))))[params.field] ?? {};
+			const named = counts.map((facet) => {
+				const label = facet.value === null ? void 0 : names[String(facet.value)];
+				return label === void 0 ? facet : {
+					...facet,
+					label
+				};
+			});
+			return {
+				field: params.field,
+				facets: named
+			};
+		}),
+		warning
+	};
 };
 const earliest = async (deps, access, fieldName, scope) => {
 	const target = access.model.fields[fieldName];

package/dist/list-photos.js

@@ -69,8 +69,7 @@ const recordPhotos = async (deps, request, model, rows) => {
 	const scope = readScope(access);
 	const warning = await deps.guard(access.entry, {
 		filter: filter.filter,
-		...sort && { sort },
-		limit: cardPhotoLimit
+		...sort && { sort }
 	}, {
 		model: access.model,
 		scope

package/dist/list-service.js

@@ -9,8 +9,9 @@ import { readTransaction } from "./transactions.js";
 import { deletedRows, listParams, pageSizeOf, parseParams, seekParams, selectedFields } from "./params.js";
 import { labelRequests, relatedLabels } from "./related-labels.js";
 import { photosRequest, recordPhotos } from "./list-photos.js";
-import { estimateCount, exactCount, listQuery, seekToPosition } from "@protobase/query";
+import { countUpTo, estimateCount, listQuery, seekToPosition } from "@protobase/query";
 //#region src/list-service.ts
+const exactCountLimit = 1e4;
 /**
 * AIP-132 list, the `:search` variant and (with `jump`) `:seek`: both take the same parameters, from the query string or a JSON body.
 * Everything the caller names (filter, order_by, fields, search) is checked against the model as this caller sees it,
@@ -82,7 +83,7 @@ const listRecords = async (deps, entry, session, input, maxFilterLength, jump =
 			etags: await Promise.all(rows.map((row, i) => etagOf(entry, page.rows[i], row))),
 			permissions: deps.rowPermissions(entry.name) ? await rowPermissions(trx, access, rows) : void 0,
 			estimate: await estimateCount(trx, queryModel, filter, scope),
-			...params.count === "exact" && { total: await exactCount(trx, queryModel, filter, scope) },
+			...params.count === "exact" && { counted: await countUpTo(trx, queryModel, 10001, filter, scope) },
 			...params.labels !== void 0 && { labels: await relatedLabels(trx, labels, rows) }
 		};
 	});
@@ -99,7 +100,7 @@ const listRecords = async (deps, entry, session, input, maxFilterLength, jump =
 			next_page_token: result.page.nextCursor ?? "",
 			...position !== void 0 && { prev_page_token: result.page.prevCursor ?? "" },
 			total_size_estimate: result.estimate,
-			...result.total !== void 0 && { total_size: result.total },
+			...result.counted !== void 0 && result.counted <= exactCountLimit && { total_size: result.counted },
 			...result.labels && { labels: result.labels },
 			...included && { photos: included.photos }
 		}

package/dist/map-service.js

@@ -48,7 +48,7 @@ const mapRecords = async (deps, entry, session, input) => {
 			body: {
 				items: body.items,
 				buckets: [],
-				total_size: body.total_size ?? found,
+				total_size: body.total_size ?? (found > 200 ? body.total_size_estimate : found),
 				...body.labels && { labels: body.labels },
 				...body.photos && { photos: body.photos }
 			}

package/dist/openapi/components.js

@@ -93,7 +93,7 @@ const parameters = {
 	count: {
 		name: "count",
 		in: "query",
-		description: "`exact` adds `total_size`, a real count(*)",
+		description: "`exact` adds `total_size`, a count of the matches when at most 10,000 match",
 		schema: {
 			type: "string",
 			enum: ["exact"]

package/dist/openapi/resource-paths.js

@@ -79,7 +79,7 @@ const page = (name, seek = false) => json({
 		},
 		total_size: {
 			type: "integer",
-			description: "Only with count=exact"
+			description: "Only with count=exact, and when at most 10,000 records match"
 		},
 		labels,
 		photos

package/dist/routes/collection.js

@@ -21,7 +21,10 @@ const collectionRoutes = (deps) => {
 			const { body, warning } = await listRecords(deps, entry, session, query);
 			return c.json(body, 200, warningHeader(warning));
 		}
-		if (action === "facets") return c.json(await facets(deps, entry, session, query));
+		if (action === "facets") {
+			const { body, warning } = await facets(deps, entry, session, query);
+			return c.json(body, 200, warningHeader(warning));
+		}
 		if (action === "series") return c.json(await timeSeries(deps, entry, session, query));
 		if (action === "histogram") return c.json(await numericHistogram(deps, entry, session, query));
 		if (action === "map") {

package/dist/scan-guard.js

@@ -1,70 +1,43 @@
 import { HttpProblem } from "./problem.js";
-import { scopeConditions, tableId } from "./sql-parts.js";
-import { filterFields, filterShape } from "./filter-shape.js";
-import { buildListQuery, compileFilter, explainGuard } from "@protobase/query";
-import { sql } from "kysely";
+import { tableIndexes, uncoveredColumns } from "@protobase/query";
 //#region src/scan-guard.ts
-const maxCached = 1e3;
-const filteredQuery = (db, entry, shape, { scope }) => {
-	const { model } = entry;
-	const conditions = [
-		...scopeConditions(scope.fullModel ?? model, scope.tenantValue),
-		...scope.rowFilter ? [compileFilter(model, scope.rowFilter)] : [],
-		...shape.filter ? [compileFilter(model, shape.filter)] : []
-	];
-	let query = db.selectFrom(sql`${tableId(model)}`.as("t")).select(sql.lit(1).as("one"));
-	for (const condition of conditions) query = query.where(condition);
-	return query;
+const reading = (column) => {
+	switch (column.use) {
+		case "filter": return `Filtering on "${column.field}"`;
+		case "access": return `The filter your role is limited by, on "${column.field}",`;
+		case "sort": return `Sorting by "${column.field}"`;
+		case "facet": return `Counting the values of "${column.field}"`;
+	}
 };
-const pagedQuery = async (db, shape, { model, scope }) => {
-	const { query } = await buildListQuery(db, model, shape, scope);
-	return query;
+const describe = (column) => {
+	const kind = column.need === "btree" ? "an index" : column.need === "gin" ? "a GIN index" : "a pg_trgm index";
+	return `${reading(column)} needs ${kind} that serves the column "${column.column}". Add one, for example: ${column.suggestion}`;
 };
-const indexStatement = (entry, fields) => {
-	const { schema, name } = entry.model.table;
-	const table = schema ? `"${schema}"."${name}"` : `"${name}"`;
-	const columns = [...new Set(fields.map((field) => field.column))];
-	if (columns.length === 0) return "an index that covers the filter your role is limited by";
-	return `create index on ${table} (${columns.map((column) => `"${column}"`).join(", ")})`;
-};
-/** Plans each list query shape once (EXPLAIN, never executed) and remembers the verdict. */
+/**
+* Checks that an index serves every column a read filters, sorts or groups by, from the table's indexes in the catalog
+* (read once per table and kept for the process lifetime), so a read is never refused or allowed by how many rows match.
+*/
 const createScanGuard = (db, options = {}) => {
-	const { mode = "reject", seqScanRows } = options;
-	const verdicts = /* @__PURE__ */ new Map();
-	const thresholds = {
-		...seqScanRows && { seqScanRows },
-		sortRows: Number.MAX_SAFE_INTEGER
-	};
-	const judge = async (entry, shape, context) => {
-		const filtered = shape.filter ? await explainGuard(db, filteredQuery(db, entry, shape, context), thresholds) : void 0;
-		const paged = await explainGuard(db, await pagedQuery(db, shape, context), thresholds);
-		const sortField = shape.sort?.[0] && entry.model.fields[shape.sort[0][0]];
-		const suggestions = [...filtered && !filtered.ok ? [indexStatement(entry, filterFields(shape.filter))] : [], ...!paged.ok && sortField ? [indexStatement(entry, [sortField])] : []];
-		return {
-			findings: [...filtered?.findings ?? [], ...paged.findings],
-			suggestion: suggestions[0] ?? indexStatement(entry, filterFields(shape.filter))
-		};
-	};
-	const verdictFor = async (entry, shape, context) => {
-		const key = `${entry.name}|${filterShape(context.scope.rowFilter)}|${filterShape(shape.filter)}|${(shape.sort ?? []).map(([name, direction]) => `${name}:${direction}`).join(",")}`;
-		const known = verdicts.get(key);
+	const { mode = "reject" } = options;
+	const tables = /* @__PURE__ */ new Map();
+	const indexesOf = async (entry) => {
+		const known = tables.get(entry.name);
 		if (known) return known;
-		const verdict = await judge(entry, shape, context);
-		if (verdicts.size >= maxCached) verdicts.delete(verdicts.keys().next().value);
-		verdicts.set(key, verdict);
-		return verdict;
+		const read = await tableIndexes(db, entry.model);
+		tables.set(entry.name, read);
+		return read;
 	};
-	/** Throws a 400 problem for an expensive shape in `reject` mode; in `warn` mode returns the warning text. */
-	return async (entry, shape, context) => {
+	/** Throws a 400 problem for a column no index serves in `reject` mode; in `warn` mode returns the warning text. */
+	return async (entry, query, context) => {
 		if (mode === "off") return void 0;
-		const { findings, suggestion } = await verdictFor(entry, shape, context);
-		const finding = findings[0];
-		if (!finding) return void 0;
-		const detail = `This query would sequentially scan "${finding.relation}" (about ${finding.rows} rows). Add an index, for example: ${suggestion}`;
+		const columns = uncoveredColumns(context.model, await indexesOf(entry), query, context.scope);
+		const first = columns[0];
+		if (!first) return void 0;
+		const detail = describe(first);
 		if (mode === "warn") return detail;
 		throw new HttpProblem(400, "expensive-query", "Bad Request", detail, {
-			findings,
-			suggestion
+			columns,
+			suggestion: first.suggestion
 		});
 	};
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/server",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "The Protobase REST API as a Hono app, with Better Auth sign-in.",
   "license": "Apache-2.0",
   "repository": {
@@ -30,9 +30,9 @@
   },
   "dependencies": {
     "@better-auth/passkey": "^1.7.7",
-    "@protobase/layout": "0.17.0",
-    "@protobase/query": "0.17.0",
-    "@protobase/schema": "0.17.0",
+    "@protobase/layout": "0.18.0",
+    "@protobase/query": "0.18.0",
+    "@protobase/schema": "0.18.0",
     "@react-email/render": "^2.1.0",
     "@scalar/hono-api-reference": "^0.12.10",
     "@sentry/core": "^11.7.0",

package/dist/filter-shape.js

@@ -1,25 +0,0 @@
-//#region src/filter-shape.ts
-/** The filter with its values blanked out: queries that differ only in values share a shape. */
-const filterShape = (filter) => {
-	if (!filter) return "";
-	switch (filter.kind) {
-		case "and":
-		case "or": return `${filter.kind}(${filter.args.map(filterShape).join(",")})`;
-		case "not": return `not(${filterShape(filter.arg)})`;
-		case "compare": return `${filter.field.name}${filter.op}?`;
-		case "search": return "search";
-		default: return `${filter.kind}:${filter.field.name}`;
-	}
-};
-const filterFields = (filter) => {
-	if (!filter) return [];
-	switch (filter.kind) {
-		case "and":
-		case "or": return filter.args.flatMap(filterFields);
-		case "not": return filterFields(filter.arg);
-		case "search": return [];
-		default: return [filter.field];
-	}
-};
-//#endregion
-export { filterFields, filterShape };