@protobase/query 0.17.0 → 0.18.0

package/dist/compile-compare.js

@@ -11,10 +11,13 @@ const operators = {
 	">": ">",
 	">=": ">="
 };
-/** `=` and `!=` on text with a `*` in the value become (NOT) ILIKE with the rest escaped. */
-const compileCompare = ({ field, op, value }) => {
+/** `=` and `!=` on text with a `*` in the value: a pattern, which compiles to (NOT) ILIKE. */
+const isPatternCompare = ({ field, op, value }) => field.type === "text" && (op === "=" || op === "!=") && value.kind === "string" && value.value.includes("*");
+/** A pattern becomes (NOT) ILIKE with the rest escaped. */
+const compileCompare = (node) => {
+	const { field, op, value } = node;
 	const col = columnRef(field);
-	if (field.type === "text" && (op === "=" || op === "!=") && value.kind === "string" && value.value.includes("*")) {
+	if (isPatternCompare(node) && value.kind === "string") {
 		const pattern = wildcardPattern(value.value);
 		return op === "=" ? sql`${col} ilike ${pattern}` : sql`${col} not ilike ${pattern}`;
 	}
@@ -22,4 +25,4 @@ const compileCompare = ({ field, op, value }) => {
 };
 const compileIn = ({ field, values }) => sql`${columnRef(field)} in (${sql.join(values.map((value) => paramFor(value, field)))})`;
 //#endregion
-export { compileCompare, compileIn };
+export { compileCompare, compileIn, isPatternCompare };

package/dist/explain.js

@@ -5,6 +5,5 @@ const explainPlan = async (db, compiled) => {
 	const output = (await db.executeQuery(CompiledQuery.raw(`EXPLAIN (VERBOSE, FORMAT JSON) ${compiled.sql}`, [...compiled.parameters]))).rows[0]["QUERY PLAN"];
 	return (typeof output === "string" ? JSON.parse(output) : output)[0].Plan;
 };
-const walkPlan = (node) => [node, ...(node.Plans ?? []).flatMap(walkPlan)];
 //#endregion
-export { explainPlan, walkPlan };
+export { explainPlan };

package/dist/index-coverage.d.ts

@@ -0,0 +1,34 @@
+import { Scope } from "./scope.js";
+import { TableIndexes } from "./table-indexes.js";
+import { CheckedFilter, ResourceModel, SortSpec } from "@protobase/schema";
+//#region src/index-coverage.d.ts
+/**
+ * The kind of index a column needs for how it is read: `btree` for comparisons, lists of values, null checks and sorts;
+ * `trigram` (pg_trgm, GIN or GiST) for patterns, regex, `similar` and text search; `digits` a trigram index on the
+ * column's digits (`regexp_replace(column, '[^0-9]', '', 'g')`) for a `digitsEnd` search field; `gin` for JSON.
+ */
+export type IndexNeed = 'btree' | 'trigram' | 'digits' | 'gin';
+/** Where a column is read: the caller's filter, the access rule's row filter, the sort, or a facet's grouping. */
+export type ColumnUse = 'filter' | 'access' | 'sort' | 'facet';
+/** A column read without an index that serves it, and a statement that adds one. */
+export type UncoveredColumn = {
+  field: string;
+  column: string;
+  use: ColumnUse;
+  need: IndexNeed;
+  suggestion: string;
+};
+/** What a read does with the table: the filter it keeps rows by, the order it walks them in, a field it groups by. */
+export type CoverageQuery = {
+  filter?: CheckedFilter;
+  sort?: SortSpec;
+  groupBy?: string;
+};
+/**
+ * The columns a read of the resource would read without an index serving them; empty when every one is served. The
+ * filter's and the access row filter's columns each need an index of the kind their operator uses, the sort's first
+ * field and a grouped field a btree index. Tenant, soft delete and `=` terms pin columns, so a later key of a btree
+ * index serves under them. A plain view or a foreign table has no indexes of its own and is not judged.
+ */
+export declare const uncoveredColumns: (model: ResourceModel, table: TableIndexes, query: CoverageQuery, scope?: Scope) => UncoveredColumn[];
+//#endregion

package/dist/index-coverage.js

@@ -0,0 +1,106 @@
+import { isPatternCompare } from "./compile-compare.js";
+import { fullModelOf } from "./scope.js";
+//#region src/index-coverage.ts
+const trigramClasses = ["gin_trgm_ops", "gist_trgm_ops"];
+const quoted = (name) => `"${name.replaceAll("\"", "\"\"")}"`;
+const tableName = (model) => {
+	const { schema, name } = model.table;
+	return schema ? `${quoted(schema)}.${quoted(name)}` : quoted(name);
+};
+const suggestion = (model, column, need) => {
+	const table = tableName(model);
+	if (need === "trigram") return `create index on ${table} using gin (${quoted(column)} gin_trgm_ops)`;
+	if (need === "digits") return `create index on ${table} using gin ((regexp_replace(${quoted(column)}, '[^0-9]', '', 'g')) gin_trgm_ops)`;
+	if (need === "gin") return `create index on ${table} using gin (${quoted(column)})`;
+	return `create index on ${table} (${quoted(column)})`;
+};
+/** The columns a filter pins to one value or a list of values in every row it keeps: `=`, `in` and `is null` under AND. */
+const pinnedColumns = (filter) => {
+	if (!filter) return [];
+	if (filter.kind === "and") return filter.args.flatMap(pinnedColumns);
+	if (filter.kind === "compare") return filter.op === "=" && !isPatternCompare(filter) ? [filter.field.column] : [];
+	if (filter.kind === "in" || filter.kind === "isNull") return [filter.field.column];
+	return [];
+};
+const btreeServes = (index, column, pinned) => {
+	if (index.method !== "btree" || index.partial) return false;
+	const position = index.keys.findIndex((key) => key.column === column);
+	return position >= 0 && index.keys.slice(0, position).every((key) => key.column !== null && pinned.has(key.column));
+};
+const mentions = (definition, column) => {
+	const name = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return new RegExp(`(^|[^\\w"])("${name}"|${name})([^\\w"]|$)`).test(definition);
+};
+const serves = (index, column, need, pinned) => {
+	if (index.partial) return false;
+	switch (need) {
+		case "btree": return btreeServes(index, column, pinned);
+		case "trigram": return index.keys.some((key) => key.column === column && trigramClasses.includes(key.opclass));
+		case "digits": return index.keys.some((key) => key.column === null && trigramClasses.includes(key.opclass) && key.definition.includes("regexp_replace") && mentions(key.definition, column));
+		case "gin": return index.method === "gin" && index.keys.some((key) => key.column === column);
+	}
+};
+const need = (ctx, field, kind, pinned, also) => ctx.indexes.some((index) => serves(index, field.column, kind, pinned) || also !== void 0 && serves(index, field.column, also, pinned)) ? [] : [{
+	field: field.name,
+	column: field.column,
+	use: ctx.use,
+	need: kind,
+	suggestion: suggestion(ctx.model, field.column, kind)
+}];
+const compareNeed = (ctx, node, pinned) => {
+	if (isPatternCompare(node)) return need(ctx, node.field, "trigram", pinned);
+	return need(ctx, node.field, "btree", pinned, node.op === "=" && node.field.type === "text" ? "trigram" : void 0);
+};
+const searchNeeds = (ctx, pinned) => (ctx.model.search ?? []).flatMap((name) => {
+	const field = ctx.model.fields[name];
+	if (!field) throw new Error(`Not implemented: search field "${name}" is not a field of "${ctx.model.name}", which the model builder refuses`);
+	return need(ctx, field, ctx.model.searchMatch?.[name] === "digitsEnd" ? "digits" : "trigram", pinned);
+});
+const walk = (ctx, node, pinned, negated) => {
+	switch (node.kind) {
+		case "and": return node.args.flatMap((arg) => walk(ctx, arg, pinned, negated));
+		case "or": return node.args.flatMap((arg) => walk(ctx, arg, negated ? pinned : /* @__PURE__ */ new Set([...pinned, ...pinnedColumns(arg)]), negated));
+		case "not": return walk(ctx, node.arg, pinned, true);
+		case "compare": return compareNeed(ctx, node, pinned);
+		case "in":
+		case "isNull": return need(ctx, node.field, "btree", pinned);
+		case "present": return need(ctx, node.field, node.field.type === "json" ? "gin" : "btree", pinned);
+		case "has": return need(ctx, node.field, "gin", pinned);
+		case "regex":
+		case "similar": return need(ctx, node.field, "trigram", pinned);
+		case "search": return searchNeeds(ctx, pinned);
+	}
+};
+const unique = (columns) => columns.filter((column, i) => columns.findIndex((other) => other.column === column.column && other.need === column.need) === i);
+/**
+* The columns a read of the resource would read without an index serving them; empty when every one is served. The
+* filter's and the access row filter's columns each need an index of the kind their operator uses, the sort's first
+* field and a grouped field a btree index. Tenant, soft delete and `=` terms pin columns, so a later key of a btree
+* index serves under them. A plain view or a foreign table has no indexes of its own and is not judged.
+*/
+const uncoveredColumns = (model, table, query, scope) => {
+	if (table.kind !== "table") return [];
+	const full = fullModelOf(model, scope);
+	const columnOf = (name) => name && Object.hasOwn(full.fields, name) ? [full.fields[name].column] : name ? [name] : [];
+	const pinned = /* @__PURE__ */ new Set([
+		...columnOf(full.tenant ?? model.tenant),
+		...columnOf(full.softDelete ?? model.softDelete),
+		...pinnedColumns(scope?.rowFilter),
+		...pinnedColumns(query.filter)
+	]);
+	const ctx = (use, of = model) => ({
+		model: of,
+		indexes: table.indexes,
+		use
+	});
+	const sortField = query.sort?.[0] && model.fields[query.sort[0][0]];
+	const groupField = query.groupBy !== void 0 ? model.fields[query.groupBy] : void 0;
+	return unique([
+		...scope?.rowFilter ? walk(ctx("access", full), scope.rowFilter, pinned, false) : [],
+		...query.filter ? walk(ctx("filter"), query.filter, pinned, false) : [],
+		...sortField ? need(ctx("sort"), sortField, "btree", pinned) : [],
+		...groupField ? need(ctx("facet"), groupField, "btree", pinned) : []
+	]);
+};
+//#endregion
+export { uncoveredColumns };

package/dist/index.d.ts

@@ -10,8 +10,10 @@ import { FacetCount, FacetOptions, facetCounts } from "./facet-counts.js";
 import { Granularity, SeriesOptions, SeriesPoint, series } from "./series.js";
 import { Histogram, HistogramBucket, histogram } from "./histogram.js";
 import { MapBounds, MapBucket, MapBuckets, MapPosition, cellsAcross, checkMapPosition, mapBuckets, mapCellPixels, maxMapBuckets, maxMapZoom } from "./map-buckets.js";
+import { withoutField } from "./prune-filter.js";
 import { ScrollMode, scrollMode } from "./scroll-mode.js";
-import { GuardFinding, GuardOptions, GuardReport, explainGuard } from "./explain-guard.js";
+import { IndexKey, TableIndex, TableIndexes, tableIndexes } from "./table-indexes.js";
+import { ColumnUse, CoverageQuery, IndexNeed, UncoveredColumn, uncoveredColumns } from "./index-coverage.js";
 import { QueryError, QueryErrorCode } from "./fields/query-error.js";
 import { SeekRequest, seekToPosition } from "./seek-to-position.js";
-export { type CompileOptions, type Db, type ExtraColumn, type FacetCount, type FacetOptions, type Granularity, type GuardFinding, type GuardOptions, type GuardReport, type Histogram, type HistogramBucket, type ListOptions, type ListPlan, type ListResult, type MapBounds, type MapBucket, type MapBuckets, type MapPosition, QueryError, type QueryErrorCode, type RelatedRowsQuery, type Scope, type ScrollMode, type SeekRequest, type SeriesOptions, type SeriesPoint, buildListQuery, buildRelatedRows, cellsAcross, checkMapPosition, compileFilter, countUpTo, estimateCount, exactCount, explainGuard, facetCounts, histogram, listQuery, mapBuckets, mapCellPixels, maxLimit, maxMapBuckets, maxMapZoom, relatedRows, scrollMode, seekToPosition, series };
+export { type ColumnUse, type CompileOptions, type CoverageQuery, type Db, type ExtraColumn, type FacetCount, type FacetOptions, type Granularity, type Histogram, type HistogramBucket, type IndexKey, type IndexNeed, type ListOptions, type ListPlan, type ListResult, type MapBounds, type MapBucket, type MapBuckets, type MapPosition, QueryError, type QueryErrorCode, type RelatedRowsQuery, type Scope, type ScrollMode, type SeekRequest, type SeriesOptions, type SeriesPoint, type TableIndex, type TableIndexes, type UncoveredColumn, buildListQuery, buildRelatedRows, cellsAcross, checkMapPosition, compileFilter, countUpTo, estimateCount, exactCount, facetCounts, histogram, listQuery, mapBuckets, mapCellPixels, maxLimit, maxMapBuckets, maxMapZoom, relatedRows, scrollMode, seekToPosition, series, tableIndexes, uncoveredColumns, withoutField };

package/dist/index.js

@@ -5,11 +5,13 @@ import { buildRelatedRows, relatedRows } from "./related-rows.js";
 import { estimateCount } from "./estimate-count.js";
 import { exactCount } from "./exact-count.js";
 import { countUpTo } from "./count-up-to.js";
+import { withoutField } from "./prune-filter.js";
 import { facetCounts } from "./facet-counts.js";
 import { series } from "./series.js";
 import { histogram } from "./histogram.js";
 import { cellsAcross, checkMapPosition, mapBuckets, mapCellPixels, maxMapBuckets, maxMapZoom } from "./map-buckets.js";
 import { scrollMode } from "./scroll-mode.js";
-import { explainGuard } from "./explain-guard.js";
+import { tableIndexes } from "./table-indexes.js";
+import { uncoveredColumns } from "./index-coverage.js";
 import { seekToPosition } from "./seek-to-position.js";
-export { QueryError, buildListQuery, buildRelatedRows, cellsAcross, checkMapPosition, compileFilter, countUpTo, estimateCount, exactCount, explainGuard, facetCounts, histogram, listQuery, mapBuckets, mapCellPixels, maxLimit, maxMapBuckets, maxMapZoom, relatedRows, scrollMode, seekToPosition, series };
+export { QueryError, buildListQuery, buildRelatedRows, cellsAcross, checkMapPosition, compileFilter, countUpTo, estimateCount, exactCount, facetCounts, histogram, listQuery, mapBuckets, mapCellPixels, maxLimit, maxMapBuckets, maxMapZoom, relatedRows, scrollMode, seekToPosition, series, tableIndexes, uncoveredColumns, withoutField };

package/dist/prune-filter.d.ts

@@ -0,0 +1,5 @@
+import { CheckedFilter } from "@protobase/schema";
+//#region src/prune-filter.d.ts
+/** Removes every condition on `field`, dropping groups that end up empty (disjunctive faceting). Text search stays. */
+export declare const withoutField: (node: CheckedFilter | undefined, field: string) => CheckedFilter | undefined;
+//#endregion

package/dist/table-indexes.d.ts

@@ -0,0 +1,28 @@
+import { Db } from "./db.js";
+import { ResourceModel } from "@protobase/schema";
+//#region src/table-indexes.d.ts
+/** One key of an index: its column, or null for an expression, which `definition` spells out. */
+export type IndexKey = {
+  column: string | null;
+  definition: string;
+  opclass: string;
+};
+/** A valid index of a table: its access method (`btree`, `gin`, ...), whether it has a WHERE, and its keys in order. */
+export type TableIndex = {
+  name: string;
+  method: string;
+  partial: boolean;
+  keys: IndexKey[];
+};
+/**
+ * What the database holds for a resource's table: `table` for anything that carries indexes (a table, a partitioned
+ * table or a materialized view), `other` for a plain view or a foreign table, whose indexes live elsewhere, and
+ * `missing` when the name resolves to nothing.
+ */
+export type TableIndexes = {
+  kind: 'table' | 'other' | 'missing';
+  indexes: TableIndex[];
+};
+/** Reads the valid indexes of the resource's table from the catalog, key by key (INCLUDE columns are no keys). */
+export declare const tableIndexes: (db: Db, model: ResourceModel) => Promise<TableIndexes>;
+//#endregion

package/dist/table-indexes.js

@@ -0,0 +1,58 @@
+import { quoteIdent } from "./db.js";
+import { sql } from "kysely";
+//#region src/table-indexes.ts
+const qualifiedName = (model) => {
+	const { schema, name } = model.table;
+	return schema ? `${quoteIdent(schema)}.${quoteIdent(name)}` : quoteIdent(name);
+};
+const indexedKinds = [
+	"r",
+	"p",
+	"m"
+];
+/** Reads the valid indexes of the resource's table from the catalog, key by key (INCLUDE columns are no keys). */
+const tableIndexes = async (db, model) => {
+	const qualified = qualifiedName(model);
+	const relkind = (await sql`select relkind from pg_class where oid = to_regclass(${qualified})`.execute(db)).rows[0]?.relkind;
+	if (relkind === void 0) return {
+		kind: "missing",
+		indexes: []
+	};
+	if (!indexedKinds.includes(relkind)) return {
+		kind: "other",
+		indexes: []
+	};
+	const keys = await sql`
+    select ic.relname as name, am.amname as method, (i.indpred is not null) as partial, a.attname::text as column,
+      pg_get_indexdef(i.indexrelid, k.ord::int, true) as definition, oc.opcname::text as opclass
+    from pg_index i
+    join pg_class ic on ic.oid = i.indexrelid
+    join pg_am am on am.oid = ic.relam
+    cross join lateral unnest(i.indkey::int2[], i.indclass::oid[]) with ordinality k(attnum, opclass, ord)
+    left join pg_attribute a on a.attrelid = i.indrelid and a.attnum = k.attnum and k.attnum > 0
+    join pg_opclass oc on oc.oid = k.opclass
+    where i.indrelid = to_regclass(${qualified}) and i.indisvalid and i.indisready and k.ord <= i.indnkeyatts
+    order by ic.relname, k.ord
+  `.execute(db);
+	const byName = /* @__PURE__ */ new Map();
+	for (const { name, method, partial, column, definition, opclass } of keys.rows) {
+		const index = byName.get(name) ?? {
+			name,
+			method,
+			partial,
+			keys: []
+		};
+		index.keys.push({
+			column,
+			definition,
+			opclass
+		});
+		byName.set(name, index);
+	}
+	return {
+		kind: "table",
+		indexes: [...byName.values()]
+	};
+};
+//#endregion
+export { tableIndexes };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/query",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "Kysely list, filter, count and aggregate queries over Protobase resources.",
   "license": "Apache-2.0",
   "repository": {
@@ -29,7 +29,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@protobase/schema": "0.17.0",
+    "@protobase/schema": "0.18.0",
     "kysely": "^0.29.6"
   }
 }

package/dist/explain-guard.d.ts

@@ -1,21 +0,0 @@
-import { Db } from "./db.js";
-import { Compilable, CompiledQuery } from "kysely";
-//#region src/explain-guard.d.ts
-export type GuardOptions = {
-  /** A Seq Scan on a table with more rows than this is a finding. */
-  seqScanRows?: number;
-  /** A Sort over more rows than this is a finding. */
-  sortRows?: number;
-};
-export type GuardFinding = {
-  kind: 'seqScan' | 'sort';
-  relation?: string;
-  rows: number;
-};
-export type GuardReport = {
-  ok: boolean;
-  findings: GuardFinding[];
-};
-/** Plans the query (never executes it) and reports expensive nodes (Seq Scan and Parallel Seq Scan). Pass a Kysely query or a compiled one. */
-export declare const explainGuard: (db: Db, query: Compilable<unknown> | CompiledQuery, options?: GuardOptions) => Promise<GuardReport>;
-//#endregion

package/dist/explain-guard.js

@@ -1,42 +0,0 @@
-import { quoteIdent } from "./db.js";
-import { explainPlan, walkPlan } from "./explain.js";
-import { toNumber } from "./numbers.js";
-import { sql } from "kysely";
-//#region src/explain-guard.ts
-const relationRows = async (db, schema, relation, planRows) => {
-	const qualified = schema ? `${quoteIdent(schema)}.${quoteIdent(relation)}` : quoteIdent(relation);
-	const result = await sql`
-    select reltuples::float8 as reltuples from pg_class where oid = to_regclass(${qualified})
-  `.execute(db);
-	const reltuples = toNumber(result.rows[0]?.reltuples ?? -1);
-	return reltuples >= 0 ? reltuples : planRows;
-};
-/** Plans the query (never executes it) and reports expensive nodes (Seq Scan and Parallel Seq Scan). Pass a Kysely query or a compiled one. */
-const explainGuard = async (db, query, options = {}) => {
-	const { seqScanRows = 1e4, sortRows = 1e5 } = options;
-	const compiled = "sql" in query ? query : query.compile();
-	const nodes = walkPlan(await explainPlan(db, compiled));
-	const findings = [];
-	for (const node of nodes) {
-		const type = node["Node Type"];
-		const relation = node["Relation Name"];
-		if (type.endsWith("Seq Scan") && relation) {
-			const rows = await relationRows(db, node.Schema, relation, node["Plan Rows"]);
-			if (rows > seqScanRows) findings.push({
-				kind: "seqScan",
-				relation,
-				rows
-			});
-		}
-		if ((type === "Sort" || type === "Incremental Sort") && node["Plan Rows"] > sortRows) findings.push({
-			kind: "sort",
-			rows: node["Plan Rows"]
-		});
-	}
-	return {
-		ok: findings.length === 0,
-		findings
-	};
-};
-//#endregion
-export { explainGuard };