@protobase/query 0.12.22 → 0.13.3

package/dist/count-up-to.d.ts

@@ -0,0 +1,7 @@
+import { Db } from "./db.js";
+import { Scope } from "./scope.js";
+import { CheckedFilter, ResourceModel } from "@protobase/schema";
+//#region src/count-up-to.d.ts
+/** How many rows match, counting no further than `limit`: enough to tell a few rows from many without a full count. */
+export declare const countUpTo: (db: Db, model: ResourceModel, limit: number, filter?: CheckedFilter, scope?: Scope) => Promise<number>;
+//#endregion

package/dist/count-up-to.js

@@ -0,0 +1,14 @@
+import { baseQuery } from "./base-query.js";
+import { assertFilterRequirements } from "./filter-requirements.js";
+import { toNumber } from "./numbers.js";
+import { sql } from "kysely";
+//#region src/count-up-to.ts
+/** How many rows match, counting no further than `limit`: enough to tell a few rows from many without a full count. */
+const countUpTo = async (db, model, limit, filter, scope) => {
+	await assertFilterRequirements(db, filter, scope);
+	const found = baseQuery(db, model, filter, scope).select(sql`1`.as("one")).limit(limit).as("found");
+	const row = await db.selectFrom(found).select(sql`count(*)::text`.as("n")).executeTakeFirstOrThrow();
+	return toNumber(row.n);
+};
+//#endregion
+export { countUpTo };

package/dist/cursor.js

@@ -1,37 +1,43 @@
 import { QueryError } from "./fields/query-error.js";
+import { filterFingerprint } from "./filter-fingerprint.js";
 import { sortSignature } from "./sort.js";
 //#region src/cursor.ts
-const invalidCursor = () => new QueryError("invalid_cursor", "Cursor is malformed or does not match the requested sort");
+const invalidCursor = () => new QueryError("invalid_cursor", "Cursor is malformed or does not match the requested sort and filter");
 const isStringList = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
 const isCellList = (value) => Array.isArray(value) && value.every((item) => item === null || typeof item === "string");
 /**
 * Opaque cursor: the sort signature plus the row's sort values as text (so timestamps and bigints round-trip exactly,
-* null for NULL values). The JSON is UTF-8 encoded before base64url, so any text value survives.
+* null for NULL values), and a fingerprint of the filter when there is one, so a cursor only continues the list it
+* came from (AIP-158). The JSON is UTF-8 encoded before base64url, so any text value survives.
 * `values` may be a prefix of the sort keys; such a boundary cursor is produced by scroll anchors.
 */
-const encodeCursor = (keys, values) => {
+const encodeCursor = (keys, values, filter) => {
+	const fingerprint = filterFingerprint(filter);
 	const payload = {
 		s: sortSignature(keys).slice(0, values.length),
-		v: values
+		v: values,
+		...fingerprint && { f: fingerprint }
 	};
 	return toBase64Url(JSON.stringify(payload));
 };
-const decodeCursor = (cursor, keys) => {
+const decodeCursor = (cursor, keys, filter) => {
 	const payload = parse(cursor);
 	const signature = sortSignature(keys);
 	const prefixOk = payload.s.length === payload.v.length && payload.s.every((part, i) => part === signature[i]);
 	const lengthOk = payload.v.length === keys.length || payload.v.length === 1;
-	if (!prefixOk || !lengthOk) throw invalidCursor();
+	const filterOk = (payload.f ?? "") === filterFingerprint(filter);
+	if (!prefixOk || !lengthOk || !filterOk) throw invalidCursor();
 	return payload.v;
 };
 const parse = (cursor) => {
 	const payload = parseJson(cursor);
 	if (typeof payload !== "object" || payload === null) throw invalidCursor();
-	const { s, v } = payload;
-	if (!isStringList(s) || !isCellList(v)) throw invalidCursor();
+	const { s, v, f } = payload;
+	if (!isStringList(s) || !isCellList(v) || f !== void 0 && typeof f !== "string") throw invalidCursor();
 	return {
 		s,
-		v
+		v,
+		...f !== void 0 && { f }
 	};
 };
 const toBase64Url = (text) => btoa(String.fromCharCode(...new TextEncoder().encode(text))).replaceAll("+", "-").replaceAll("/", "_").replaceAll("=", "");

package/dist/filter-fingerprint.js

@@ -0,0 +1,24 @@
+//#region src/filter-fingerprint.ts
+/** The filter's structure without source spans, each field named by its column, so equal filters match however they were written. */
+const canonical = (filter) => JSON.stringify(filter, (key, value) => {
+	if (key === "span") return void 0;
+	if (key === "field") return value.column;
+	return value;
+});
+/** cyrb53, a fast 53-bit string hash; the fingerprint guards against mistakes, not forgery. */
+const hash = (text) => {
+	let h1 = 3735928559;
+	let h2 = 1103547991;
+	for (let i = 0; i < text.length; i++) {
+		const code = text.charCodeAt(i);
+		h1 = Math.imul(h1 ^ code, 2654435761);
+		h2 = Math.imul(h2 ^ code, 1597334677);
+	}
+	h1 = Math.imul(h1 ^ h1 >>> 16, 2246822507) ^ Math.imul(h2 ^ h2 >>> 13, 3266489909);
+	h2 = Math.imul(h2 ^ h2 >>> 16, 2246822507) ^ Math.imul(h1 ^ h1 >>> 13, 3266489909);
+	return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36);
+};
+/** A short fingerprint of the filter a cursor was issued for; empty without a filter. */
+const filterFingerprint = (filter) => filter ? hash(canonical(filter)) : "";
+//#endregion
+export { filterFingerprint };

package/dist/index.d.ts

@@ -4,11 +4,13 @@ import { Scope } from "./scope.js";
 import { ExtraColumn, ListOptions, ListPlan, ListResult, buildListQuery, listQuery, maxLimit } from "./list-query.js";
 import { estimateCount } from "./estimate-count.js";
 import { exactCount } from "./exact-count.js";
+import { countUpTo } from "./count-up-to.js";
 import { FacetCount, FacetOptions, facetCounts } from "./facet-counts.js";
 import { Granularity, SeriesOptions, SeriesPoint, series } from "./series.js";
 import { Histogram, HistogramBucket, histogram } from "./histogram.js";
+import { MapBounds, MapBucket, MapBuckets, MapPosition, cellsAcross, checkMapPosition, mapBuckets, mapCellPixels, maxMapBuckets, maxMapZoom } from "./map-buckets.js";
 import { ScrollMode, scrollMode } from "./scroll-mode.js";
 import { GuardFinding, GuardOptions, GuardReport, explainGuard } from "./explain-guard.js";
 import { QueryError, QueryErrorCode } from "./fields/query-error.js";
 import { SeekRequest, seekToPosition } from "./seek-to-position.js";
-export { type CompileOptions, type Db, type ExtraColumn, type FacetCount, type FacetOptions, type Granularity, type GuardFinding, type GuardOptions, type GuardReport, type Histogram, type HistogramBucket, type ListOptions, type ListPlan, type ListResult, QueryError, type QueryErrorCode, type Scope, type ScrollMode, type SeekRequest, type SeriesOptions, type SeriesPoint, buildListQuery, compileFilter, estimateCount, exactCount, explainGuard, facetCounts, histogram, listQuery, maxLimit, scrollMode, seekToPosition, series };
+export { type CompileOptions, type Db, type ExtraColumn, type FacetCount, type FacetOptions, type Granularity, type GuardFinding, type GuardOptions, type GuardReport, type Histogram, type HistogramBucket, type ListOptions, type ListPlan, type ListResult, type MapBounds, type MapBucket, type MapBuckets, type MapPosition, QueryError, type QueryErrorCode, type Scope, type ScrollMode, type SeekRequest, type SeriesOptions, type SeriesPoint, buildListQuery, cellsAcross, checkMapPosition, compileFilter, countUpTo, estimateCount, exactCount, explainGuard, facetCounts, histogram, listQuery, mapBuckets, mapCellPixels, maxLimit, maxMapBuckets, maxMapZoom, scrollMode, seekToPosition, series };

package/dist/index.js

@@ -3,10 +3,12 @@ import { compileFilter } from "./compile-filter.js";
 import { buildListQuery, listQuery, maxLimit } from "./list-query.js";
 import { estimateCount } from "./estimate-count.js";
 import { exactCount } from "./exact-count.js";
+import { countUpTo } from "./count-up-to.js";
 import { facetCounts } from "./facet-counts.js";
 import { series } from "./series.js";
 import { histogram } from "./histogram.js";
+import { cellsAcross, checkMapPosition, mapBuckets, mapCellPixels, maxMapBuckets, maxMapZoom } from "./map-buckets.js";
 import { scrollMode } from "./scroll-mode.js";
 import { explainGuard } from "./explain-guard.js";
 import { seekToPosition } from "./seek-to-position.js";
-export { QueryError, buildListQuery, compileFilter, estimateCount, exactCount, explainGuard, facetCounts, histogram, listQuery, maxLimit, scrollMode, seekToPosition, series };
+export { QueryError, buildListQuery, cellsAcross, checkMapPosition, compileFilter, countUpTo, estimateCount, exactCount, explainGuard, facetCounts, histogram, listQuery, mapBuckets, mapCellPixels, maxLimit, maxMapBuckets, maxMapZoom, scrollMode, seekToPosition, series };

package/dist/list-query.js

@@ -33,7 +33,7 @@ const buildListQuery = async (db, model, query, scope, options = {}) => {
 	const limit = clampLimit(query.limit);
 	await assertFilterRequirements(db, query.filter, scope);
 	const keys = effectiveSort(model, query.sort);
-	const cursorValues = query.cursor === void 0 ? void 0 : decodeCursor(query.cursor, keys);
+	const cursorValues = query.cursor === void 0 ? void 0 : decodeCursor(query.cursor, keys, query.filter);
 	return {
 		query: selectPage(db, model, query, scope, keys, cursorValues, options),
 		limit,
@@ -48,7 +48,7 @@ const listQuery = async (db, model, query, scope, options = {}) => {
 	const more = fetched.length > limit;
 	const page = more ? fetched.slice(0, limit) : fetched;
 	if (reverse) page.reverse();
-	const cursorOf = (row) => encodeCursor(keys, keys.map((_, i) => row[keyAlias(i)] === null ? null : String(row[keyAlias(i)])));
+	const cursorOf = (row) => encodeCursor(keys, keys.map((_, i) => row[keyAlias(i)] === null ? null : String(row[keyAlias(i)])), query.filter);
 	const first = page[0];
 	const last = page.at(-1);
 	return {

package/dist/map-buckets.d.ts

@@ -0,0 +1,48 @@
+import { Db } from "./db.js";
+import { Scope } from "./scope.js";
+import { CheckedFilter, ResourceModel } from "@protobase/schema";
+//#region src/map-buckets.d.ts
+/** MapLibre's zoom levels, from the whole world at 0 to a building at 22. */
+export declare const maxMapZoom = 22;
+/** About how many pixels across a bucket's cell is on the map. */
+export declare const mapCellPixels = 64;
+/** The most buckets one request returns, the fullest first; a large screen's map holds fewer cells. */
+export declare const maxMapBuckets = 1000;
+/** The fields that hold a record's position, in decimal degrees. */
+export type MapPosition = {
+  latitude: string;
+  longitude: string;
+};
+/** The corners of the records a bucket stands for, in decimal degrees. */
+export type MapBounds = {
+  south: number;
+  west: number;
+  north: number;
+  east: number;
+};
+/** The records in one cell of the map's grid: how many, their mean position, and the bounds around them. */
+export type MapBucket = {
+  count: number;
+  latitude: number;
+  longitude: number;
+  bounds: MapBounds;
+};
+/** The buckets, and how many records they hold together, the ones beyond `maxMapBuckets` included. */
+export type MapBuckets = {
+  total: number;
+  buckets: MapBucket[];
+};
+/** How many cells span the world's width, and its Web Mercator height, at a zoom. */
+export declare const cellsAcross: (zoom: number) => number;
+/** The position's fields, refused unless each is a filterable number of degrees. */
+export declare const checkMapPosition: (model: ResourceModel, position: MapPosition) => {
+  latitude: import("@protobase/schema").FieldModel;
+  longitude: import("@protobase/schema").FieldModel;
+};
+/**
+ * The matching records with a position, counted per cell of a grid that is square on the map at `zoom` (Web
+ * Mercator, as MapLibre draws it) and about `mapCellPixels` across. The grid starts at the antimeridian and the north
+ * edge, so a cell stays put as the map pans. Postgres groups the rows, so the filter can use the position's indexes.
+ */
+export declare const mapBuckets: (db: Db, model: ResourceModel, position: MapPosition, zoom: number, filter?: CheckedFilter, scope?: Scope) => Promise<MapBuckets>;
+//#endregion

package/dist/map-buckets.js

@@ -0,0 +1,75 @@
+import { QueryError } from "./fields/query-error.js";
+import { columnRef, requireField } from "./columns.js";
+import { baseQuery } from "./base-query.js";
+import { assertFilterRequirements } from "./filter-requirements.js";
+import { toNumber } from "./numbers.js";
+import { sql } from "kysely";
+//#region src/map-buckets.ts
+/** MapLibre's zoom levels, from the whole world at 0 to a building at 22. */
+const maxMapZoom = 22;
+/** About how many pixels across a bucket's cell is on the map. */
+const mapCellPixels = 64;
+/** The most buckets one request returns, the fullest first; a large screen's map holds fewer cells. */
+const maxMapBuckets = 1e3;
+const worldPixels = 512;
+const maxLatitude = 85.051129;
+const coordinateTypes = [
+	"integer",
+	"bigint",
+	"decimal"
+];
+/** How many cells span the world's width, and its Web Mercator height, at a zoom. */
+const cellsAcross = (zoom) => worldPixels * 2 ** zoom / 64;
+const coordinate = (model, name) => {
+	const target = requireField(model, name, "filterable");
+	if (!coordinateTypes.includes(target.type)) throw new QueryError("unsupported_field_type", `A position needs number fields, "${name}" is ${target.type}`);
+	return target;
+};
+/** The position's fields, refused unless each is a filterable number of degrees. */
+const checkMapPosition = (model, position) => ({
+	latitude: coordinate(model, position.latitude),
+	longitude: coordinate(model, position.longitude)
+});
+/**
+* The matching records with a position, counted per cell of a grid that is square on the map at `zoom` (Web
+* Mercator, as MapLibre draws it) and about `mapCellPixels` across. The grid starts at the antimeridian and the north
+* edge, so a cell stays put as the map pans. Postgres groups the rows, so the filter can use the position's indexes.
+*/
+const mapBuckets = async (db, model, position, zoom, filter, scope) => {
+	const fields = checkMapPosition(model, position);
+	if (!Number.isInteger(zoom) || zoom < 0 || zoom > 22) throw new QueryError("invalid_option", `zoom must be a whole number from 0 to 22`);
+	await assertFilterRequirements(db, filter, scope);
+	const latitude = sql`${columnRef(fields.latitude)}::float8`;
+	const longitude = sql`${columnRef(fields.longitude)}::float8`;
+	const cells = cellsAcross(zoom);
+	const last = cells - 1;
+	const clamped = sql`least(greatest(${latitude}, ${-85.051129}::float8), ${maxLatitude}::float8)`;
+	const column = sql`least(floor((${longitude} + 180) / 360 * ${cells}::float8), ${last}::float8)`;
+	const row = sql`least(floor((1 - ln(tan(pi() / 4 + radians(${clamped}) / 2)) / pi()) / 2 * ${cells}::float8), ${last}::float8)`;
+	const rows = await baseQuery(db, model, filter, scope).where(sql`${columnRef(fields.latitude)} is not null and ${columnRef(fields.longitude)} is not null`).select([
+		sql`count(*)::text`.as("n"),
+		sql`(sum(count(*)) over ())::text`.as("total"),
+		sql`avg(${latitude})::text`.as("latitude"),
+		sql`avg(${longitude})::text`.as("longitude"),
+		sql`min(${latitude})::text`.as("south"),
+		sql`min(${longitude})::text`.as("west"),
+		sql`max(${latitude})::text`.as("north"),
+		sql`max(${longitude})::text`.as("east")
+	]).groupBy([column, row]).orderBy(sql`count(*)`, "desc").limit(maxMapBuckets).execute();
+	return {
+		total: rows.length === 0 ? 0 : toNumber(rows[0].total),
+		buckets: rows.map((bucket) => ({
+			count: toNumber(bucket.n),
+			latitude: toNumber(bucket.latitude),
+			longitude: toNumber(bucket.longitude),
+			bounds: {
+				south: toNumber(bucket.south),
+				west: toNumber(bucket.west),
+				north: toNumber(bucket.north),
+				east: toNumber(bucket.east)
+			}
+		}))
+	};
+};
+//#endregion
+export { cellsAcross, checkMapPosition, mapBuckets, mapCellPixels, maxMapBuckets, maxMapZoom };

package/dist/scroll-anchors.js

@@ -60,7 +60,7 @@ const scrollAnchors = async (db, model, sortField, options = {}) => {
 		seekToPosition: async (position) => {
 			if (position <= 0) return void 0;
 			const anchor = anchors.findLast((candidate) => candidate.position <= position);
-			return anchor && anchor.position > 0 ? encodeCursor(keys, [anchor.value]) : void 0;
+			return anchor && anchor.position > 0 ? encodeCursor(keys, [anchor.value], filter) : void 0;
 		}
 	};
 };

package/dist/scroll-exact.js

@@ -21,7 +21,7 @@ const exactSeek = async (db, model, keys, scope, filter, position) => {
 	if (position <= 0) return void 0;
 	const row = await db.selectFrom(rankedRows(db, model, keys, scope, filter).as("r")).selectAll().where(sql`r.rn = ${position}`).executeTakeFirst();
 	if (!row) return void 0;
-	return encodeCursor(keys, keys.map((_, i) => row[`k${i}`] === null ? null : String(row[`k${i}`])));
+	return encodeCursor(keys, keys.map((_, i) => row[`k${i}`] === null ? null : String(row[`k${i}`])), filter);
 };
 //#endregion
 export { exactAnchors, exactSeek };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/query",
-  "version": "0.12.22",
+  "version": "0.13.3",
   "description": "Kysely list, filter, count and aggregate queries over Protobase resources.",
   "license": "Apache-2.0",
   "repository": {
@@ -29,7 +29,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@protobase/schema": "0.12.22",
+    "@protobase/schema": "0.13.3",
     "kysely": "^0.29.6"
   }
 }