@protobase/query 0.0.0-stage → 0.1.1

package/dist/scroll-anchors.js

@@ -0,0 +1,68 @@
+import { QueryError } from "./fields/query-error.js";
+import { requireField } from "./columns.js";
+import { effectiveSort } from "./sort.js";
+import { encodeCursor } from "./cursor.js";
+import { estimateCount } from "./estimate-count.js";
+import { exactCount } from "./exact-count.js";
+import { scrollMode } from "./scroll-mode.js";
+import { ascendingAnchors, descendingAnchors } from "./anchor-positions.js";
+import { exactAnchors, exactSeek } from "./scroll-exact.js";
+import { readColumnStats } from "./scroll-stats.js";
+//#region src/scroll-anchors.ts
+const defaultAnchorCount = 100;
+/**
+* Internal: statistics-mode anchors are table-wide pg_stats values, which may belong to rows the caller cannot see,
+* so no route may return them; use seekToPosition, which only returns visible rows.
+* Scrollbar support for a sort on `sortField` (primary key as tie-breaker, as in listQuery).
+* exact: window-function positions (under 100k rows). statistics: pg_stats histogram, positions are approximate.
+* With a filter, exact mode ranks only matching rows. Statistics mode scales the table-wide positions by the planner's
+* selectivity estimate for the filter (filtered estimate / table estimate), so they are rougher still.
+* With a tenant scope, exact mode is exact within the tenant; statistics mode scales the table-wide histogram to the
+* tenant's row count, which assumes the tenant's values are distributed like the table's.
+* In statistics mode seekToPosition returns a boundary cursor on the leading sort column, which listQuery accepts
+* for any sort that begins with the same field and direction.
+*/
+const scrollAnchors = async (db, model, sortField, options = {}) => {
+	const { direction = "asc", anchorCount = defaultAnchorCount, filter, scope, sort = [[sortField, direction]] } = options;
+	if (!Number.isInteger(anchorCount) || anchorCount < 2) throw new QueryError("invalid_option", "anchorCount must be at least 2");
+	const field = requireField(model, sortField, "sortable");
+	const keys = effectiveSort(model, sort);
+	const mode = scrollMode(await estimateCount(db, model, filter, scope));
+	if (mode === "sampled") throw new Error("Not implemented: sampled scroll mode for tables above 5M rows needs a sampling strategy, statistics are too coarse there");
+	if (mode === "exact") {
+		const totalRows = await exactCount(db, model, filter, scope);
+		const step = Math.max(1, Math.ceil(totalRows / anchorCount));
+		return {
+			mode,
+			totalRows,
+			anchors: await exactAnchors(db, model, keys, step, scope, filter),
+			mostCommon: [],
+			seekToPosition: (position) => exactSeek(db, model, keys, scope, filter, Math.min(position, totalRows - 1))
+		};
+	}
+	const stats = await readColumnStats(db, model, field);
+	const tableRows = await estimateCount(db, model, void 0, scope);
+	const totalRows = filter ? await estimateCount(db, model, filter, scope) : tableRows;
+	const selectivity = tableRows > 0 ? Math.min(1, totalRows / tableRows) : 1;
+	const ascending = ascendingAnchors(stats, tableRows, field.type).map((anchor) => ({
+		...anchor,
+		position: Math.round(anchor.position * selectivity)
+	}));
+	const anchors = direction === "asc" ? ascending : descendingAnchors(ascending, totalRows);
+	return {
+		mode,
+		totalRows,
+		anchors,
+		mostCommon: stats.commonValues.map((value, i) => ({
+			value,
+			frequency: stats.commonFrequencies[i] ?? 0
+		})),
+		seekToPosition: async (position) => {
+			if (position <= 0) return void 0;
+			const anchor = anchors.findLast((candidate) => candidate.position <= position);
+			return anchor && anchor.position > 0 ? encodeCursor(keys, [anchor.value]) : void 0;
+		}
+	};
+};
+//#endregion
+export { scrollAnchors };

package/dist/scroll-exact.js

@@ -0,0 +1,27 @@
+import { columnRef } from "./columns.js";
+import { baseQuery } from "./base-query.js";
+import { encodeCursor } from "./cursor.js";
+import { orderTerm } from "./order.js";
+import { toNumber } from "./numbers.js";
+import { sql } from "kysely";
+//#region src/scroll-exact.ts
+const rankedRows = (db, model, keys, scope, filter) => {
+	const order = sql.join(keys.map((key) => orderTerm(key)));
+	return baseQuery(db, model, filter, scope).select(keys.map((key, i) => sql`${columnRef(key.field)}::text`.as(`k${i}`))).select(sql`row_number() over (order by ${order})`.as("rn"));
+};
+/** Every `step`-th row's leading sort value with its exact 0-based position. */
+const exactAnchors = async (db, model, keys, step, scope, filter) => {
+	return (await db.selectFrom(rankedRows(db, model, keys, scope, filter).as("r")).select([sql`r.k0`.as("value"), sql`r.rn::text`.as("rn")]).where(sql`(r.rn - 1) % ${step} = 0`).orderBy(sql`r.rn`).execute()).map((row) => ({
+		position: toNumber(row.rn) - 1,
+		value: String(row.value)
+	}));
+};
+/** Cursor for the row just before `position`, so a page read after it starts exactly at `position`. */
+const exactSeek = async (db, model, keys, scope, filter, position) => {
+	if (position <= 0) return void 0;
+	const row = await db.selectFrom(rankedRows(db, model, keys, scope, filter).as("r")).selectAll().where(sql`r.rn = ${position}`).executeTakeFirst();
+	if (!row) return void 0;
+	return encodeCursor(keys, keys.map((_, i) => row[`k${i}`] === null ? null : String(row[`k${i}`])));
+};
+//#endregion
+export { exactAnchors, exactSeek };

package/dist/scroll-mode.d.ts

@@ -0,0 +1,4 @@
+//#region src/scroll-mode.d.ts
+export type ScrollMode = 'exact' | 'statistics' | 'sampled';
+export declare const scrollMode: (rows: number) => ScrollMode;
+//#endregion

package/dist/scroll-mode.js

@@ -0,0 +1,10 @@
+//#region src/scroll-mode.ts
+const exactBelowRows = 1e5;
+const statisticsUpToRows = 5e6;
+const scrollMode = (rows) => {
+	if (rows < 1e5) return "exact";
+	if (rows <= 5e6) return "statistics";
+	return "sampled";
+};
+//#endregion
+export { exactBelowRows, scrollMode, statisticsUpToRows };

package/dist/scroll-stats.js

@@ -0,0 +1,27 @@
+import { QueryError } from "./fields/query-error.js";
+import { toNumber } from "./numbers.js";
+import { sql } from "kysely";
+//#region src/scroll-stats.ts
+/** Reads pg_stats for one column; values are cast to text[] so any column type parses uniformly. */
+const readColumnStats = async (db, model, field) => {
+	const row = (await sql`
+    select null_frac::float8 as null_frac,
+      histogram_bounds::text::text[] as bounds,
+      most_common_vals::text::text[] as common_vals,
+      most_common_freqs::float8[] as common_freqs
+    from pg_stats
+    where schemaname = coalesce(${model.table.schema ?? null}::text, current_schema())
+      and tablename = ${model.table.name} and attname = ${field.column}
+    order by inherited
+    limit 1
+  `.execute(db)).rows[0];
+	if (!row?.bounds || row.bounds.length < 2) throw new QueryError("no_statistics", `No histogram for "${field.name}"; run ANALYZE or sort by a higher-cardinality column`);
+	return {
+		nullFraction: toNumber(row.null_frac),
+		bounds: row.bounds,
+		commonValues: row.common_vals ?? [],
+		commonFrequencies: (row.common_freqs ?? []).map(toNumber)
+	};
+};
+//#endregion
+export { readColumnStats };

package/dist/seek-to-position.d.ts

@@ -0,0 +1,20 @@
+import { Db } from "./db.js";
+import { Scope } from "./scope.js";
+import { ListOptions, ListResult } from "./list-query.js";
+import { CheckedFilter, ResourceModel, SortSpec } from "@protobase/schema";
+//#region src/seek-to-position.d.ts
+export type SeekRequest = {
+  /** 0-based row position within the caller's visible rows, in `orderBy` order. */
+  position: number;
+  orderBy: SortSpec;
+  filter?: CheckedFilter;
+  columns?: string[];
+  limit?: number;
+};
+/**
+ * Resolves a scroll position to a page of rows, server-side, with no OFFSET. Anchors are used internally only:
+ * positions are scaled to the caller's scoped estimate, and the page is read with the full scope (tenant, row filter,
+ * user filter), so it holds only visible rows. `options` (extra columns) pass through to the page read. The result carries page tokens for the visible rows, never anchor values.
+ */
+export declare const seekToPosition: (db: Db, model: ResourceModel, request: SeekRequest, scope?: Scope, options?: ListOptions) => Promise<ListResult>;
+//#endregion

package/dist/seek-to-position.js

@@ -0,0 +1,30 @@
+import { QueryError } from "./fields/query-error.js";
+import { listQuery } from "./list-query.js";
+import { scrollAnchors } from "./scroll-anchors.js";
+//#region src/seek-to-position.ts
+const defaultLimit = 50;
+/**
+* Resolves a scroll position to a page of rows, server-side, with no OFFSET. Anchors are used internally only:
+* positions are scaled to the caller's scoped estimate, and the page is read with the full scope (tenant, row filter,
+* user filter), so it holds only visible rows. `options` (extra columns) pass through to the page read. The result carries page tokens for the visible rows, never anchor values.
+*/
+const seekToPosition = async (db, model, request, scope, options = {}) => {
+	const [lead] = request.orderBy;
+	if (!lead) throw new QueryError("invalid_sort", "orderBy needs at least one field");
+	if (!Number.isInteger(request.position) || request.position < 0) throw new QueryError("invalid_option", "position must be a non-negative integer");
+	const cursor = await (await scrollAnchors(db, model, lead[0], {
+		direction: lead[1],
+		sort: request.orderBy,
+		filter: request.filter,
+		scope
+	})).seekToPosition(request.position);
+	return listQuery(db, model, {
+		limit: request.limit ?? defaultLimit,
+		sort: request.orderBy,
+		filter: request.filter,
+		columns: request.columns,
+		cursor
+	}, scope, options);
+};
+//#endregion
+export { seekToPosition };

package/dist/seek.js

@@ -0,0 +1,52 @@
+import { columnRef } from "./columns.js";
+import { flip } from "./sort.js";
+import { sql } from "kysely";
+//#region src/seek.ts
+const compare = (direction, inclusive) => sql.raw(direction === "asc" ? inclusive ? ">=" : ">" : inclusive ? "<=" : "<");
+/**
+* Rows positioned beyond `value` when reading in `direction` (asc puts nulls last, desc puts them first).
+* `inclusive` also keeps rows equal to value; for a null value that means the null rows themselves.
+*/
+const beyond = (key, direction, value, inclusive) => {
+	const column = columnRef(key.field);
+	if (value === null) {
+		if (direction === "asc") return inclusive ? sql`${column} is null` : sql`false`;
+		return inclusive ? sql`true` : sql`${column} is not null`;
+	}
+	const ordered = sql`${column} ${compare(direction, inclusive)} ${value}`;
+	return key.field.nullable && direction === "asc" ? sql`(${ordered} or ${column} is null)` : ordered;
+};
+const same = (key, value) => {
+	const column = columnRef(key.field);
+	return value === null ? sql`${column} is null` : sql`${column} = ${value}`;
+};
+/** Rows from a boundary cursor (leading column only): `after` starts at the value, `before` ends just short of it. */
+const seekBoundary = (key, value, reading) => reading === "after" ? beyond(key, key.direction, value, true) : beyond(key, flip(key.direction), value, false);
+/** Expanded form: (a beyond x) or (a = x and b beyond y) or ... Handles mixed directions and NULL boundaries. */
+const seekExpanded = (keys, values) => {
+	const branches = keys.map((key, i) => {
+		const equal = keys.slice(0, i).map((prior, j) => same(prior, values[j] ?? null));
+		return sql`(${sql.join([...equal, beyond(key, key.direction, values[i] ?? null, false)], sql` and `)})`;
+	});
+	return sql`(${sql.join(branches, sql` or `)})`;
+};
+/**
+* Condition selecting rows strictly after (or before) the cursor row in the requested sort.
+* `keys` carry the user-facing directions; `reading` only picks which side of the cursor to read.
+* Row-value comparison is used when every direction matches and no key can be NULL (it cannot order NULLs).
+*/
+const seekCondition = (keys, values, reading) => {
+	const [first] = keys;
+	if (values.length === 1 && keys.length > 1 && first) return seekBoundary(first, values[0] ?? null, reading);
+	const effective = keys.map((key) => ({
+		...key,
+		direction: reading === "after" ? key.direction : flip(key.direction)
+	}));
+	const uniform = effective.every((key) => key.direction === effective[0]?.direction);
+	const nullable = effective.some((key) => key.field.nullable);
+	if (!uniform || nullable) return seekExpanded(effective, values);
+	const cols = sql.join(effective.map((key) => columnRef(key.field)));
+	return sql`(${cols}) ${compare(effective[0].direction, false)} (${sql.join(values)})`;
+};
+//#endregion
+export { seekCondition };

package/dist/series.d.ts

@@ -0,0 +1,33 @@
+import { Db } from "./db.js";
+import { Scope } from "./scope.js";
+import { CheckedFilter, ResourceModel } from "@protobase/schema";
+//#region src/series.d.ts
+declare const approxMs: {
+  hour: number;
+  day: number;
+  week: number;
+  month: number;
+  year: number;
+};
+export type Granularity = keyof typeof approxMs;
+export type SeriesOptions = {
+  range: {
+    from: string | Date;
+    to: string | Date;
+  };
+  granularity: Granularity;
+  timeZone?: string;
+  filter?: CheckedFilter;
+  scope?: Scope;
+};
+export type SeriesPoint = {
+  bucket: string;
+  count: number;
+};
+/**
+ * Row counts per calendar bucket, in `timeZone`, with empty buckets present. `to` is exclusive.
+ * Buckets are returned as local ISO strings without offset. `date` fields ignore the time zone;
+ * `timestamp` columns without a zone are read in the session time zone.
+ */
+export declare const series: (db: Db, model: ResourceModel, field: string, options: SeriesOptions) => Promise<SeriesPoint[]>;
+//#endregion

package/dist/series.js

@@ -0,0 +1,63 @@
+import { QueryError } from "./fields/query-error.js";
+import { columnRef, requireField } from "./columns.js";
+import { baseQuery } from "./base-query.js";
+import { assertFilterRequirements } from "./filter-requirements.js";
+import { toNumber } from "./numbers.js";
+import { rangeBound } from "./range-bound.js";
+import { sql } from "kysely";
+//#region src/series.ts
+const maxBuckets = 1e4;
+const approxMs = {
+	hour: 36e5,
+	day: 864e5,
+	week: 6048e5,
+	month: 26298e5,
+	year: 315576e5
+};
+const checkTimeZone = (timeZone) => {
+	try {
+		new Intl.DateTimeFormat("en", { timeZone });
+	} catch (error) {
+		if (error instanceof RangeError) throw new QueryError("invalid_option", `Unknown time zone "${timeZone}"`);
+		throw error;
+	}
+};
+/**
+* Row counts per calendar bucket, in `timeZone`, with empty buckets present. `to` is exclusive.
+* Buckets are returned as local ISO strings without offset. `date` fields ignore the time zone;
+* `timestamp` columns without a zone are read in the session time zone.
+*/
+const series = async (db, model, field, options) => {
+	const target = requireField(model, field, "filterable");
+	if (target.type !== "date" && target.type !== "timestamp") throw new QueryError("unsupported_field_type", `Series needs a date or timestamp field, "${field}" is ${target.type}`);
+	if (!Object.hasOwn(approxMs, options.granularity)) throw new QueryError("invalid_option", "Unknown granularity");
+	const timeZone = options.timeZone ?? "UTC";
+	checkTimeZone(timeZone);
+	const from = rangeBound(options.range.from, target);
+	const to = rangeBound(options.range.to, target);
+	const spanMs = new Date(to).getTime() - new Date(from).getTime();
+	if (!(spanMs > 0)) throw new QueryError("invalid_option", "range.to must be after range.from");
+	if (spanMs / approxMs[options.granularity] > maxBuckets) throw new QueryError("invalid_option", `Range spans more than ${maxBuckets} buckets`);
+	await assertFilterRequirements(db, options.filter, options.scope);
+	const col = columnRef(target);
+	const unit = sql.lit(options.granularity);
+	const step = sql`${sql.lit(`1 ${options.granularity}`)}::interval`;
+	const isDate = target.type === "date";
+	const bound = sql.raw(isDate ? "date" : "timestamptz");
+	const local = (value) => isDate ? sql`${value}::timestamp` : sql`(${value}::timestamptz at time zone ${timeZone}::text)`;
+	const laterBucket = sql`date_trunc(${unit}, ${local(to)} - interval '1 microsecond')`;
+	const counts = baseQuery(db, model, options.filter, options.scope).where(sql`${col} >= ${from}::${bound} and ${col} < ${to}::${bound}`).select([sql`date_trunc(${unit}, ${isDate ? sql`${col}::timestamp` : local(col)})`.as("bucket"), sql`count(*)::text`.as("n")]).groupBy(sql`1`);
+	return (await sql`
+    with b as (
+      select generate_series(date_trunc(${unit}, ${local(from)}), ${laterBucket}, ${step}) as bucket
+    ), c as (${counts})
+    select to_char(b.bucket, 'YYYY-MM-DD"T"HH24:MI:SS') as bucket, coalesce(c.n, '0') as n
+    from b left join c on c.bucket = b.bucket
+    order by b.bucket
+  `.execute(db)).rows.map((row) => ({
+		bucket: row.bucket,
+		count: toNumber(row.n)
+	}));
+};
+//#endregion
+export { series };

package/dist/sort.d.ts

@@ -0,0 +1,8 @@
+import { FieldModel, ResourceModel, SortSpec } from "@protobase/schema";
+//#region src/sort.d.ts
+export type Direction = 'asc' | 'desc';
+export type SortKey = {
+  field: FieldModel;
+  direction: Direction;
+};
+//#endregion

package/dist/sort.js

@@ -0,0 +1,33 @@
+import { QueryError } from "./fields/query-error.js";
+import { primaryKeyFields, requireField } from "./columns.js";
+//#region src/sort.ts
+const flip = (direction) => direction === "asc" ? "desc" : "asc";
+const checkDirection = (direction) => {
+	if (direction === "asc" || direction === "desc") return direction;
+	throw new QueryError("invalid_sort", "Sort direction must be \"asc\" or \"desc\"");
+};
+/**
+* The requested sort with the primary key appended as final tie-breaker, so ordering is total.
+* The tie-breaker follows the first sort direction, which keeps uniform sorts eligible for row-value seeks.
+* Nullable columns sort NULLS LAST for asc and NULLS FIRST for desc (see order.ts and seek.ts).
+*/
+const effectiveSort = (model, sort = []) => {
+	const keys = [];
+	for (const [name, direction] of sort) {
+		const field = requireField(model, name, "sortable");
+		if (keys.some((key) => key.field === field)) continue;
+		keys.push({
+			field,
+			direction: checkDirection(direction)
+		});
+	}
+	const tieBreaker = keys[0]?.direction ?? "asc";
+	for (const field of primaryKeyFields(model)) if (!keys.some((key) => key.field === field)) keys.push({
+		field,
+		direction: tieBreaker
+	});
+	return keys;
+};
+const sortSignature = (keys) => keys.map((key) => `${key.field.name}:${key.direction}`);
+//#endregion
+export { effectiveSort, flip, sortSignature };

package/dist/with-statement-timeout.js

@@ -0,0 +1,32 @@
+import { QueryError } from "./fields/query-error.js";
+import { sql } from "kysely";
+//#region src/with-statement-timeout.ts
+const setTimeout = (db, value) => sql`select set_config('statement_timeout', ${value}, true)`.execute(db);
+/**
+* Runs `run` with Postgres cancelling statements after `timeoutMs`. On a plain connection it opens its own
+* transaction. Given a transaction (for example a read-only one) it sets the timeout locally inside it and
+* restores the previous value afterwards, so the caller's other statements are unaffected.
+*/
+const withStatementTimeout = async (db, timeoutMs, run) => {
+	if (!Number.isInteger(timeoutMs) || timeoutMs < 1) throw new QueryError("invalid_option", "statementTimeoutMs must be a positive integer");
+	if (!db.isTransaction) return db.transaction().execute(async (trx) => {
+		await setTimeout(trx, String(timeoutMs));
+		return run(trx);
+	});
+	const previous = await sql`select current_setting('statement_timeout') as value`.execute(db);
+	const restore = () => setTimeout(db, previous.rows[0].value);
+	await setTimeout(db, String(timeoutMs));
+	let result;
+	try {
+		result = await run(db);
+	} catch (error) {
+		await restore().catch((restoreError) => {
+			if (restoreError.code !== "25P02") throw restoreError;
+		});
+		throw error;
+	}
+	await restore();
+	return result;
+};
+//#endregion
+export { withStatementTimeout };

package/package.json

@@ -1,6 +1,34 @@
 {
   "name": "@protobase/query",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.1",
+  "description": "Kysely list, filter, count and aggregate queries over Protobase resources.",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/PrototeamHQ/protobase.git",
+    "directory": "packages/query"
+  },
+  "homepage": "https://docs.protobase.net",
+  "type": "module",
+  "sideEffects": false,
+  "engines": {
+    "node": ">=22.12.0"
+  },
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "kysely": "^0.29.6",
+    "@protobase/schema": "0.1.1"
+  },
+  "scripts": {
+    "build": "tsdown"
+  }
 }

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.