@pplethai/cli 0.7.0 → 0.8.0

package/templates/golden-path/src/dataset-query.ts

@@ -0,0 +1,470 @@
+// Filter, sort and keyset paging for one Dataset Object Type (ADR-0030).
+//
+// A copy of workers/src/shared/registry/query.ts. This file is copied into
+// a Builder App, which cannot import the platform package. Offline
+// readDataset runs it over mock rows. A cursor binds to the filters and the
+// sort: the same cursor on a different query is invalid_query. A descending
+// sort breaks ties by id descending, matching the warehouse's backward index
+// walk. cli/test/registry-query.test.ts keeps this copy in step with the
+// platform.
+
+import { canonicalJson, isPlainObject, valueProblems } from "./dataset-schema.js";
+
+export type QueryValue = string | number | boolean;
+
+export type DatasetFilterOp = "eq" | "in" | "gt" | "gte" | "lt" | "lte";
+
+export interface DatasetFilter {
+  field: string;
+  op: DatasetFilterOp;
+  values: QueryValue[];
+}
+
+/** A collection the query module can page. A singleton is not a query. */
+export interface DatasetQuerySpec {
+  idType: "string" | "integer";
+  index: readonly { name: string; schema: Record<string, unknown> }[];
+}
+
+export type DatasetCursor =
+  | { kind: "id"; id: string }
+  | { kind: "field"; id: string; value: QueryValue | null };
+
+export interface DatasetQuery {
+  filters: DatasetFilter[];
+  /** Null is the default: id ascending. */
+  sort: { field: string; direction: "asc" | "desc" } | null;
+  limit: number;
+  after: DatasetCursor | null;
+  /** Hash of the filters and the sort. Limit and the cursor are not part of it. */
+  hash: string;
+}
+
+export interface DatasetRow {
+  id: string;
+  object: Record<string, unknown>;
+}
+
+const DEFAULT_LIMIT = 100;
+const MAX_LIMIT = 1000;
+const RESERVED = new Set(["limit", "sort", "next"]);
+
+function rangeOp(value: string | undefined): "gt" | "gte" | "lt" | "lte" | null {
+  if (value === "gt" || value === "gte" || value === "lt" || value === "lte") return value;
+  return null;
+}
+
+type FieldKind = "string" | "number" | "integer" | "boolean";
+
+/**
+ * Parse a collection query. `invalid_query` is the only failure: a bad value,
+ * a field that is not an index, a limit out of range, or a cursor from
+ * another query.
+ */
+export function parseDatasetQuery(
+  params: URLSearchParams,
+  spec: DatasetQuerySpec,
+): { ok: true; query: DatasetQuery } | { ok: false; error: "invalid_query" } {
+  const invalid = { ok: false, error: "invalid_query" } as const;
+  if (params.getAll("limit").length > 1 || params.getAll("sort").length > 1 || params.getAll("next").length > 1) {
+    return invalid;
+  }
+
+  const limit = parseLimit(params.get("limit"));
+  if (limit === null) return invalid;
+
+  const sort = parseSort(params.get("sort"), spec);
+  if (sort === undefined) return invalid;
+
+  const filters = parseFilters(params, spec);
+  if (filters === null) return invalid;
+
+  const hash = datasetOpaqueHash({
+    filters,
+    sort: sort ?? { field: "id", direction: "asc" },
+  });
+
+  const rawCursor = params.get("next");
+  let after: DatasetCursor | null = null;
+  if (rawCursor !== null) {
+    if (rawCursor === "") return invalid;
+    const cursor = decodeCursor(rawCursor, spec, sort, hash);
+    if (cursor === null) return invalid;
+    after = cursor;
+  }
+
+  return { ok: true, query: { filters, sort, limit, after, hash } };
+}
+
+/** One page of `rows`, in query order. `next` is null on the last page. */
+export function applyDatasetQuery(
+  rows: readonly DatasetRow[],
+  query: DatasetQuery,
+  spec: DatasetQuerySpec,
+): { objects: Record<string, unknown>[]; next: string | null } {
+  const sorted = rows.filter((row) => rowMatches(row, query, spec)).slice();
+  sorted.sort((left, right) => compareRows(left, right, query, spec));
+  let start = 0;
+  if (query.after) {
+    const cursor = cursorRow(query);
+    const found = sorted.findIndex((row) => compareRows(row, cursor, query, spec) > 0);
+    start = found === -1 ? sorted.length : found;
+  }
+  const page = sorted.slice(start, start + query.limit);
+  const last = page[page.length - 1];
+  const more = start + query.limit < sorted.length;
+  return {
+    objects: page.map((row) => row.object),
+    next: more && last ? datasetPageCursor(query, last, spec) : null,
+  };
+}
+
+/** The cursor a caller sends to get the page after `last`. */
+export function datasetPageCursor(query: DatasetQuery, last: DatasetRow, spec: DatasetQuerySpec): string {
+  if (query.sort === null) return encodeCursor({ h: query.hash, id: last.id });
+  return encodeCursor({
+    h: query.hash,
+    id: last.id,
+    s: 1,
+    v: exposedValue(last.object, query.sort.field, spec),
+  });
+}
+
+/**
+ * Hash of everything that changes a page except the snapshot: the query, the
+ * page size and the cursor. Two pages therefore do not share an ETag.
+ */
+export function datasetPageHash(query: DatasetQuery): string {
+  return datasetOpaqueHash({ hash: query.hash, limit: query.limit, after: query.after });
+}
+
+/** Stable hash of a JSON value. Used for cursors and ETags, not as a secret. */
+export function datasetOpaqueHash(value: unknown): string {
+  const text = canonicalJson(value);
+  return `${fnv(text, 0x811c9dc5)}${fnv(text, 0x01000193)}`;
+}
+
+function parseLimit(raw: string | null): number | null {
+  if (raw === null) return DEFAULT_LIMIT;
+  const text = raw.trim();
+  if (!/^\d+$/.test(text)) return null;
+  const value = Number(text);
+  if (!Number.isSafeInteger(value) || value < 1 || value > MAX_LIMIT) return null;
+  return value;
+}
+
+function parseSort(
+  raw: string | null,
+  spec: DatasetQuerySpec,
+): { field: string; direction: "asc" | "desc" } | null | undefined {
+  if (raw === null) return null;
+  const text = raw.trim();
+  if (text === "") return undefined;
+  const descending = text.startsWith("-");
+  const field = descending ? text.slice(1) : text;
+  if (field === "" || !spec.index.some((item) => item.name === field)) return undefined;
+  return { field, direction: descending ? "desc" : "asc" };
+}
+
+function parseFilters(params: URLSearchParams, spec: DatasetQuerySpec): DatasetFilter[] | null {
+  const equals = new Map<string, string[]>();
+  const ranges = new Map<string, Partial<Record<"gt" | "gte" | "lt" | "lte", string>>>();
+  const known = new Map(spec.index.map((field) => [field.name, field.schema]));
+
+  for (const [key, raw] of params) {
+    if (RESERVED.has(key)) continue;
+    const range = /^([^[\]]+)\[(gt|gte|lt|lte)]$/.exec(key);
+    if (range) {
+      const name = range[1];
+      const op = rangeOp(range[2]);
+      if (!name || !op) return null;
+      if (!known.has(name)) return null;
+      const current = ranges.get(name) ?? {};
+      // The same bound twice is not a range a caller can mean.
+      if (current[op] !== undefined) return null;
+      current[op] = raw;
+      ranges.set(name, current);
+      continue;
+    }
+    if (key.includes("[") || key.includes("]")) return null;
+    if (!known.has(key)) return null;
+    const list = equals.get(key) ?? [];
+    list.push(raw);
+    equals.set(key, list);
+  }
+
+  const filters: DatasetFilter[] = [];
+  for (const [field, raws] of equals) {
+    const schema = known.get(field);
+    if (!schema) return null;
+    const parsed = uniqueValues(raws.map((raw) => parseScalar(raw, schema)));
+    if (parsed.some((value) => value === undefined)) return null;
+    const values = parsed.filter((value): value is QueryValue => value !== undefined);
+    filters.push({ field, op: values.length > 1 ? "in" : "eq", values: values.sort(byCanonical) });
+  }
+  for (const [field, ops] of ranges) {
+    const schema = known.get(field);
+    if (!schema) return null;
+    for (const op of ["gt", "gte", "lt", "lte"] as const) {
+      const raw = ops[op];
+      if (raw === undefined) continue;
+      const value = parseScalar(raw, schema);
+      if (value === undefined) return null;
+      filters.push({ field, op, values: [value] });
+    }
+  }
+  filters.sort((left, right) => compareText(left.field, right.field) || opRank(left.op) - opRank(right.op));
+  return filters;
+}
+
+function parseScalar(raw: string, schema: Record<string, unknown>): QueryValue | undefined {
+  for (const type of declaredTypes(schema)) {
+    const value = coerce(raw, type);
+    if (value === undefined) continue;
+    if (valueProblems(schema, value).length === 0) return value;
+  }
+  return undefined;
+}
+
+function declaredTypes(schema: Record<string, unknown>): string[] {
+  const type = schema.type;
+  const list = typeof type === "string" ? [type] : Array.isArray(type) ? type : [];
+  return list.filter((item): item is string => typeof item === "string" && item !== "null");
+}
+
+function coerce(raw: string, type: string): QueryValue | undefined {
+  if (type === "boolean") {
+    if (raw === "true") return true;
+    if (raw === "false") return false;
+    return undefined;
+  }
+  if (type === "integer") {
+    if (!/^-?\d+$/.test(raw)) return undefined;
+    const value = Number(raw);
+    if (!Number.isSafeInteger(value)) return undefined;
+    return value;
+  }
+  if (type === "number") {
+    if (!/^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/.test(raw)) return undefined;
+    const value = Number(raw);
+    if (!Number.isFinite(value)) return undefined;
+    return value;
+  }
+  if (type === "string") return raw;
+  return undefined;
+}
+
+function uniqueValues(values: readonly (QueryValue | undefined)[]): (QueryValue | undefined)[] {
+  const seen = new Set<string>();
+  const out: (QueryValue | undefined)[] = [];
+  for (const value of values) {
+    const key = value === undefined ? "\0" : canonicalJson(value);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(value);
+  }
+  return out;
+}
+
+function byCanonical(left: QueryValue, right: QueryValue): number {
+  return compareText(canonicalJson(left), canonicalJson(right));
+}
+
+function opRank(op: DatasetFilterOp): number {
+  if (op === "eq") return 0;
+  if (op === "in") return 1;
+  if (op === "gt") return 2;
+  if (op === "gte") return 3;
+  if (op === "lt") return 4;
+  return 5;
+}
+
+function decodeCursor(
+  token: string,
+  spec: DatasetQuerySpec,
+  sort: { field: string; direction: "asc" | "desc" } | null,
+  hash: string,
+): DatasetCursor | null {
+  let value: unknown;
+  try {
+    value = JSON.parse(base64UrlDecode(token));
+  } catch {
+    return null;
+  }
+  if (!isPlainObject(value) || typeof value.h !== "string" || value.h !== hash) return null;
+  if (typeof value.id !== "string" || !idFits(value.id, spec.idType)) return null;
+  const fieldSort = value.s === 1;
+  if (sort === null) {
+    if (fieldSort || "v" in value) return null;
+    return { kind: "id", id: value.id };
+  }
+  if (!fieldSort) return null;
+  const sortValue = "v" in value ? value.v : null;
+  if (!sortValueFits(sortValue, sort.field, spec)) return null;
+  return { kind: "field", id: value.id, value: sortValue as QueryValue | null };
+}
+
+function idFits(id: string, idType: DatasetQuerySpec["idType"]): boolean {
+  if (id === "") return false;
+  if (idType === "string") return true;
+  if (!/^-?\d+$/.test(id)) return false;
+  return Number.isSafeInteger(Number(id));
+}
+
+function sortValueFits(value: unknown, field: string, spec: DatasetQuerySpec): value is QueryValue | null {
+  if (value === null) return true;
+  const schema = spec.index.find((item) => item.name === field)?.schema;
+  if (!schema) return false;
+  const kind = fieldKind(schema);
+  if (kind === "boolean") return typeof value === "boolean";
+  if (kind === "string") return typeof value === "string";
+  if (typeof value !== "number" || !Number.isFinite(value)) return false;
+  return kind === "number" || Number.isInteger(value);
+}
+
+function encodeCursor(body: { h: string; id: string; s?: 1; v?: QueryValue | null }): string {
+  const payload: Record<string, unknown> = { h: body.h, id: body.id };
+  if (body.s === 1) {
+    payload.s = 1;
+    payload.v = body.v ?? null;
+  }
+  return base64UrlEncode(JSON.stringify(payload));
+}
+
+function rowMatches(row: DatasetRow, query: DatasetQuery, spec: DatasetQuerySpec): boolean {
+  return query.filters.every((filter) => filterMatches(row.object, filter, spec));
+}
+
+function filterMatches(object: Record<string, unknown>, filter: DatasetFilter, spec: DatasetQuerySpec): boolean {
+  const schema = spec.index.find((item) => item.name === filter.field)?.schema;
+  if (!schema) return false;
+  const kind = fieldKind(schema);
+  const stored = orderedValue(object, filter.field, kind);
+  if (filter.op === "eq" || filter.op === "in") {
+    return filter.values.some((value) => compareOrdered(stored, orderQueryValue(value, kind)) === 0);
+  }
+  const target = filter.values[0];
+  if (target === undefined || stored === null) return false;
+  const compared = compareOrdered(stored, orderQueryValue(target, kind));
+  if (filter.op === "gt") return compared > 0;
+  if (filter.op === "gte") return compared >= 0;
+  if (filter.op === "lt") return compared < 0;
+  return compared <= 0;
+}
+
+function compareRows(left: DatasetRow, right: DatasetRow, query: DatasetQuery, spec: DatasetQuerySpec): number {
+  if (query.sort) {
+    const schema = spec.index.find((item) => item.name === query.sort?.field)?.schema;
+    const kind = schema ? fieldKind(schema) : "string";
+    const compared = compareOrdered(
+      orderedValue(left.object, query.sort.field, kind),
+      orderedValue(right.object, query.sort.field, kind),
+      query.sort.direction,
+    );
+    if (compared !== 0) return compared;
+  }
+  // Ties follow the sort direction, as the warehouse's (ix_n, id) index walk does.
+  return compareOrdered(
+    orderedId(left.id, spec.idType),
+    orderedId(right.id, spec.idType),
+    query.sort?.direction ?? "asc",
+  );
+}
+
+function cursorRow(query: DatasetQuery): DatasetRow {
+  if (query.after === null) return { id: "", object: {} };
+  if (query.after.kind === "id" || query.sort === null) return { id: query.after.id, object: {} };
+  return {
+    id: query.after.id,
+    object: { [query.sort.field]: query.after.value },
+  };
+}
+
+function exposedValue(object: Record<string, unknown>, field: string, spec: DatasetQuerySpec): QueryValue | null {
+  const schema = spec.index.find((item) => item.name === field)?.schema;
+  if (!schema || !Object.hasOwn(object, field)) return null;
+  const value = object[field];
+  if (value === null || value === undefined) return null;
+  const kind = fieldKind(schema);
+  if (kind === "boolean") return value === true ? true : value === false ? false : null;
+  if (kind === "integer" || kind === "number") return typeof value === "number" && Number.isFinite(value) ? value : null;
+  return typeof value === "string" ? value : null;
+}
+
+function fieldKind(schema: Record<string, unknown>): FieldKind {
+  const names = new Set(declaredTypes(schema));
+  if (names.has("boolean")) return "boolean";
+  if (names.has("integer")) return "integer";
+  if (names.has("number")) return "number";
+  return "string";
+}
+
+function orderedValue(object: Record<string, unknown>, field: string, kind: FieldKind): string | number | null {
+  if (!Object.hasOwn(object, field)) return null;
+  const value = object[field];
+  if (value === null || value === undefined) return null;
+  return orderQueryValue(value as QueryValue, kind);
+}
+
+function orderQueryValue(value: QueryValue | null, kind: FieldKind): string | number | null {
+  if (value === null) return null;
+  if (kind === "boolean") {
+    if (value === true || value === 1) return 1;
+    if (value === false || value === 0) return 0;
+    return null;
+  }
+  if (kind === "integer" || kind === "number") return typeof value === "number" && Number.isFinite(value) ? value : null;
+  return typeof value === "string" ? value : null;
+}
+
+function orderedId(id: string, idType: DatasetQuerySpec["idType"]): string | number {
+  return idType === "integer" ? Number(id) : id;
+}
+
+/**
+ * SQLite orders nulls first on ASC and last on DESC. Booleans are 0 and 1,
+ * which is how a generated INTEGER column stores JSON true and false.
+ */
+function compareOrdered(
+  left: string | number | null,
+  right: string | number | null,
+  direction: "asc" | "desc" = "asc",
+): number {
+  if (left === null && right === null) return 0;
+  if (left === null) return direction === "asc" ? -1 : 1;
+  if (right === null) return direction === "asc" ? 1 : -1;
+  let compared = 0;
+  if (typeof left === "number" && typeof right === "number") compared = left - right;
+  else compared = compareText(String(left), String(right));
+  if (compared === 0) return 0;
+  return direction === "asc" ? compared : -compared;
+}
+
+function compareText(left: string, right: string): number {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}
+
+function fnv(text: string, seed: number): string {
+  let hash = seed >>> 0;
+  for (let i = 0; i < text.length; i++) {
+    hash ^= text.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return (hash >>> 0).toString(16).padStart(8, "0");
+}
+
+function base64UrlEncode(text: string): string {
+  const bytes = new TextEncoder().encode(text);
+  let binary = "";
+  for (const byte of bytes) binary += String.fromCharCode(byte);
+  return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/, "");
+}
+
+function base64UrlDecode(token: string): string {
+  const padded = token.replaceAll("-", "+").replaceAll("_", "/") + "=".repeat((4 - (token.length % 4)) % 4);
+  const binary = atob(padded);
+  const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0));
+  return new TextDecoder().decode(bytes);
+}