@spooky-sync/query-builder 0.0.1-canary.21 → 0.0.1-canary.211

package/src/query-builder.ts

@@ -7,6 +7,12 @@ import type {
   RelatedQuery,
   SchemaAwareQueryModifier,
   SchemaAwareQueryModifierBuilder,
+  WhereInput,
+  QueryPlan,
+  RelationPlan,
+  WhereNode,
+  WhereComparison,
+  ComparisonOp,
 } from './types';
 import type {
   TableNames,
@@ -19,6 +25,59 @@ import type {
   ColumnSchema,
 } from './table-schema';
 
+/**
+ * Reject a query clause that references an `-- @opaque` column.
+ *
+ * An `@opaque` value is synced down to the client but never held server-side, so
+ * the SSP has nothing to evaluate a predicate against. Left unchecked, the
+ * failure is silent and asymmetric: the local cache DOES hold the value, so the
+ * clause filters correctly on screen while the server-side membership set it is
+ * compared against was computed without it. The result reads as rows randomly
+ * appearing and vanishing rather than as an error, so fail loudly at the call
+ * site instead.
+ *
+ * `clause` names the offending API ('where', 'orderBy', …) for the message.
+ */
+function assertNotOpaque(
+  schema: SchemaStructure,
+  tableName: string,
+  clause: string,
+  fields: readonly string[]
+): void {
+  const table = schema.tables.find((t) => t.name === tableName);
+  if (!table) return;
+  for (const field of fields) {
+    // A nested path (`meta.secret`) is checked against its root, since the column
+    // flag lives on `meta`. Comparison operators use `{ field: { _op, _val } }`
+    // rather than a name suffix, so the key is always the bare column name.
+    const column: ColumnSchema | undefined =
+      table.columns[field] ?? table.columns[field.split('.')[0]];
+    if (column?.opaque) {
+      throw new Error(
+        `Cannot use '${field}' in ${clause}(): it is marked '-- @opaque' on ` +
+          `${tableName}, so the sync engine never stores its value and cannot ` +
+          `evaluate it. Read the field from query results instead, or remove ` +
+          `'-- @opaque' from the schema if you need to query on it.`
+      );
+    }
+  }
+}
+
+/** Every column name a `where` clause touches, including `_or` branches. */
+function whereFieldNames(conditions: Record<string, unknown>): string[] {
+  const out: string[] = [];
+  for (const [key, value] of Object.entries(conditions)) {
+    if (key === '_or') {
+      for (const branch of (value ?? []) as Record<string, unknown>[]) {
+        out.push(...whereFieldNames(branch ?? {}));
+      }
+      continue;
+    }
+    out.push(key);
+  }
+  return out;
+}
+
 /**
  * Parse a string ID to RecordId
  * - If it's in the format "table:id", use it as-is
@@ -172,7 +231,7 @@ export class InnerQuery<
 /**
  * Helper type to get the model type for a related table
  */
-type GetRelatedModel<S extends SchemaStructure, RelatedTableName extends string> =
+type _GetRelatedModel<S extends SchemaStructure, RelatedTableName extends string> =
   RelatedTableName extends TableNames<S> ? TableModel<GetTable<S, RelatedTableName>> : never;
 
 /**
@@ -234,6 +293,7 @@ export class FinalQuery<
   S extends SchemaStructure,
   TableName extends TableNames<S>,
   T extends { columns: Record<string, ColumnSchema> },
+  // oxlint-disable-next-line no-unused-vars -- RelatedFields is used externally for type inference
   RelatedFields extends RelatedFieldsMap,
   IsOne extends boolean,
   R = void,
@@ -299,7 +359,13 @@ class SchemaAwareQueryModifierBuilderImpl<
     private readonly schema: S
   ) {}
 
-  where(conditions: Partial<TableModel<GetTable<S, TableName>>>): this {
+  where(conditions: WhereInput<TableModel<GetTable<S, TableName>>>): this {
+    assertNotOpaque(
+      this.schema,
+      this.tableName,
+      'where',
+      whereFieldNames(conditions as Record<string, unknown>)
+    );
     this.options.where = { ...this.options.where, ...conditions };
     return this;
   }
@@ -326,6 +392,7 @@ class SchemaAwareQueryModifierBuilderImpl<
     field: keyof TableModel<GetTable<S, TableName>> & string,
     direction: 'asc' | 'desc' = 'asc'
   ): this {
+    assertNotOpaque(this.schema, this.tableName, 'orderBy', [field]);
     this.options.orderBy = {
       ...this.options.orderBy,
       [field]: direction,
@@ -365,9 +432,18 @@ class SchemaAwareQueryModifierBuilderImpl<
       );
 
       if (!relationship) {
-        throw new Error(
-          `Relationship '${String(relatedField)}' not found for table '${this.tableName}'`
-        );
+        // No such relationship in the client schema — e.g. a table owned by a
+        // devOnly backend (the outbox `job`) that a free/Cloudflare deployment
+        // never provisions, so codegen omits it + its relationships. Skip the
+        // projection instead of throwing, which would take the whole query
+        // (and its other `.related()` siblings — author, comments) down.
+        // Mirrors the server's "unpermitted subquery → empty" degradation.
+        if (typeof console !== 'undefined') {
+          console.warn(
+            `[sp00ky] .related('${String(relatedField)}') skipped — no such relationship on '${this.tableName}' in the client schema`
+          );
+        }
+        return this as any;
       }
 
       const relatedTable = relationship.to;
@@ -412,8 +488,14 @@ export class QueryBuilder<
    * Add additional where conditions
    */
   where(
-    conditions: Partial<TableModel<GetTable<S, TableName>>>
+    conditions: WhereInput<TableModel<GetTable<S, TableName>>>
   ): QueryBuilder<S, TableName, R, RelatedFields, IsOne> {
+    assertNotOpaque(
+      this.schema,
+      this.tableName,
+      'where',
+      whereFieldNames(conditions as Record<string, unknown>)
+    );
     this.options.where = { ...this.options.where, ...conditions };
     return this;
   }
@@ -438,6 +520,7 @@ export class QueryBuilder<
     field: TableFieldNames<GetTable<S, TableName>>,
     direction: 'asc' | 'desc' = 'asc'
   ): QueryBuilder<S, TableName, R, RelatedFields, IsOne> {
+    assertNotOpaque(this.schema, this.tableName, 'orderBy', [field as string]);
     this.options.orderBy = {
       ...this.options.orderBy,
       [field]: direction,
@@ -515,7 +598,15 @@ export class QueryBuilder<
     );
 
     if (!relationship) {
-      throw new Error(`Relationship '${String(field)}' not found for table '${this.tableName}'`);
+      // See the note on the other `.related()` overload: skip an unknown
+      // relationship (warn) rather than throwing, so a table absent from the
+      // client schema (e.g. the free-plan `job` outbox) can't crash the query.
+      if (typeof console !== 'undefined') {
+        console.warn(
+          `[sp00ky] .related('${String(field)}') skipped — no such relationship on '${this.tableName}' in the client schema`
+        );
+      }
+      return this as any;
     }
 
     // Determine cardinality and modifier based on arguments
@@ -600,6 +691,28 @@ export class QueryBuilder<
   }
 }
 
+/**
+ * The surql param name an `_or` branch condition binds under: the branch FIELD,
+ * a `__or` marker and the branch's position. The position alone makes it unique
+ * within the query; the field prefix is what lets a consumer type the value by
+ * looking the column up (see {@link baseFieldOfParam}). Non-identifier
+ * characters (a nested path like `database.owner`) are folded to `_` so the
+ * result is a legal param name.
+ */
+export function orParamName(field: string, index: number): string {
+  return `${field.replace(/[^A-Za-z0-9_]/g, '_')}__or${index}`;
+}
+
+/**
+ * Inverse of {@link orParamName}: the field a param name was built from, or the
+ * name itself when it is a plain top-level param (`field = $field`). Lets a
+ * consumer resolve `white__or0` back to the `white` column.
+ */
+export function baseFieldOfParam(name: string): string {
+  const m = /^(.+)__or\d+$/.exec(name);
+  return m ? m[1] : name;
+}
+
 export function cyrb53(str: string, seed: number = 0): number {
   let h1 = 0xdeadbeef ^ seed,
     h2 = 0x41c6ce57 ^ seed;
@@ -641,6 +754,7 @@ export function extractSubqueryQueryInfos<S extends SchemaStructure>(
     if (relationship) {
       // Determine foreign key field
       // rel.alias is guaranteed to be defined if relationship is found (matched r.field)
+      // oxlint-disable-next-line no-non-null-assertion -- alias is guaranteed defined when relationship is found
       let foreignKeyField = rel.alias!;
 
       if (relationship.cardinality === 'many') {
@@ -751,32 +865,57 @@ export function buildQueryFromOptions<TModel extends GenericModel, IsOne extends
   const vars: Record<string, unknown> = {};
   if (parsedWhere && Object.keys(parsedWhere).length > 0) {
     const conditions: string[] = [];
-    for (const [key, value] of Object.entries(parsedWhere)) {
-      const varName = key;
 
-      // Handle operator objects { _op, _val }
+    // Build a single condition for `field`, binding its value under `varName`.
+    // Supports operator objects `{ _op, _val, _swap }` (e.g. `{ _op: '<=', _val:
+    // 5 }`); a `$`-prefixed string `_val` references an existing param verbatim.
+    // Plain values mean equality (`field = $varName`).
+    const buildCondition = (field: string, value: unknown, varName: string): string => {
       if (value && typeof value === 'object' && '_op' in value && '_val' in value) {
         const { _op, _val, _swap } = value as { _op: string; _val: unknown; _swap?: boolean };
-
-        let rightSide = '';
+        let rightSide: string;
         if (typeof _val === 'string' && _val.startsWith('$')) {
           rightSide = _val;
         } else {
           vars[varName] = _val;
           rightSide = `$${varName}`;
         }
+        return _swap ? `${rightSide} ${_op} ${field}` : `${field} ${_op} ${rightSide}`;
+      }
+      vars[varName] = value;
+      return `${field} = $${varName}`;
+    };
 
-        if (_swap) {
-          conditions.push(`${rightSide} ${_op} ${key}`);
-        } else {
-          conditions.push(`${key} ${_op} ${rightSide}`);
+    for (const [key, value] of Object.entries(parsedWhere)) {
+      // OR-group: `{ _or: [ {field: val}, {field: {_op,_val}}, ... ] }` compiles
+      // to one parenthesised `(c1 OR c2 ...)` conjunct. Each branch condition gets
+      // a unique, position-indexed param name (`white__or0`, `black__or1`, …) so
+      // it never collides with a top-level condition on the same field (e.g. a
+      // `white = $white` filter alongside an opponent `_or` on white/black) -
+      // keeping the surql + vars, and thus the query hash, stable and
+      // deterministic. The FIELD is part of the name on purpose: the consumer
+      // types a param by looking its name up in the table's columns
+      // (`parseQueryParams` in @spooky-sync/core), and a name it cannot resolve
+      // used to be dropped from the registration, which left `$or0` unbound and
+      // every `_or` query matching nothing.
+      if (key === '_or' && Array.isArray(value)) {
+        const orParts: string[] = [];
+        let i = 0;
+        for (const branch of value) {
+          if (branch && typeof branch === 'object') {
+            for (const [bField, bVal] of Object.entries(branch as Record<string, unknown>)) {
+              orParts.push(buildCondition(bField, bVal, orParamName(bField, i++)));
+            }
+          }
         }
-      } else {
-        vars[varName] = value;
-        conditions.push(`${key} = $${varName}`);
+        if (orParts.length > 0) conditions.push(`(${orParts.join(' OR ')})`);
+        continue;
       }
+
+      conditions.push(buildCondition(key, value, key));
     }
-    query += ` WHERE ${conditions.join(' AND ')}`;
+
+    if (conditions.length > 0) query += ` WHERE ${conditions.join(' AND ')}`;
   }
 
   // Add PATCH for UPDATE
@@ -815,9 +954,193 @@ export function buildQueryFromOptions<TModel extends GenericModel, IsOne extends
       0
     ),
     vars: Object.keys(vars).length > 0 ? vars : undefined,
+    // Engine-neutral plan mirrors the SELECT above for non-SurrealQL backends.
+    // Only SELECT carries a plan; the isOne→limit=1 mutation above is already
+    // reflected in `options.limit`, so the plan sees it too.
+    plan: method === 'SELECT' ? buildQueryPlan(tableName, options, schema) : undefined,
   };
 }
 
+/**
+ * Build the engine-neutral {@link QueryPlan} for a SELECT. Mirrors the string
+ * assembly in {@link buildQueryFromOptions} / {@link buildSubquery} exactly so a
+ * non-SurrealQL backend produces results identical to the SurrealQL path.
+ */
+function buildQueryPlan<TModel extends GenericModel, IsOne extends boolean>(
+  tableName: string,
+  options: QueryOptions<TModel, IsOne>,
+  schema: SchemaStructure
+): QueryPlan {
+  const plan: QueryPlan = { table: tableName };
+
+  if (options.select && options.select.length > 0 && !options.select.includes('*')) {
+    plan.select = options.select.filter((f) => f !== '*') as string[];
+  }
+
+  const parsedWhere = options.where
+    ? (parseObjectIdsToRecordId(options.where, tableName) as Record<string, unknown>)
+    : undefined;
+  if (parsedWhere && Object.keys(parsedWhere).length > 0) {
+    // slaveToParams: top-level filters materialize from `params` (the query's
+    // identity), not a baked literal — see buildWhereNodes. Prevents a query's
+    // rows ever coming from a different query's plan.
+    const nodes = buildWhereNodes(parsedWhere, true);
+    if (nodes.length > 0) plan.where = nodes;
+  }
+
+  if (options.orderBy && Object.keys(options.orderBy).length > 0) {
+    plan.orderBy = Object.entries(options.orderBy).map(
+      ([field, direction]) => [field, direction as 'asc' | 'desc']
+    );
+  }
+
+  if (options.limit !== undefined) plan.limit = options.limit;
+  if (options.offset !== undefined) plan.offset = options.offset;
+
+  if (options.related && options.related.length > 0) {
+    plan.relations = options.related.map((rel) => buildRelationPlan(rel, schema));
+  }
+
+  return plan;
+}
+
+/**
+ * Engine-neutral counterpart of {@link buildSubquery}. Resolves the same
+ * cardinality / foreign-key / nested-relation metadata but returns a structured
+ * {@link RelationPlan} instead of a SurrealQL subquery string.
+ */
+function buildRelationPlan(
+  rel: RelatedQuery & { foreignKeyField?: string },
+  schema: SchemaStructure
+): RelationPlan {
+  const { relatedTable, alias, modifier, cardinality } = rel;
+  // Same fallback chain as buildSubquery (`rel.foreignKeyField || alias`); the
+  // top-level foreignKeyField is already reverse-resolved by `.related()`.
+  const foreignKeyField = (rel.foreignKeyField || alias || relatedTable) as string;
+
+  const plan: RelationPlan = {
+    alias: (alias || relatedTable) as string,
+    table: relatedTable,
+    cardinality,
+    foreignKeyField,
+  };
+
+  if (modifier) {
+    const modifierBuilder = new SchemaAwareQueryModifierBuilderImpl(relatedTable, schema);
+    modifier(modifierBuilder as any);
+    const subOptions = modifierBuilder._getOptions();
+
+    if (subOptions.select && subOptions.select.length > 0 && !subOptions.select.includes('*')) {
+      plan.select = subOptions.select.filter((f) => f !== '*') as string[];
+    }
+
+    if (subOptions.where && Object.keys(subOptions.where).length > 0) {
+      const parsedSubWhere = parseObjectIdsToRecordId(subOptions.where, relatedTable) as Record<
+        string,
+        unknown
+      >;
+      const nodes = buildWhereNodes(parsedSubWhere);
+      if (nodes.length > 0) plan.where = nodes;
+    }
+
+    if (subOptions.orderBy && Object.keys(subOptions.orderBy).length > 0) {
+      plan.orderBy = Object.entries(subOptions.orderBy).map(
+        ([field, direction]) => [field, direction as 'asc' | 'desc']
+      );
+    }
+
+    if (subOptions.limit !== undefined) plan.limit = subOptions.limit;
+
+    // Nested relations: re-resolve exactly as buildSubquery does — the child's
+    // foreignKeyField comes from the relatedTable-based lookup, not the reverse
+    // heuristic used for top-level relations.
+    if (subOptions.related && subOptions.related.length > 0) {
+      const resolvedNestedRels = subOptions.related.map((nestedRel) => {
+        const relationship = schema.relationships.find(
+          (r) => r.from === relatedTable && r.field === nestedRel.alias
+        );
+        if (relationship) {
+          const nestedForeignKeyField =
+            relationship.cardinality === 'many' ? relatedTable : (nestedRel.alias as string);
+          return {
+            ...nestedRel,
+            relatedTable: relationship.to,
+            cardinality: relationship.cardinality,
+            foreignKeyField: nestedForeignKeyField,
+          } as RelatedQuery & { foreignKeyField: string };
+        }
+        return nestedRel;
+      });
+      plan.relations = resolvedNestedRels.map((nestedRel) => buildRelationPlan(nestedRel, schema));
+    }
+  }
+
+  // one-to-one gets an implicit per-parent LIMIT 1 (matches buildSubquery).
+  if (cardinality === 'one' && plan.limit === undefined) {
+    plan.limit = 1;
+  }
+
+  return plan;
+}
+
+/**
+ * Convert a parsed WHERE object (string IDs already → RecordId) into the
+ * engine-neutral {@link WhereNode}[] conjunction. Mirrors the `_or` / comparison
+ * / equality handling in {@link buildQueryFromOptions}. A `$`-prefixed `_val`
+ * becomes a `paramRef` (with the leading `$` stripped).
+ */
+/**
+ * @param slaveToParams When true, top-level plain/operator-literal comparisons
+ *   ALSO carry a `paramRef` equal to the field name — the same var name
+ *   `buildQueryFromOptions` binds the value under (`field = $field`). The
+ *   engines then materialize by reading `params[field]` (falling back to the
+ *   baked `value` when the param is absent), so a query's rows are slaved to
+ *   its `params` (its identity) and can never come from a different query's
+ *   baked plan. Only safe at the TOP LEVEL, where the field is a schema column
+ *   that survives `parseQueryParams` and the caller passes `params`. NOT used
+ *   for relation sub-wheres (rendered with a params-less ctx) or `_or` branches
+ *   (bound under synthetic `white__or0` names) - those stay baked.
+ */
+function buildWhereNodes(
+  parsedWhere: Record<string, unknown>,
+  slaveToParams = false
+): WhereNode[] {
+  const toComparison = (field: string, value: unknown, slave: boolean): WhereComparison => {
+    if (value && typeof value === 'object' && '_op' in value && '_val' in value) {
+      const { _op, _val, _swap } = value as ComparisonOp;
+      if (typeof _val === 'string' && _val.startsWith('$')) {
+        return { field, op: _op, value: undefined, paramRef: _val.slice(1), swap: _swap };
+      }
+      // Literal operand: keep `value` as a fallback and add `paramRef: field`
+      // (slave mode) so materialization reads the query's own `params[field]`.
+      return slave
+        ? { field, op: _op, value: _val, paramRef: field, swap: _swap }
+        : { field, op: _op, value: _val, swap: _swap };
+    }
+    return slave ? { field, op: '=', value, paramRef: field } : { field, op: '=', value };
+  };
+
+  const nodes: WhereNode[] = [];
+  for (const [key, value] of Object.entries(parsedWhere)) {
+    if (key === '_or' && Array.isArray(value)) {
+      const or: WhereComparison[] = [];
+      for (const branch of value) {
+        if (branch && typeof branch === 'object') {
+          for (const [bField, bVal] of Object.entries(branch as Record<string, unknown>)) {
+            // OR branches bind under synthetic `white__or0` names (see
+            // orParamName / buildQueryFromOptions); the plan keeps them baked.
+            or.push(toComparison(bField, bVal, false));
+          }
+        }
+      }
+      if (or.length > 0) nodes.push({ or });
+      continue;
+    }
+    nodes.push(toComparison(key, value, slaveToParams));
+  }
+  return nodes;
+}
+
 /**
  * Build a subquery for a related field
  */

package/src/repro_relationship.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { QueryBuilder, SchemaStructure } from './index';
+import { QueryBuilder } from './index';
 
 describe('QueryBuilder Relationship Inference', () => {
   const schema = {

package/src/table-schema.ts

@@ -1,16 +1,42 @@
 /**
  * Supported value types in the schema
  */
-export type ValueType = 'string' | 'number' | 'boolean' | 'null' | 'json';
+export type ValueType = 'string' | 'number' | 'boolean' | 'null' | 'json' | 'Uint8Array';
 
 /**
  * Column metadata defining the type and optionality of a field
  */
+/**
+ * CRDT types supported by Sp00ky's Loro integration
+ */
+export type CrdtType = 'text' | 'map' | 'list' | 'counter';
+
 export interface ColumnSchema {
   readonly type: ValueType;
   readonly optional: boolean;
   readonly dateTime?: boolean;
   readonly recordId?: boolean;
+  readonly crdt?: CrdtType;
+  readonly cursor?: boolean;
+  /** True for `TYPE bytes` columns. Runtime values are `Uint8Array`. */
+  readonly bytes?: boolean;
+  /**
+   * True for `TYPE array<...>` columns. `type` then names the ELEMENT type, so
+   * the runtime value is `ElementType[]` (e.g. `array<string>` → `string[]`).
+   */
+  readonly array?: boolean;
+  /**
+   * True for `-- @opaque` columns: the value IS synced to the client and can be
+   * read from a query result, but the sync engine never stores it server-side.
+   *
+   * That makes it unusable for anything the server has to evaluate — `where`,
+   * `orderBy`, joins, aggregates, table permissions — because the SSP has no
+   * value to evaluate against. A predicate on such a column would appear to work
+   * locally (the local cache does hold the value) while matching nothing
+   * server-side, so the query builder rejects it outright instead of letting the
+   * two diverge silently.
+   */
+  readonly opaque?: boolean;
 }
 
 /**
@@ -62,16 +88,25 @@ export type TypeNameToTypeMap = {
   boolean: boolean;
   null: null;
   json: unknown;
+  Uint8Array: Uint8Array;
 };
 
+/**
+ * The element/base TS type of a column, wrapping in an array for `array: true`
+ * columns (where `type` names the element type).
+ */
+export type ColumnBaseTSType<T extends ColumnSchema> = T extends { array: true }
+  ? TypeNameToTypeMap[T['type']][]
+  : TypeNameToTypeMap[T['type']];
+
 /**
  * Convert a column type to its TypeScript type
  */
 export type ColumnToTSType<T extends ColumnSchema> = T extends {
   optional: true;
 }
-  ? TypeNameToTypeMap[T['type']] | null
-  : TypeNameToTypeMap[T['type']];
+  ? ColumnBaseTSType<T> | null
+  : ColumnBaseTSType<T>;
 
 /**
  * Helper to extract relationship field names for a table