@arkiv-network/sdk 0.8.1-dev.1 → 0.9.0-next.2

package/src/query/queryBuilder.ts

@@ -1,101 +1,159 @@
-import type { Hex } from "viem"
-import { addr } from "../attr"
 import type { ArkivClient } from "../clients/baseClient"
+import { isSchema, type Schema, SchemaDefinitionError, SchemaMismatchError } from "../schema"
 import type { Entity } from "../types/entity"
-import type { RpcSelect } from "../types/rpcSchema"
+import { QueryBuilderBase } from "./builderBase"
 import { type QueryRequest, runQuery } from "./engine"
-import { InvalidPredicateError } from "./errors"
-import { and, type Expression, eq, type or } from "./expression"
+import type { or } from "./expression"
 import { QueryResult } from "./queryResult"
-import { type SelectArg, toRpcSelect } from "./selection"
+import {
+  type Projection,
+  projectSelection,
+  type ResultOf,
+  type SelectArg,
+  snapshotSelection,
+  toRow,
+  type UnknownSelection,
+} from "./selection"
+
+/**
+ * `from(schema)`'s check that every selected key is an entity field or a column — `unknown` when
+ * it holds, and an object naming the offenders, which no schema satisfies, when it does not.
+ */
+type SelectionCheck<TSelection, TSchema extends Schema> = [
+  UnknownSelection<TSelection, TSchema>,
+] extends [never]
+  ? unknown
+  : { "selected names that are not columns of this schema": UnknownSelection<TSelection, TSchema> }
+
+/**
+ * How a query through a schema treats a matching entity that does not fit the schema — one not
+ * tagged with a named schema's name, or missing a selected required column or holding one of
+ * another type. Columns the selection leaves out are not checked.
+ */
+export type FromOptions = {
+  /**
+   * `"skip"` (the default) leaves the entity out of the page — so a page can hold fewer rows than
+   * its limit, and a page whose every entity was skipped is empty while later pages still have rows.
+   * Each page reports how many it left out as {@link QueryResult.skipped}. Arkiv is public, so
+   * anyone can write an entity that matches a query but not the schema; skipping keeps one such
+   * entity from failing every read of the others. `"throw"` fails the fetch with a
+   * {@link SchemaMismatchError} instead.
+   */
+  onMismatch?: "throw" | "skip" | undefined
+}
 
 /**
  * Builds and runs a query.
  *
  * The selection is fixed when the builder is created and the filter is added by chaining, so the
  * result type is known before a single predicate is written. Create one with `client.select(...)`
- * rather than constructing it directly — that is what infers the entity shape from the selection.
+ * rather than constructing it directly — that is what infers the result shape from the selection.
+ *
+ * The query returns flat rows: the selected entity fields and attributes side by side. Naming a
+ * schema with {@link from} changes what the selection means for nothing but the selected
+ * attributes — they come back typed and checked as the schema declares — and has `where` checked
+ * against it, so that filtering on a column the schema does not index, or comparing one against the
+ * wrong type, is a compile error. A named schema also restricts the query to entities tagged with
+ * its name, and leaves that tag out of the rows' `$attributes` and `$attributeSchema`.
  *
  * Every filter added with {@link where}, {@link ownedBy} and {@link createdBy} must hold; use
  * {@link or} for alternatives inside one call.
  *
- * @typeParam TEntity - The projected entity shape, inferred from the selection by `client.select()`.
+ * @typeParam TSelection - The selection, inferred by `client.select()`.
+ * @typeParam TSchema - The schema named with {@link from}, if any.
  *
  * @example
  * import { createPublicClient } from "@arkiv-network/sdk"
- * import { i32 } from "@arkiv-network/sdk/attr"
+ * import { dec } from "@arkiv-network/sdk/schema"
  * import { tiramisu } from "@arkiv-network/sdk/chains"
- * import { eq, gte, or } from "@arkiv-network/sdk/query"
+ * import { and, eq, gt } from "@arkiv-network/sdk/query"
  * import { http } from "viem"
  *
  * const client = createPublicClient({ chain: tiramisu, transport: http() })
  *
  * const page = await client
- *   .select({ key: true, attributes: true })
- *   .where(gte("level", i32(10)), or(eq("status", "open"), eq("status", "review")))
- *   .ownedBy(owner)
- *   .limit(100)
+ *   .select({ $key: true, author: true, title: true })
+ *   .from(Posts)
+ *   .where(and(eq("category", "docs"), gt("score", dec("4.0"))))
+ *   .limit(20)
  *   .fetch()
  *
- * for (const entity of page.entities) console.log(entity.key, entity.attributes)
+ * for (const row of page.entities) console.log(row.$key, row.title.value)
  */
-export class SelectQueryBuilder<TEntity = Entity> {
-  private readonly _client: ArkivClient
-  private readonly _select: RpcSelect
-  private readonly _filters: Expression[] = []
-  private _ownedBy: Expression | undefined
-  private _createdBy: Expression | undefined
+export class SelectQueryBuilder<
+  TSelection = "*",
+  TSchema extends Schema | undefined = undefined,
+  TCount extends bigint | undefined = undefined,
+> extends QueryBuilderBase<TSchema> {
+  private readonly _selection: SelectArg | undefined
+  private _projection: Projection
+  private _onMismatch: "throw" | "skip" = "skip"
   private _limit: number | undefined
   private _cursor: string | undefined
-  private _atBlock: bigint | undefined
+  private _withCount = false
 
+  /**
+   * @throws If the selection is not `"*"` or an object, asks for nothing, or names an attribute
+   * outside the name grammar.
+   */
   constructor(client: ArkivClient, selection?: SelectArg) {
-    this._client = client
-    this._select = toRpcSelect(selection)
+    super(client)
+    // A copy: the caller's object may change after this, and the builder is typed for it as it was.
+    this._selection = snapshotSelection(selection)
+    this._projection = projectSelection(this._selection)
   }
 
   /**
-   * Adds filters. Everything passed here, and across repeated calls, must hold.
+   * Runs the query through a schema. The selection keeps its meaning and the rows their shape;
+   * each selected attribute becomes a column — typed as the schema declares, and checked against
+   * it — and `where` is checked against the schema's indexed columns.
    *
-   * @param expressions - The expressions, as separate arguments or one array.
+   * A named schema also filters: only entities tagged with its name match, so a query through it
+   * sees only entities written through it, and needs no other filter. The tag is how the query finds
+   * them rather than one of their attributes, so rows leave it out of `$attributes` and
+   * `$attributeSchema`. An unnamed schema types the query without filtering it — `where` alone
+   * decides which entities match.
    *
-   * @example
-   * builder.where(eq("category", "docs"))
-   * builder.where(gte("level", i32(10)), lt("level", i32(20)))
-   * builder.where(or(eq("status", "open"), not(exists("closedAt"))))
-   */
-  where(expressions: readonly Expression[]): this
-  where(...expressions: Expression[]): this
-  where(...expressions: (Expression | readonly Expression[])[]): this {
-    this._filters.push(...expressions.flat())
-    return this
-  }
-
-  /**
-   * Restricts the results to entities this account owns — shorthand for
-   * `where(eq("$owner", addr(owner)))`. Calling it again replaces the filter.
+   * A matching entity that does not fit — a selected required column missing or of another type,
+   * or, for a named schema, a tag that does not match — is left out of the page, or fails the fetch
+   * with a {@link SchemaMismatchError} with `{ onMismatch: "throw" }`. Columns the selection leaves
+   * out are not checked.
    *
-   * @example
-   * builder.ownedBy("0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045")
-   */
-  ownedBy(owner: Hex): this {
-    this._ownedBy = eq("$owner", addr(owner))
-    return this
-  }
-
-  /**
-   * Restricts the results to entities this account created — shorthand for
-   * `where(eq("$creator", addr(creator)))`. Calling it again replaces the filter.
+   * Returns a new builder, carrying whatever this one already holds; this one is left as it was.
+   *
+   * @param schema - The schema, built with `defineSchema`.
+   * @param options - How to treat entities that do not fit. {@link FromOptions}
+   * @throws {SchemaDefinitionError} If `schema` was not built with `defineSchema`.
+   * @throws If the selection names something that is neither an entity field nor a column.
    *
-   * The creator never changes; the owner does, so these differ after a transfer.
+   * @example
+   * client.select({ $key: true, title: true }).from(Posts)   // rows typed { $key; title }
+   * client.select().from(Posts)                               // every field and every column
+   * client.select().from(Posts, { onMismatch: "throw" })      // fail on entities that do not fit
    */
-  createdBy(creator: Hex): this {
-    this._createdBy = eq("$creator", addr(creator))
-    return this
+  from<S extends Schema>(
+    schema: S & SelectionCheck<TSelection, S>,
+    options: FromOptions = {},
+  ): SelectQueryBuilder<TSelection, S, TCount> {
+    if (!isSchema(schema)) {
+      throw new SchemaDefinitionError("from() takes a schema built with defineSchema(...)")
+    }
+    const { onMismatch = "skip" } = options
+    if (onMismatch !== "throw" && onMismatch !== "skip") {
+      throw new TypeError(`onMismatch must be "throw" or "skip", got ${String(onMismatch)}`)
+    }
+    // Resolved here rather than at fetch time, so a selection naming something that is neither an
+    // entity field nor a column fails at the call that made it wrong.
+    const projection = projectSelection(this._selection, schema)
+    const typed = this.copy<SelectQueryBuilder<TSelection, S, TCount>>()
+    typed._schema = schema
+    typed._projection = projection
+    typed._onMismatch = onMismatch
+    return typed
   }
 
   /**
-   * Sets the page size, up to the node maximum of 200. Without it the node picks the page size.
+   * Sets the page size, up to the node maximum of 100. Without it the node picks the page size.
    *
    * @example
    * builder.limit(100)
@@ -118,71 +176,89 @@ export class SelectQueryBuilder<TEntity = Entity> {
   }
 
   /**
-   * Reads the state as of a given block rather than the head. The block must be within the range
-   * the node retains.
+   * Also counts every entity the query matches, regardless of the page size. Each page then carries
+   * the total as {@link QueryResult.count}, typed `bigint`, so a paginated list can show it without
+   * a second query. To count without fetching rows, use `client.count()` instead.
    *
-   * @example
-   * builder.atBlock(1_297_000n)
-   */
-  atBlock(block: bigint): this {
-    this._atBlock = block
-    return this
-  }
-
-  /**
-   * The query string this builder will send — the same text the node parses.
+   * The node counts what the filters match. Through a schema, that includes entities the pages
+   * leave out for not fitting it, so the rows of a whole walk can add up to less than the count:
+   * the difference is the sum of every page's {@link QueryResult.skipped}.
+   *
+   * @throws If, when fetched, the node returns no count — it predates the count projection.
+   *
+   * Returns a new builder, carrying whatever this one already holds; this one is left as it was.
    *
-   * @throws {InvalidPredicateError} If no filter has been added.
+   * @example
+   * const page = await builder.withCount().limit(20).fetch()
+   * page.count // every match, not just these 20
    */
-  toString(): string {
-    const filters = [...this._filters]
-    if (this._ownedBy) filters.push(this._ownedBy)
-    if (this._createdBy) filters.push(this._createdBy)
-    if (filters.length === 0) {
-      throw new InvalidPredicateError(
-        "A query needs at least one filter — the language has no spelling for 'match every " +
-          'entity". Add a where(...) predicate, or narrow by ownedBy(...) / createdBy(...).',
-      )
-    }
-    return and(filters).toString()
+  withCount(): SelectQueryBuilder<TSelection, TSchema, bigint> {
+    const counted = this.copy<SelectQueryBuilder<TSelection, TSchema, bigint>>()
+    counted._withCount = true
+    return counted
   }
 
   /**
    * Runs the query and returns the first page.
    *
-   * @throws {InvalidPredicateError} If no filter has been added.
+   * @throws {InvalidPredicateError} If no filter has been added, or a filter does not fit the
+   * schema.
+   * @throws If the {@link limit} is not an integer from 1 to 100.
+   * @throws {SchemaMismatchError} If, through a schema, a matching entity does not fit the selected
+   * columns or a named schema's tag, and `from()` was given `{ onMismatch: "throw" }`.
    * @throws {QueryError} If the node rejects the query.
    *
    * @example
-   * const page = await client.select({ key: true }).where(eq("category", "docs")).fetch()
+   * const page = await client.select({ $key: true }).where(eq("category", "docs")).fetch()
    * page.entities        // this page
    * await page.next()    // the next one
    */
+  fetch(): Promise<QueryResult<ResultOf<TSelection, TSchema>, TCount>> {
+    return this.fetchFirstPage()
+  }
+
   // Async so that a builder with no filter rejects like every other failure here, rather than
   // throwing synchronously out of a call the caller is awaiting.
-  async fetch(): Promise<QueryResult<TEntity>> {
+  private async fetchFirstPage(): Promise<QueryResult<ResultOf<TSelection, TSchema>, TCount>> {
     // Snapshot the request here, so the pages of one walk are all pages of the *same* query. The
     // builder stays mutable and reusable; a cursor is bound to the query, block and selection it
     // was issued for, so re-reading a since-edited builder would page with a mismatched cursor.
+    const projection = this._projection
     const request = Object.freeze({
       query: this.toString(),
-      select: this._select,
+      select: projection.select,
       limit: this._limit,
       atBlock: this._atBlock,
+      count: this._withCount,
     })
-    return fetchPage<TEntity>(this._client, request, this._cursor)
+    // Every entity is laid out as a row of the selection — and, through a schema, checked against it.
+    // The static narrowing to the selected fields comes from `client.select()`'s inference and
+    // nothing at runtime depends on it.
+    const skip = this._onMismatch === "skip"
+    const decode = ((entity: Entity) => {
+      try {
+        return toRow(entity, projection)
+      } catch (error) {
+        if (skip && error instanceof SchemaMismatchError) return SKIPPED
+        throw error
+      }
+    }) as (entity: Entity) => ResultOf<TSelection, TSchema> | typeof SKIPPED
+    return fetchPage(this._client, request, this._cursor, decode) as Promise<
+      QueryResult<ResultOf<TSelection, TSchema>, TCount>
+    >
   }
 
   /**
-   * Walks every page, yielding one entity at a time.
+   * Walks every page, yielding one row at a time.
    *
    * @example
-   * for await (const entity of client.select({ key: true }).where(eq("category", "docs"))) {
-   *   console.log(entity.key)
+   * for await (const row of client.select({ $key: true }).where(eq("category", "docs"))) {
+   *   console.log(row.$key)
    * }
    */
-  async *[Symbol.asyncIterator](): AsyncGenerator<TEntity> {
-    let page: QueryResult<TEntity> | undefined = await this.fetch()
+  async *[Symbol.asyncIterator](): AsyncGenerator<ResultOf<TSelection, TSchema>> {
+    let page: QueryResult<ResultOf<TSelection, TSchema>, TCount> | undefined =
+      await this.fetchFirstPage()
     while (page) {
       yield* page.entities
       page = page.hasNextPage() ? await page.next() : undefined
@@ -190,25 +266,42 @@ export class SelectQueryBuilder<TEntity = Entity> {
   }
 }
 
+/** What `decode` returns for an entity `{ onMismatch: "skip" }` leaves out of its page. */
+const SKIPPED: unique symbol = Symbol("skipped")
+
 /**
  * Fetches one page and wires up the next.
  *
  * Free-standing rather than a method: a page holds this closure for its lifetime, and closing over
  * a frozen request keeps the builder — and everything it references — out of that reference.
  */
-async function fetchPage<TEntity>(
+async function fetchPage<TResult>(
   client: ArkivClient,
   request: Readonly<Omit<QueryRequest, "cursor">>,
   cursor: string | undefined,
-): Promise<QueryResult<TEntity>> {
+  decode: (entity: Entity) => TResult | typeof SKIPPED,
+): Promise<QueryResult<TResult>> {
   const response = await runQuery(client, { ...request, cursor })
+  // withCount() types the count as present, so a node that leaves it out is an error, not a page
+  // whose count is silently undefined.
+  if (request.count && response.count === undefined) {
+    throw new Error("The node returned no count. It may predate the count projection.")
+  }
+
+  const results: TResult[] = []
+  let skipped = 0
+  for (const entity of response.entities) {
+    const result = decode(entity)
+    if (result === SKIPPED) skipped++
+    else results.push(result)
+  }
 
-  return new QueryResult<TEntity>(
-    // Every row decodes to a full Entity; the static narrowing to the selected fields comes from
-    // `client.select()`'s inference and nothing at runtime depends on it.
-    response.entities as unknown as TEntity[],
+  return new QueryResult<TResult>(
+    results,
     response.blockNumber,
     response.cursor,
-    (next) => fetchPage<TEntity>(client, request, next),
+    skipped,
+    response.count,
+    (next) => fetchPage(client, request, next, decode),
   )
 }

package/src/query/queryResult.ts

@@ -1,8 +1,9 @@
 import { NoMoreResultsError } from "../errors"
-import type { Entity } from "../types/entity"
 
 /** Fetches the page a cursor points at. */
-type FetchPage<TEntity> = (cursor: string) => Promise<QueryResult<TEntity>>
+type FetchPage<TEntity, TCount extends bigint | undefined> = (
+  cursor: string,
+) => Promise<QueryResult<TEntity, TCount>>
 
 /**
  * One page of query results.
@@ -10,16 +11,21 @@ type FetchPage<TEntity> = (cursor: string) => Promise<QueryResult<TEntity>>
  * A page is immutable: {@link next} returns the following page rather than mutating this one, so a
  * page that has been read stays readable.
  *
- * @typeParam TEntity - The shape of each entity, inferred from the selection by `client.select()`.
+ * @typeParam TEntity - The shape of each row, inferred from the selection by `client.select()`.
+ * @typeParam TCount - `bigint` when the query was built with `withCount()`, and `undefined` when it
+ * was not.
  *
  * @example
- * let page = await client.select({ key: true }).where(eq("category", "docs")).limit(100).fetch()
+ * let page = await client.select({ $key: true }).where(eq("category", "docs")).limit(100).fetch()
  * while (page.hasNextPage()) {
  *   page = await page.next()
  * }
  */
-export class QueryResult<TEntity = Entity> {
-  /** The entities on this page. */
+export class QueryResult<
+  TEntity = Record<string, unknown>,
+  TCount extends bigint | undefined = bigint | undefined,
+> {
+  /** The rows on this page — flat, as the selection lays them out. */
   readonly entities: readonly TEntity[]
   /**
    * The block this page was read at.
@@ -30,18 +36,38 @@ export class QueryResult<TEntity = Entity> {
   readonly blockNumber: bigint
   /** The cursor for the next page, or `undefined` when this was the last one. */
   readonly cursor: string | undefined
+  /**
+   * How many matching entities this page left out because they did not fit the schema — always
+   * `0` without a schema, or with `.from(schema, { onMismatch: "throw" })`.
+   *
+   * Counted per page, not per walk: `entities.length + skipped` is what the node returned for this
+   * page, so a short or empty page with a cursor is explained here rather than mistaken for the
+   * end of the results.
+   */
+  readonly skipped: number
+  /**
+   * How many entities the query matches in all, regardless of the page size — a `bigint` when the
+   * query was built with `withCount()`, and `undefined` otherwise. Every page of one walk reads the
+   * same block, so every page carries the same count. It includes entities left out for not fitting
+   * the schema, which {@link skipped} reports per page.
+   */
+  readonly count: TCount
 
-  private readonly fetchPage: FetchPage<TEntity>
+  private readonly fetchPage: FetchPage<TEntity, TCount>
 
   constructor(
     entities: readonly TEntity[],
     blockNumber: bigint,
     cursor: string | undefined,
-    fetchPage: FetchPage<TEntity>,
+    skipped: number,
+    count: TCount,
+    fetchPage: FetchPage<TEntity, TCount>,
   ) {
     this.entities = entities
     this.blockNumber = blockNumber
     this.cursor = cursor
+    this.skipped = skipped
+    this.count = count
     this.fetchPage = fetchPage
   }
 
@@ -64,7 +90,7 @@ export class QueryResult<TEntity = Entity> {
    */
   // Async so that "no more pages" arrives as a rejection like every other failure here, rather
   // than as a synchronous throw the caller's `.catch` would miss.
-  async next(): Promise<QueryResult<TEntity>> {
+  async next(): Promise<QueryResult<TEntity, TCount>> {
     if (this.cursor === undefined) {
       throw new NoMoreResultsError()
     }