@gaganref/confect-test 10.3.1-fork.90 → 10.3.1-fork.92

package/src/internal/Execution.ts

@@ -9,7 +9,13 @@ import {
   type SchemaDefinition,
   type UserIdentity,
 } from "convex/server";
-import { convexToJson, jsonToConvex, type Value } from "convex/values";
+import {
+  CommitTsPlaceholder,
+  compareValues,
+  convexToJson,
+  jsonToConvex,
+  type Value,
+} from "convex/values";
 import { KernelAttempt } from "./Attempt.js";
 import * as Clock from "./Clock.js";
 import * as Globals from "./Globals.js";
@@ -17,6 +23,7 @@ import * as Invocation from "./Invocation.js";
 import * as Kernel from "./Kernel.js";
 import * as KernelQuery from "./KernelQuery.js";
 import * as KernelRefusal from "./KernelRefusal.js";
+import * as Metering from "./Metering.js";
 import type * as ReadSet from "./ReadSet.js";
 import * as Scheduling from "./Scheduling.js";
 import type {
@@ -31,6 +38,7 @@ import {
   deserializeConvexErrorData,
   evaluateValue,
   hasPendingValues,
+  isSimpleObject,
   parseArgs,
   parseFunctionHandle,
   resolveCommittedValues,
@@ -137,6 +145,47 @@ export type KernelApply =
     }
   | { readonly kind: "refused"; readonly tag: string; readonly error: unknown };
 
+/**
+ * One top-level query run and every dependency it took: the commit and reads it
+ * read, whether it asked for the caller's identity, the pinned time it read and
+ * the end of its paginated read.
+ */
+export interface QueryRun {
+  readonly outcome: Outcome;
+  readonly snapshot: Kernel.Version;
+  readonly reads: ReadonlyArray<ReadSet.Read>;
+  readonly observedIdentity: boolean;
+  /**
+   * The pinned time the query read, or `null` when it never read the time.
+   */
+  readonly time: number | null;
+  /**
+   * Where the query's paginated read ended, or `null` when it ran none. A rerun
+   * ends its page there, as the binary's query journal does.
+   */
+  readonly journal: string | null;
+}
+
+/**
+ * The one paginated read a query and its nested calls may run. `journal` is
+ * where a previous run's page ended, and `end` where this run's page ends.
+ */
+interface Pagination {
+  calls: number;
+  readonly journal: string | null;
+  end: string | null;
+}
+
+/**
+ * A top-level query whose attempt and page the caller keeps, at a pinned start
+ * time or the kernel's time now.
+ */
+interface TopQuery {
+  readonly attempt: KernelAttempt;
+  readonly pagination: Pagination;
+  readonly startTime: number | undefined;
+}
+
 interface Held {
   readonly view: HeldAttempt;
   readonly attempt: KernelAttempt;
@@ -174,8 +223,13 @@ interface Execution {
    * calls share it, and `Date` answers it inside them.
    */
   readonly startTime: number;
-  readonly pagination: { calls: number } | undefined;
+  readonly pagination: Pagination | undefined;
   readonly turn: Turn;
+  /**
+   * The transaction limits this function runs under: the defaults at the top
+   * level, its caller's inside a nested call, tightened by the call's budget.
+   */
+  readonly limits: Metering.Limits;
 }
 
 /**
@@ -227,14 +281,15 @@ const kernelBrand = Symbol.for("@confect/test/Kernel");
 
 /**
  * The time `Date` answers inside a kernel invocation: a query's or mutation's
- * start, or the clock as it runs in an action. Outside one, none.
+ * start, or the clock as it runs in an action. Outside one, none. A query or
+ * mutation that reads it takes a dependency on that time.
  */
 const pinnedTime = (): number | undefined => {
   const execution = executions.getStore();
   if (execution === undefined) return undefined;
-  return execution.udfType === "action"
-    ? execution.world.now()
-    : execution.startTime;
+  if (execution.udfType === "action") return execution.world.now();
+  execution.attempt?.observeTime(execution.startTime);
+  return execution.startTime;
 };
 
 /**
@@ -642,6 +697,51 @@ export class World {
     return Array.from(this.holding.values(), (held) => held.view);
   }
 
+  // Subscribed queries.
+
+  /**
+   * Run a query at the latest commit as a subscription runs it, under `caller`,
+   * and return its answer with every dependency it took. `time` pins the time
+   * it reads, the kernel's time now by default. `journal` ends its paginated
+   * read where an earlier run's ended.
+   */
+  subscribed(
+    caller: Caller,
+    reference: any,
+    args: any,
+    pinned: { readonly time?: number; readonly journal: string | null },
+  ): Promise<QueryRun> {
+    return this.open().then(() => {
+      const functionPath = pathOfReference(reference, ROOT);
+      const resolved = this.resolve(functionPath, "query");
+      validateFunctionArgs(resolved.args, args ?? {}, functionPath, false);
+      const top: TopQuery = {
+        attempt: new KernelAttempt(this.state),
+        pagination: { calls: 0, journal: pinned.journal, end: null },
+        startTime: pinned.time,
+      };
+      const run = (outcome: Outcome): QueryRun => ({
+        outcome,
+        snapshot: top.attempt.snapshot.version,
+        reads: top.attempt.reads,
+        observedIdentity: top.attempt.observedIdentity,
+        time: top.attempt.observedTime,
+        journal: pinned.journal ?? top.pagination.end,
+      });
+      return this.runQuery(
+        functionPath,
+        resolved,
+        args,
+        { caller, request: newRequest(caller) },
+        false,
+        top,
+      ).then(
+        (value: Value) => run({ _tag: "returned", value }),
+        (error: unknown) => run({ _tag: "failed", error }),
+      );
+    });
+  }
+
   // Scheduled functions, on the kernel's clock.
 
   /**
@@ -828,7 +928,26 @@ export class World {
         `Invalid arguments for "${target.udfPath}": a scheduled function cannot take an unresolved placeholder value.`,
       );
     }
+    refuse(attempt.meter.schedule(jobArgs, execution.limits));
+    const jobsIndex = indexKey(
+      execution.componentPath,
+      Scheduling.table,
+      "by_id",
+    );
+    // A scheduled function that schedules reads its own job first
+    // (`SchedulerModel::schedule`, `crates/model/src/scheduled_jobs/mod.rs:238-300`).
+    const parentJob = execution.request.scheduledFunctionId;
+    if (parentJob !== null) {
+      refuse(
+        attempt.meter.readInterval(
+          jobsIndex,
+          Metering.point([parentJob]),
+          execution.limits,
+        ),
+      );
+    }
     const id = this.generateId(Scheduling.table);
+    attempt.meter.deriveInterval(jobsIndex, Metering.point([id]));
     attempt.write(
       execution.componentPath,
       id,
@@ -855,10 +974,10 @@ export class World {
   /**
    * Cancel a job that has not finished. A finished job stays as it ended.
    */
-  private cancelJob(attempt: KernelAttempt, componentPath: string, id: any) {
+  private cancelJob(attempt: KernelAttempt, execution: Execution, id: any) {
+    const { componentPath } = execution;
     validateId(Scheduling.table, id);
-    attempt.readDocument(componentPath, id);
-    const doc = attempt.get(componentPath, id);
+    const doc = this.readById(attempt, execution, id);
     if (doc === null) {
       throw new Error(`Patch on non-existent document with ID "${id}"`);
     }
@@ -914,7 +1033,8 @@ export class World {
     udfType: UdfType,
     attempt: KernelAttempt | undefined,
     from: From,
-    pagination: { calls: number } | undefined,
+    pagination: Pagination | undefined,
+    startTime?: number,
   ): Execution {
     // A function in a component runs with no identity, whoever called it, as
     // Convex gives a component no `ctx.auth` of the caller's. A nested call in
@@ -925,6 +1045,18 @@ export class World {
         : from.parent !== undefined
           ? from.parent.identity
           : from.caller.identity;
+    // A nested call runs under its caller's limits, which its budget can only
+    // tighten, and gives them back when it ends (`run_udf`,
+    // `crates/isolate/src/environment/udf/async_syscall.rs:700-706`, and
+    // `commit_subtransaction`, `crates/database/src/transaction.rs:431-441`).
+    const inherited =
+      attempt === undefined || from.parent?.attempt === undefined
+        ? Metering.defaults
+        : from.parent.limits;
+    const limits =
+      attempt === undefined || from.budget === undefined
+        ? inherited
+        : Metering.fromBudget(from.budget, attempt.meter.used(), inherited);
     return {
       world: this,
       componentPath: functionPath.componentPath,
@@ -934,11 +1066,13 @@ export class World {
       identity,
       request: from.request,
       startTime:
-        from.parent !== undefined && from.parent.udfType !== "action"
+        startTime ??
+        (from.parent !== undefined && from.parent.udfType !== "action"
           ? from.parent.startTime
-          : this.now(),
+          : this.now()),
       pagination,
       turn: new Turn(),
+      limits,
     };
   }
 
@@ -948,21 +1082,29 @@ export class World {
     args: any,
     from: From,
     snapshotOnly = false,
+    top?: TopQuery,
   ): Promise<any> {
     const parentAttempt = from.parent?.attempt;
     const isNested = parentAttempt !== undefined;
     const attempt =
-      parentAttempt === undefined
+      top?.attempt ??
+      (parentAttempt === undefined
         ? new KernelAttempt(this.state)
         : snapshotOnly
-          ? new KernelAttempt(parentAttempt.snapshot)
-          : parentAttempt;
+          ? parentAttempt.staleSnapshot()
+          : parentAttempt);
     const execution = this.execution(
       functionPath,
       "query",
       attempt,
       from,
-      (isNested && from.parent?.pagination) || { calls: 0 },
+      top?.pagination ??
+        ((isNested && from.parent?.pagination) || {
+          calls: 0,
+          journal: null,
+          end: null,
+        }),
+      top?.startTime,
     );
     const q = queryGeneric({
       handler: (ctx: any, a: any) =>
@@ -1278,18 +1420,83 @@ export class World {
 
   private view(execution: Execution): KernelQuery.TableView {
     const attempt = transaction(execution);
+    const { componentPath, limits } = execution;
     return {
-      schema: this.schema(execution.componentPath),
-      scan: (tableName) => attempt.scan(execution.componentPath, tableName),
-      read: (interval, after) =>
-        attempt.readInterval({
+      schema: this.schema(componentPath),
+      scan: (tableName) => attempt.scan(componentPath, tableName),
+      read: ({ index, ...interval }, after) => {
+        const advance = attempt.readInterval({
           ...interval,
-          componentPath: execution.componentPath,
+          componentPath,
           after,
-        }),
+        });
+        return (through) => {
+          advance(through);
+          const examined = Metering.examined(
+            interval.range,
+            interval.order,
+            after,
+            through,
+          );
+          if (examined !== null) {
+            refuse(
+              attempt.meter.readInterval(
+                indexKey(componentPath, interval.table, index),
+                examined,
+                limits,
+              ),
+            );
+          }
+        };
+      },
+      examine: (table, doc) =>
+        refuse(attempt.meter.readDocument(doc, isSystemTable(table), limits)),
     };
   }
 
+  /**
+   * Read one document by ID, as the binary's `get_inner` does for a get and for
+   * the document a patch, replace, delete or cancel changes: it reads the ID's
+   * interval of the table's `by_id` index, then charges the document if there
+   * is one (`crates/database/src/transaction.rs:1188-1250`).
+   */
+  private readById(
+    attempt: KernelAttempt,
+    execution: Execution,
+    id: DocumentId,
+  ): StoredDocument | null {
+    const { componentPath, limits } = execution;
+    const table = tableNameFromId(id)!;
+    attempt.readDocument(componentPath, id);
+    refuse(
+      attempt.meter.readInterval(
+        indexKey(componentPath, table, "by_id"),
+        Metering.point([id]),
+        limits,
+      ),
+    );
+    const doc = attempt.get(componentPath, id);
+    if (doc !== null) {
+      refuse(attempt.meter.readDocument(doc, isSystemTable(table), limits));
+    }
+    return doc;
+  }
+
+  /**
+   * Stage a write to a user table, counted against the execution's limits
+   * first. A write over a limit is not staged (`Writes::update`,
+   * `crates/database/src/writes.rs:241-310`).
+   */
+  private stage(
+    execution: Execution,
+    id: DocumentId,
+    doc: StoredDocument | null,
+  ) {
+    const attempt = transaction(execution);
+    refuse(attempt.meter.write(doc, execution.limits));
+    attempt.write(execution.componentPath, id, doc);
+  }
+
   private generateId(table: string): DocumentId {
     const counter = String(this.nextDocId++).padStart(
       TYPICAL_CONVEX_ID_LENGTH - table.length,
@@ -1393,9 +1600,12 @@ export class World {
             );
           }
         }
-        return JSON.stringify(
-          convexToJson(KernelQuery.paginate(this.view(execution), args) as any),
-        );
+        const page = KernelQuery.paginate(this.view(execution), {
+          ...args,
+          endCursor: args.endCursor ?? pagination?.journal ?? null,
+        });
+        if (pagination) pagination.end = page.continueCursor;
+        return JSON.stringify(convexToJson(page as any));
       }
       case "1.0/insert":
         return JSON.stringify({
@@ -1412,6 +1622,15 @@ export class World {
         return JSON.stringify({});
       case "1.0/count": {
         const attempt = transaction(execution);
+        // The binary counts a table by reading its whole `by_id` index, and no
+        // document (`count_tablet`, `crates/database/src/bootstrap_model/table.rs:133-160`).
+        refuse(
+          attempt.meter.readInterval(
+            indexKey(execution.componentPath, args.table, "by_id"),
+            Metering.everything,
+            execution.limits,
+          ),
+        );
         attempt.readInterval({
           kind: "interval",
           componentPath: execution.componentPath,
@@ -1458,8 +1677,12 @@ export class World {
         return run.then((value) => JSON.stringify(convexToJson(value)));
       }
       case "1.0/runUdf": {
-        const { udfType, args: udfArgs } = args;
-        const from: From = { request: execution.request, parent: execution };
+        const { udfType, args: udfArgs, transactionLimits } = args;
+        const from: From = {
+          request: execution.request,
+          parent: execution,
+          budget: budgetOf(transactionLimits),
+        };
         const functionPath = functionPathOf(args, execution.componentPath);
         const value = jsonToConvex(udfArgs);
         const run =
@@ -1498,15 +1721,11 @@ export class World {
           ),
         );
       case "1.0/cancel_job":
-        this.cancelJob(
-          transaction(execution),
-          execution.componentPath,
-          args.id,
-        );
+        this.cancelJob(transaction(execution), execution, args.id);
         return JSON.stringify({});
       case "1.0/actions/cancel_job":
         this.commitNow((attempt) =>
-          this.cancelJob(attempt, execution.componentPath, args.id),
+          this.cancelJob(attempt, execution, args.id),
         );
         return JSON.stringify({});
       case "1.0/storageDelete":
@@ -1514,7 +1733,14 @@ export class World {
       case "1.0/storageGenerateUploadUrl":
         throw KernelRefusal.rowPending("storage", 6);
       case "1.0/getTransactionMetrics":
-        throw KernelRefusal.rowPending("transaction limits", 6);
+        // An action has no transaction, and the binary's action syscalls do
+        // not know this one (`crates/isolate/src/environment/action/async_syscall.rs:105`).
+        if (execution.attempt === undefined) {
+          throw new Error(`Unknown async operation ${op}`);
+        }
+        return JSON.stringify(
+          execution.attempt.meter.metrics(execution.limits),
+        );
       case "1.0/actions/vectorSearch":
         throw KernelRefusal.requiresBinary("vectorSearch");
       default:
@@ -1536,9 +1762,7 @@ export class World {
 
   private get(execution: Execution, tableName: unknown, id: unknown) {
     validateId(tableName, id);
-    const attempt = transaction(execution);
-    attempt.readDocument(execution.componentPath, id);
-    return attempt.get(execution.componentPath, id);
+    return this.readById(transaction(execution), execution, id);
   }
 
   private insert(execution: Execution, table: string, value: any): DocumentId {
@@ -1549,7 +1773,12 @@ export class World {
       value,
     );
     const _id = this.generateId(table);
-    attempt.write(execution.componentPath, _id, {
+    // A new ID joins the table's `by_id` reads without counting as one.
+    attempt.meter.deriveInterval(
+      indexKey(execution.componentPath, table, "by_id"),
+      Metering.point([_id]),
+    );
+    this.stage(execution, _id, {
       ...value,
       _id,
       _creationTime: this.creationTime(execution.startTime),
@@ -1581,9 +1810,7 @@ export class World {
         `Invalid argument \`value\` in \`db.${write}\`, expected object but got '${typeof value}': ${value as any}`,
       );
     }
-    const attempt = transaction(execution);
-    attempt.readDocument(execution.componentPath, id);
-    const document = attempt.get(execution.componentPath, id);
+    const document = this.readById(transaction(execution), execution, id);
     if (document === null) {
       throw new Error(
         write === "delete"
@@ -1628,16 +1855,14 @@ export class World {
     const document = this.existing(execution, tableName, id, "patch", value);
     const { _id, _creationTime, ...fields } = document;
     const merged = { ...fields, ...this.fields(document, value) };
+    const next = { _id, _creationTime, ...merged } as StoredDocument;
+    if (sameDocument(document, next)) return;
     KernelQuery.validateDocument(
       this.schema(execution.componentPath),
       tableNameFromId(_id)!,
       merged,
     );
-    transaction(execution).write(execution.componentPath, _id, {
-      _id,
-      _creationTime,
-      ...merged,
-    } as StoredDocument);
+    this.stage(execution, _id, next);
   }
 
   private replace(
@@ -1648,16 +1873,18 @@ export class World {
   ) {
     const document = this.existing(execution, tableName, id, "replace", value);
     const fields = this.fields(document, value);
+    const next = {
+      ...fields,
+      _id: document._id,
+      _creationTime: document._creationTime,
+    } as StoredDocument;
+    if (sameDocument(document, next)) return;
     KernelQuery.validateDocument(
       this.schema(execution.componentPath),
       tableNameFromId(document._id)!,
       fields,
     );
-    transaction(execution).write(execution.componentPath, document._id, {
-      ...fields,
-      _id: document._id,
-      _creationTime: document._creationTime,
-    } as StoredDocument);
+    this.stage(execution, document._id, next);
   }
 
   private remove(execution: Execution, tableName: unknown, id: unknown) {
@@ -1668,7 +1895,7 @@ export class World {
       "delete",
       undefined,
     );
-    transaction(execution).write(execution.componentPath, document._id, null);
+    this.stage(execution, document._id, null);
   }
 }
 
@@ -1681,8 +1908,99 @@ type From =
       readonly caller: Caller;
       readonly request: Request;
       readonly parent?: undefined;
+      readonly budget?: undefined;
     }
-  | { readonly parent: Execution; readonly request: Request };
+  | {
+      readonly parent: Execution;
+      readonly request: Request;
+      readonly budget?: Metering.Budget | undefined;
+    };
+
+const refuse = (error: Metering.TransactionLimitExceeded | undefined) => {
+  if (error !== undefined) throw error;
+};
+
+/**
+ * One index of one table in one component, as the binary keys a read set.
+ */
+const indexKey = (componentPath: string, table: string, index: string) =>
+  `${componentPath}\u0000${table}\u0000${index}`;
+
+const isSystemTable = (table: string) => table.startsWith("_");
+
+const isFieldMap = (value: unknown): value is object =>
+  typeof value === "object" && value !== null && isSimpleObject(value);
+
+/**
+ * Whether two stored values are the same value, an unresolved commit timestamp
+ * included. The binary compares pending values structurally, so a pending
+ * timestamp equals only another pending timestamp, at any depth (`PendingValue`
+ * derives `PartialEq`, `crates/value/src/pending.rs:56-75`, as
+ * `PendingDocument` does, `crates/common/src/document.rs:624-632`). Concrete
+ * values compare by `ConvexValue`'s total order
+ * (`crates/value/src/sorting.rs:468-472`), which `compareValues` follows.
+ * `compareValues` alone ranks a pending timestamp as the largest int64, so it
+ * cannot tell the two apart. An object field set to `undefined` is absent.
+ */
+const sameValue = (a: unknown, b: unknown): boolean => {
+  const aPending = a instanceof CommitTsPlaceholder;
+  const bPending = b instanceof CommitTsPlaceholder;
+  if (aPending || bPending) return aPending && bPending;
+  if (Array.isArray(a) || Array.isArray(b)) {
+    return (
+      Array.isArray(a) &&
+      Array.isArray(b) &&
+      a.length === b.length &&
+      a.every((value, i) => sameValue(value, b[i]))
+    );
+  }
+  if (isFieldMap(a) || isFieldMap(b)) {
+    if (!isFieldMap(a) || !isFieldMap(b)) return false;
+    const fields = (value: object) =>
+      Object.entries(value).filter(([, field]) => field !== undefined);
+    const aFields = fields(a);
+    const bFields = new Map(fields(b));
+    return (
+      aFields.length === bFields.size &&
+      aFields.every(
+        ([key, field]) =>
+          bFields.has(key) && sameValue(field, bFields.get(key)),
+      )
+    );
+  }
+  return compareValues(a as Value, b as Value) === 0;
+};
+
+/**
+ * Whether a patch or replace leaves a document as it was. The binary then
+ * writes nothing (`patch_inner` and `replace_inner`,
+ * `crates/database/src/transaction.rs:713-790`).
+ */
+const sameDocument = (before: StoredDocument, after: StoredDocument) =>
+  sameValue(before, after);
+
+/**
+ * A nested call's `transactionLimits`. The binary reads each field as an
+ * unsigned integer and ignores fields it does not know (`TransactionLimits`,
+ * `crates/database/src/execution_size.rs:62-95`).
+ */
+const budgetOf = (limits: unknown): Metering.Budget | undefined => {
+  if (limits === undefined || limits === null) return undefined;
+  const budget: Partial<Record<keyof Metering.Limits, number>> = {};
+  for (const metric of Object.keys(Metering.defaults) as Array<
+    keyof Metering.Limits
+  >) {
+    const value = (limits as Record<string, unknown>)[metric];
+    if (value === undefined) continue;
+    if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
+      throw new Error(
+        `Invalid arguments for \`runUdf\`: transactionLimits.${metric} must be a non-negative integer, got ${JSON.stringify(value)}`,
+      );
+    }
+    budget[metric] = value;
+  }
+  return budget;
+};
 
 const closed = (execution: Execution, operation: string) =>
   new KernelRefusal.ClosedAttempt({
@@ -1700,7 +2018,10 @@ const transaction = (execution: Execution): KernelAttempt => {
 };
 
 const queryAuth = (execution: Execution) => ({
-  getUserIdentity: () => Promise.resolve(execution.identity),
+  getUserIdentity: () => {
+    execution.attempt?.observeIdentity();
+    return Promise.resolve(execution.identity);
+  },
 });
 
 const mutationAuth = (execution: Execution) => ({

package/src/internal/Invocation.ts

@@ -214,7 +214,8 @@ const pin = (
   const host = globalThis;
   const previous = { Date: host.Date, Math: host.Math, crypto: host.crypto };
   // On the kernel, `Date` already answers the running function's time on the
-  // kernel's clock, so `runAfter` schedules on it. The bridge leaves it there.
+  // kernel's clock, so `runAfter` schedules on it, and a query that reads it
+  // records the read. The bridge leaves it there.
   const kernelDate = Clock.answersInvocation(previous.Date);
   const pinned = {
     Date: kernelDate