@rocicorp/zero 0.25.0-canary.7 → 0.25.0-canary.8

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -1,281 +1,274 @@
-import { PG_SERIALIZATION_FAILURE } from '@drdgvhbh/postgres-error-codes';
-import { resolver } from '@rocicorp/resolver';
-import postgres from 'postgres';
+import { PG_SERIALIZATION_FAILURE } from "@drdgvhbh/postgres-error-codes";
+import { resolver } from "@rocicorp/resolver";
+import postgres from "postgres";
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
-import * as v from "../../../../shared/src/valita.js";
-import { ErrorKind } from "../../../../zero-protocol/src/error-kind.js";
-import * as MutationType from "../../../../zero-protocol/src/mutation-type-enum.js";
-import { primaryKeyValueSchema, } from "../../../../zero-protocol/src/primary-key.js";
-import {} from "../../../../zero-protocol/src/push.js";
+import { parse } from "../../../../shared/src/valita.js";
+import { MutationRateLimited, InvalidPush, MutationFailed } from "../../../../zero-protocol/src/error-kind-enum.js";
+import { CRUD } from "../../../../zero-protocol/src/mutation-type-enum.js";
+import { primaryKeyValueSchema } from "../../../../zero-protocol/src/primary-key.js";
+import "../../../../zero-protocol/src/push.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import { WriteAuthorizerImpl, } from "../../auth/write-authorizer.js";
-import {} from "../../config/zero-config.js";
-import * as Mode from "../../db/mode-enum.js";
+import { WriteAuthorizerImpl } from "../../auth/write-authorizer.js";
+import "../../config/zero-config.js";
+import { SERIALIZABLE } from "../../db/mode-enum.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
 import { throwProtocolErrorIfSchemaVersionNotSupported } from "../../types/schema-versions.js";
-import { appSchema, upstreamSchema } from "../../types/shards.js";
+import { upstreamSchema, appSchema } from "../../types/shards.js";
 import { SlidingWindowLimiter } from "../limiter/sliding-window-limiter.js";
 import { MutationAlreadyProcessedError } from "./error.js";
-import { isProtocolError, ProtocolError, } from "../../../../zero-protocol/src/error.js";
-import { ErrorOrigin } from "../../../../zero-protocol/src/error-origin.js";
-export class MutagenService {
-    id;
-    #lc;
-    #upstream;
-    #shard;
-    #stopped = resolver();
-    #replica;
-    #writeAuthorizer;
-    #limiter;
-    #refCount = 0;
-    #isStopped = false;
-    #crudMutations = getOrCreateCounter('mutation', 'crud', 'Number of CRUD mutations processed');
-    constructor(lc, shard, clientGroupID, upstream, config) {
-        this.id = clientGroupID;
-        this.#lc = lc;
-        this.#upstream = upstream;
-        this.#shard = shard;
-        this.#replica = new Database(this.#lc, config.replica.file, {
-            fileMustExist: true,
-        });
-        this.#writeAuthorizer = new WriteAuthorizerImpl(this.#lc, config, this.#replica, shard.appID, clientGroupID);
-        if (config.perUserMutationLimit.max !== undefined) {
-            this.#limiter = new SlidingWindowLimiter(config.perUserMutationLimit.windowMs, config.perUserMutationLimit.max);
-        }
+import { isProtocolError, ProtocolError } from "../../../../zero-protocol/src/error.js";
+import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
+class MutagenService {
+  id;
+  #lc;
+  #upstream;
+  #shard;
+  #stopped = resolver();
+  #replica;
+  #writeAuthorizer;
+  #limiter;
+  #refCount = 0;
+  #isStopped = false;
+  #crudMutations = getOrCreateCounter(
+    "mutation",
+    "crud",
+    "Number of CRUD mutations processed"
+  );
+  constructor(lc, shard, clientGroupID, upstream, config) {
+    this.id = clientGroupID;
+    this.#lc = lc;
+    this.#upstream = upstream;
+    this.#shard = shard;
+    this.#replica = new Database(this.#lc, config.replica.file, {
+      fileMustExist: true
+    });
+    this.#writeAuthorizer = new WriteAuthorizerImpl(
+      this.#lc,
+      config,
+      this.#replica,
+      shard.appID,
+      clientGroupID
+    );
+    if (config.perUserMutationLimit.max !== void 0) {
+      this.#limiter = new SlidingWindowLimiter(
+        config.perUserMutationLimit.windowMs,
+        config.perUserMutationLimit.max
+      );
     }
-    ref() {
-        assert(!this.#isStopped, 'MutagenService is already stopped');
-        ++this.#refCount;
-    }
-    unref() {
-        assert(!this.#isStopped, 'MutagenService is already stopped');
-        --this.#refCount;
-        if (this.#refCount <= 0) {
-            void this.stop();
-        }
-    }
-    hasRefs() {
-        return this.#refCount > 0;
-    }
-    processMutation(mutation, authData, schemaVersion, customMutatorsEnabled = false) {
-        if (this.#limiter?.canDo() === false) {
-            return Promise.resolve([
-                ErrorKind.MutationRateLimited,
-                'Rate limit exceeded',
-            ]);
-        }
-        this.#crudMutations.add(1, {
-            clientGroupID: this.id,
-        });
-        return processMutation(this.#lc, authData, this.#upstream, this.#shard, this.id, mutation, this.#writeAuthorizer, schemaVersion, undefined, customMutatorsEnabled);
+  }
+  ref() {
+    assert(!this.#isStopped, "MutagenService is already stopped");
+    ++this.#refCount;
+  }
+  unref() {
+    assert(!this.#isStopped, "MutagenService is already stopped");
+    --this.#refCount;
+    if (this.#refCount <= 0) {
+      void this.stop();
     }
-    run() {
-        return this.#stopped.promise;
+  }
+  hasRefs() {
+    return this.#refCount > 0;
+  }
+  processMutation(mutation, authData, schemaVersion, customMutatorsEnabled = false) {
+    if (this.#limiter?.canDo() === false) {
+      return Promise.resolve([
+        MutationRateLimited,
+        "Rate limit exceeded"
+      ]);
     }
-    stop() {
-        if (this.#isStopped) {
-            return this.#stopped.promise;
-        }
-        this.#writeAuthorizer.destroy();
-        this.#isStopped = true;
-        this.#stopped.resolve();
-        return this.#stopped.promise;
+    this.#crudMutations.add(1, {
+      clientGroupID: this.id
+    });
+    return processMutation(
+      this.#lc,
+      authData,
+      this.#upstream,
+      this.#shard,
+      this.id,
+      mutation,
+      this.#writeAuthorizer,
+      schemaVersion,
+      void 0,
+      customMutatorsEnabled
+    );
+  }
+  run() {
+    return this.#stopped.promise;
+  }
+  stop() {
+    if (this.#isStopped) {
+      return this.#stopped.promise;
     }
+    this.#writeAuthorizer.destroy();
+    this.#isStopped = true;
+    this.#stopped.resolve();
+    return this.#stopped.promise;
+  }
 }
 const MAX_SERIALIZATION_ATTEMPTS = 10;
-export async function processMutation(lc, authData, db, shard, clientGroupID, mutation, writeAuthorizer, schemaVersion, onTxStart, // for testing
-customMutatorsEnabled = false) {
-    assert(mutation.type === MutationType.CRUD, 'Only CRUD mutations are supported');
-    lc = lc.withContext('mutationID', mutation.id);
-    lc = lc.withContext('processMutation');
-    lc.debug?.('Process mutation start', mutation);
-    // Record mutation processing attempt for telemetry (regardless of success/failure)
-    recordMutation('crud');
-    let result;
-    const start = Date.now();
-    try {
-        // Mutations can fail for a variety of reasons:
-        //
-        // - application error
-        // - network/db error
-        // - zero bug
-        //
-        // For application errors what we want is to re-run the mutation in
-        // "error mode", which skips the actual mutation and just updates the
-        // lastMutationID. Then return the error to the app.
-        //
-        // However, it's hard to tell the difference between application errors
-        // and the other types.
-        //
-        // A reasonable policy ends up being to just retry every mutation once
-        // in error mode. If the error mode mutation succeeds then we assume it
-        // was an application error and return the error to the app. Otherwise,
-        // we know it was something internal and we log it.
-        //
-        // This is not 100% correct - there are theoretical cases where we
-        // return an internal error to the app that shouldn't have been. But it
-        // would have to be a crazy coincidence: we'd have to have a network
-        // error on the first attempt that resolves by the second attempt.
-        //
-        // One might ask why not try/catch just the calls to the mutators and
-        // consider those application errors. That is actually what we do in
-        // Replicache:
-        //
-        // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131
-        //
-        // We don't do it here because:
-        //
-        // 1. It's still not perfect. It's hard to isolate SQL errors in
-        //    mutators due to app developer mistakes from SQL errors due to
-        //    Zero mistakes.
-        // 2. It's not possible to do this with the pg library we're using in
-        //    Zero anyway: https://github.com/porsager/postgres/issues/455.
-        //
-        // Personally I think this simple retry policy is nice.
-        let errorMode = false;
-        for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {
-            try {
-                await db.begin(Mode.SERIALIZABLE, async (tx) => {
-                    // Simulates a concurrent request for testing. In production this is a noop.
-                    const done = onTxStart?.();
-                    try {
-                        return await processMutationWithTx(lc, tx, authData, shard, clientGroupID, schemaVersion, mutation, errorMode, writeAuthorizer);
-                    }
-                    finally {
-                        await done;
-                    }
-                });
-                if (errorMode) {
-                    lc.debug?.('Ran mutation successfully in error mode');
-                }
-                break;
-            }
-            catch (e) {
-                if (e instanceof MutationAlreadyProcessedError) {
-                    lc.debug?.(e.message);
-                    // Don't double-count already processed mutations, but they were counted above
-                    return undefined;
-                }
-                if (isProtocolError(e) &&
-                    !errorMode &&
-                    e.kind === ErrorKind.InvalidPush &&
-                    customMutatorsEnabled &&
-                    i < 2) {
-                    // We're temporarily supporting custom mutators AND CRUD mutators at the same time.
-                    // This can create a lot of OOO mutation errors since we do not know when the API server
-                    // has applied a custom mutation before moving on to process CRUD mutations.
-                    // The temporary workaround (since CRUD is being deprecated) is to retry the mutation
-                    // after a small delay. Users are not expected to be running both CRUD and Custom mutators.
-                    // They should migrate completely to custom mutators.
-                    lc.info?.('Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.', e);
-                    await new Promise(resolve => setTimeout(resolve, 100));
-                    continue;
-                }
-                if (isProtocolError(e) || errorMode) {
-                    lc.error?.('Process mutation error', e);
-                    throw e;
-                }
-                if (e instanceof postgres.PostgresError &&
-                    e.code === PG_SERIALIZATION_FAILURE) {
-                    lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);
-                    continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.
-                }
-                result = [ErrorKind.MutationFailed, String(e)];
-                if (errorMode) {
-                    break;
-                }
-                lc.error?.('Got error running mutation, re-running in error mode', e);
-                errorMode = true;
-                i--;
-            }
+async function processMutation(lc, authData, db, shard, clientGroupID, mutation, writeAuthorizer, schemaVersion, onTxStart, customMutatorsEnabled = false) {
+  assert(
+    mutation.type === CRUD,
+    "Only CRUD mutations are supported"
+  );
+  lc = lc.withContext("mutationID", mutation.id);
+  lc = lc.withContext("processMutation");
+  lc.debug?.("Process mutation start", mutation);
+  recordMutation("crud");
+  let result;
+  const start = Date.now();
+  try {
+    let errorMode = false;
+    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {
+      try {
+        await db.begin(SERIALIZABLE, async (tx) => {
+          const done = onTxStart?.();
+          try {
+            return await processMutationWithTx(
+              lc,
+              tx,
+              authData,
+              shard,
+              clientGroupID,
+              schemaVersion,
+              mutation,
+              errorMode,
+              writeAuthorizer
+            );
+          } finally {
+            await done;
+          }
+        });
+        if (errorMode) {
+          lc.debug?.("Ran mutation successfully in error mode");
         }
-    }
-    finally {
-        lc.debug?.('Process mutation complete in', Date.now() - start);
-    }
-    return result;
-}
-export async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, schemaVersion, mutation, errorMode, authorizer) {
-    const tasks = [];
-    async function execute(stmt) {
-        try {
-            return await stmt.execute();
+        break;
+      } catch (e) {
+        if (e instanceof MutationAlreadyProcessedError) {
+          lc.debug?.(e.message);
+          return void 0;
         }
-        finally {
-            const q = stmt;
-            lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);
+        if (isProtocolError(e) && !errorMode && e.kind === InvalidPush && customMutatorsEnabled && i < 2) {
+          lc.info?.(
+            "Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.",
+            e
+          );
+          await new Promise((resolve) => setTimeout(resolve, 100));
+          continue;
         }
+        if (isProtocolError(e) || errorMode) {
+          lc.error?.("Process mutation error", e);
+          throw e;
+        }
+        if (e instanceof postgres.PostgresError && e.code === PG_SERIALIZATION_FAILURE) {
+          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);
+          continue;
+        }
+        result = [MutationFailed, String(e)];
+        if (errorMode) {
+          break;
+        }
+        lc.error?.("Got error running mutation, re-running in error mode", e);
+        errorMode = true;
+        i--;
+      }
     }
-    authorizer.reloadPermissions();
-    if (!errorMode) {
-        const { ops } = mutation.args[0];
-        const normalizedOps = authorizer.normalizeOps(ops);
-        const [canPre, canPost] = await Promise.all([
-            authorizer.canPreMutation(authData, normalizedOps),
-            authorizer.canPostMutation(authData, normalizedOps),
-        ]);
-        if (canPre && canPost) {
-            for (const op of ops) {
-                switch (op.op) {
-                    case 'insert':
-                        tasks.push(() => execute(getInsertSQL(tx, op)));
-                        break;
-                    case 'upsert':
-                        tasks.push(() => execute(getUpsertSQL(tx, op)));
-                        break;
-                    case 'update':
-                        tasks.push(() => execute(getUpdateSQL(tx, op)));
-                        break;
-                    case 'delete':
-                        tasks.push(() => execute(getDeleteSQL(tx, op)));
-                        break;
-                    default:
-                        unreachable(op);
-                }
-            }
+  } finally {
+    lc.debug?.("Process mutation complete in", Date.now() - start);
+  }
+  return result;
+}
+async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, schemaVersion, mutation, errorMode, authorizer) {
+  const tasks = [];
+  async function execute(stmt) {
+    try {
+      return await stmt.execute();
+    } finally {
+      const q = stmt;
+      lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);
+    }
+  }
+  authorizer.reloadPermissions();
+  if (!errorMode) {
+    const { ops } = mutation.args[0];
+    const normalizedOps = authorizer.normalizeOps(ops);
+    const [canPre, canPost] = await Promise.all([
+      authorizer.canPreMutation(authData, normalizedOps),
+      authorizer.canPostMutation(authData, normalizedOps)
+    ]);
+    if (canPre && canPost) {
+      for (const op of ops) {
+        switch (op.op) {
+          case "insert":
+            tasks.push(() => execute(getInsertSQL(tx, op)));
+            break;
+          case "upsert":
+            tasks.push(() => execute(getUpsertSQL(tx, op)));
+            break;
+          case "update":
+            tasks.push(() => execute(getUpdateSQL(tx, op)));
+            break;
+          case "delete":
+            tasks.push(() => execute(getDeleteSQL(tx, op)));
+            break;
+          default:
+            unreachable();
         }
+      }
     }
-    // Confirm the mutation even though it may have been blocked by the authorizer.
-    // Authorizer blocking a mutation is not an error but the correct result of the mutation.
-    tasks.unshift(() => checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGroupID, schemaVersion, mutation.clientID, mutation.id));
-    // Note: An error thrown from any Promise aborts the entire transaction.
-    await Promise.all(tasks.map(task => task()));
+  }
+  tasks.unshift(
+    () => checkSchemaVersionAndIncrementLastMutationID(
+      tx,
+      shard,
+      clientGroupID,
+      schemaVersion,
+      mutation.clientID,
+      mutation.id
+    )
+  );
+  await Promise.all(tasks.map((task) => task()));
 }
-export function getInsertSQL(tx, create) {
-    return tx `INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;
+function getInsertSQL(tx, create) {
+  return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;
 }
-export function getUpsertSQL(tx, set) {
-    const { tableName, primaryKey, value } = set;
-    return tx `
+function getUpsertSQL(tx, set) {
+  const { tableName, primaryKey, value } = set;
+  return tx`
     INSERT INTO ${tx(tableName)} ${tx(value)}
     ON CONFLICT (${tx(primaryKey)})
     DO UPDATE SET ${tx(value)}
   `;
 }
 function getUpdateSQL(tx, update) {
-    const table = update.tableName;
-    const { primaryKey, value } = update;
-    const id = {};
-    for (const key of primaryKey) {
-        id[key] = v.parse(value[key], primaryKeyValueSchema);
-    }
-    return tx `UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(id).flatMap(([key, value], i) => i ? [tx `AND`, tx `${tx(key)} = ${value}`] : tx `${tx(key)} = ${value}`)}`;
+  const table = update.tableName;
+  const { primaryKey, value } = update;
+  const id = {};
+  for (const key of primaryKey) {
+    id[key] = parse(value[key], primaryKeyValueSchema);
+  }
+  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(
+    id
+  ).flatMap(
+    ([key, value2], i) => i ? [tx`AND`, tx`${tx(key)} = ${value2}`] : tx`${tx(key)} = ${value2}`
+  )}`;
 }
 function getDeleteSQL(tx, deleteOp) {
-    const { tableName, primaryKey, value } = deleteOp;
-    const conditions = [];
-    for (const key of primaryKey) {
-        if (conditions.length > 0) {
-            conditions.push(tx `AND`);
-        }
-        conditions.push(tx `${tx(key)} = ${value[key]}`);
+  const { tableName, primaryKey, value } = deleteOp;
+  const conditions = [];
+  for (const key of primaryKey) {
+    if (conditions.length > 0) {
+      conditions.push(tx`AND`);
     }
-    return tx `DELETE FROM ${tx(tableName)} WHERE ${conditions}`;
+    conditions.push(tx`${tx(key)} = ${value[key]}`);
+  }
+  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;
 }
 async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGroupID, schemaVersion, clientID, receivedMutationID) {
-    const [[{ lastMutationID }], supportedVersionRange] = await Promise.all([
-        tx `
+  const [[{ lastMutationID }], supportedVersionRange] = await Promise.all([
+    tx`
     INSERT INTO ${tx(upstreamSchema(shard))}.clients 
       as current ("clientGroupID", "clientID", "lastMutationID")
           VALUES (${clientGroupID}, ${clientID}, ${1})
@@ -283,25 +276,35 @@ async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGro
       DO UPDATE SET "lastMutationID" = current."lastMutationID" + 1
       RETURNING "lastMutationID"
   `,
-        schemaVersion === undefined
-            ? undefined
-            : tx `SELECT "minSupportedVersion", "maxSupportedVersion" 
-        FROM ${tx(appSchema(shard))}."schemaVersions"`,
-    ]);
-    // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.
-    if (receivedMutationID < lastMutationID) {
-        throw new MutationAlreadyProcessedError(clientID, receivedMutationID, lastMutationID);
-    }
-    else if (receivedMutationID > lastMutationID) {
-        throw new ProtocolError({
-            kind: ErrorKind.InvalidPush,
-            message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,
-            origin: ErrorOrigin.ZeroCache,
-        });
-    }
-    if (schemaVersion !== undefined && supportedVersionRange !== undefined) {
-        assert(supportedVersionRange.length === 1);
-        throwProtocolErrorIfSchemaVersionNotSupported(schemaVersion, supportedVersionRange[0]);
-    }
+    schemaVersion === void 0 ? void 0 : tx`SELECT "minSupportedVersion", "maxSupportedVersion" 
+        FROM ${tx(appSchema(shard))}."schemaVersions"`
+  ]);
+  if (receivedMutationID < lastMutationID) {
+    throw new MutationAlreadyProcessedError(
+      clientID,
+      receivedMutationID,
+      lastMutationID
+    );
+  } else if (receivedMutationID > lastMutationID) {
+    throw new ProtocolError({
+      kind: InvalidPush,
+      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,
+      origin: ZeroCache
+    });
+  }
+  if (schemaVersion !== void 0 && supportedVersionRange !== void 0) {
+    assert(supportedVersionRange.length === 1);
+    throwProtocolErrorIfSchemaVersionNotSupported(
+      schemaVersion,
+      supportedVersionRange[0]
+    );
+  }
 }
-//# sourceMappingURL=mutagen.js.map
+export {
+  MutagenService,
+  getInsertSQL,
+  getUpsertSQL,
+  processMutation,
+  processMutationWithTx
+};
+//# sourceMappingURL=mutagen.js.map