@jskit-ai/resource-runtime 0.2.12 → 0.2.14

package/test/hiddenFields.test.js

@@ -1,18 +1,19 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import knexLib from "knex";
 import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
 import { countQueries } from "@jskit-ai/database-runtime/server/testDatabases";
 import { withoutHiddenFields } from "../src/server/index.js";
-import { createTestResourceRuntime } from "../src/test/index.js";
+import { createPackageTestKnex, createTestResourceRuntime } from "../src/test/index.js";
+
+// SQLite in memory, or a PostgreSQL schema of its own on a DB_CLIENT=pg run (see createPackageTestKnex).
+const testKnex = (t) => createPackageTestKnex(t, { environment: process.env });
 
 const WORKSPACE_A = Object.freeze({ workspace: { id: "1" } });
 
 // Cards with a hidden owner, a hidden bearer token, a hidden flag, a hidden count and a hidden time: two in workspace
 // 1 and one in workspace 2.
 async function createCardFixture(t) {
-  const knex = knexLib({ client: "better-sqlite3", connection: { filename: ":memory:" }, useNullAsDefault: true });
-  t.after(() => knex.destroy());
+  const knex = await testKnex(t);
   await knex.schema.createTable("cards", (table) => {
     table.increments("id");
     table.integer("workspace_id").notNullable();
@@ -46,8 +47,8 @@ async function createCardFixture(t) {
   // The tables each SELECT reads, in order.
   const selects = async (work) => (await countQueries(knex, work)).queries
     .filter((sql) => /^select/iu.test(sql))
-    .map((sql) => [...sql.matchAll(/from `(\w+)`/gu)].map(([, table]) => table)[0]);
-  return { cards, resource, system, selects, options: { context: WORKSPACE_A } };
+    .map((sql) => [...sql.matchAll(/from [`"](\w+)[`"]/gu)].map(([, table]) => table)[0]);
+  return { knex, cards, resource, system, selects, options: { context: WORKSPACE_A } };
 }
 
 test("a handle's plain read answers the hidden fields its fields names, in one more statement, and no read without them does", async (t) => {
@@ -113,43 +114,61 @@ test("withoutHiddenFields refuses another feature's read naming a hidden field,
   assert.equal(withoutHiddenFields(cards, plain), cards);
 });
 
-test("withoutHiddenFields refuses another feature's scopedQuery, whose builder carries the hidden columns, and the owner's join source still joins", async (t) => {
-  const { cards, resource, system, options } = await createCardFixture(t);
-  // The owner's join source, a method its decorateRepository adds with the handle it receives.
+test("withoutHiddenFields refuses another feature's scopedQuery, and any builder a method answers but a joinSource()'s, which carries no hidden column but the ownership fields", async (t) => {
+  const { knex, cards, resource, system, options } = await createCardFixture(t);
+  // Methods an owner's decorateRepository adds with the handle it receives, answering its scoped rows as scopedQuery()
+  // gives them, every column with them: wrapped, and at once.
   async function cardSource(alias, callOptions) {
     const { query } = await cards.scopedQuery({ alias }, callOptions);
     return { query, column: (field) => cards.column(field, alias) };
   }
-  const exposed = withoutHiddenFields(Object.freeze({ ...cards, cardSource }), resource);
+  const everyColumn = () => knex("cards");
+  const exposed = withoutHiddenFields(Object.freeze({ ...cards, cardSource, everyColumn }), resource);
 
   // Another feature's own builder would select the token: refused before it reads anything.
   await assert.rejects(
     (async () => (await exposed.scopedQuery({}, options)).query.select(exposed.column("publicToken")))(),
-    /cards\.scopedQuery\(\) answers every column, its hidden fields too, which only the table's owner, or a system context, reads/u
+    /cards\.scopedQuery\(\) answers every column, its hidden fields too, which only the table's owner, or a system context, reads: read the rows through the repository's reads, or join its joinSource\(\{ alias \}, options\)/u
   );
   await assert.rejects(exposed.withTransaction(options, (tx) => exposed.scopedQuery({ alias: "c" }, tx)), /cards\.scopedQuery\(\)/u);
+  await assert.rejects(exposed.cardSource("c", options),
+    /cards\.cardSource\(\) answers a query builder, which can select every column, the hidden fields too, which only the table's owner, or a system context, reads: answer the table's joinSource\(\{ alias \}, options\)/u);
+  assert.throws(() => exposed.everyColumn(), /cards\.everyColumn\(\) answers a query builder/u);
 
-  // A system context, and one it derived, still builds on it.
+  // A system context, and one it derived, still builds on them.
   const { query: everyWorkspace } = await exposed.scopedQuery({}, { context: system });
-  assert.deepEqual((await everyWorkspace.count({ cards: exposed.column("id") }))[0].cards, 3);
+  assert.deepEqual(Number((await everyWorkspace.count({ cards: exposed.column("id") }))[0].cards), 3);
   const { query: workspaceTwo } = await exposed.scopedQuery({}, { context: system.forWorkspace("2") });
   assert.deepEqual(await workspaceTwo.select({ title: exposed.column("title") }), [{ title: "Kit" }]);
+  const everyCard = await exposed.cardSource("c", { context: system });
+  assert.deepEqual(await everyCard.query.select({ token: everyCard.column("publicToken") }).orderBy(everyCard.column("id")),
+    [{ token: "t1" }, { token: "t2" }, { token: "t3" }]);
 
-  // The cross-owner report pattern: the owner's method answers its scoped rows, which the report selects from.
-  const source = await exposed.cardSource("c", options);
+  // The cross-owner report pattern: the table's joinSource() answers the caller's scoped rows, which the report
+  // selects from, with the columns a read answers and the ownership fields, and no other.
+  const source = await exposed.joinSource({ alias: "c" }, options);
   const rows = await source.query.select({ title: source.column("title") }).orderBy(source.column("id"));
   assert.deepEqual(rows, [{ title: "Rex" }, { title: "Bo" }]);
+  const { query: rex } = await exposed.joinSource({ alias: "c", filters: { title: "Rex" } }, options);
+  assert.deepEqual((await rex).map((row) => Object.keys(row)), [["id", "workspace_id", "title"]]);
+  assert.throws(() => source.column("publicToken"),
+    /cards\.joinSource\(\) carries no hidden field but its ownership fields, so it has no column for "publicToken"/u);
+  // The hidden workspaceId is a join key the source carries: a system report counts each workspace's cards on it.
+  const perWorkspace = await exposed.joinSource({ alias: "c" }, { context: system });
+  const workspaceId = perWorkspace.column("workspaceId");
+  const counts = await perWorkspace.query.select({ workspaceId }).count({ cards: perWorkspace.column("id") })
+    .groupBy(workspaceId).orderBy(workspaceId);
+  assert.deepEqual(counts.map((row) => [Number(row.workspaceId), Number(row.cards)]), [[1, 2], [2, 1]]);
+  const again = await exposed.joinSource({ alias: "c" }, options);
+  await assert.rejects(async () => again.query.select(cards.column("publicToken", "c")), /public_token/u);
+  await assert.rejects(exposed.joinSource({}, options), /cards\.joinSource\(\) needs the alias the report joins the cards under/u);
 });
 
-test("withoutHiddenFields answers raw(), column() and an owner's Knex values as they are, so another feature builds on them in a scoped query and a subquery", async (t) => {
+test("withoutHiddenFields answers raw(), column(), joinSource() and an owner's Knex values as they are, so another feature builds on them in a scoped query and a subquery", async (t) => {
   const { cards, resource, options } = await createCardFixture(t);
-  async function cardSource(alias, callOptions) {
-    const { query } = await cards.scopedQuery({ alias }, callOptions);
-    return { query, column: (field) => cards.column(field, alias) };
-  }
   // An owner's method answering a Knex value at once: an expression over its own column.
   const lowerTitle = (alias) => cards.raw("LOWER(??)", [cards.column("title", alias)]);
-  const exposed = withoutHiddenFields(Object.freeze({ ...cards, cardSource, lowerTitle }), resource);
+  const exposed = withoutHiddenFields(Object.freeze({ ...cards, lowerTitle }), resource);
 
   assert.equal(exposed.column("title", "c"), cards.column("title", "c"));
   const shout = exposed.raw("UPPER(??)", [exposed.column("title", "c")]);
@@ -157,24 +176,25 @@ test("withoutHiddenFields answers raw(), column() and an owner's Knex values as
   assert.equal(typeof exposed.lowerTitle("i").toSQL, "function");
 
   // In the owner's scoped rows, as a select.
-  const source = await exposed.cardSource("c", options);
+  const source = await exposed.joinSource({ alias: "c" }, options);
   assert.deepEqual(await source.query.select({ shout }).orderBy(source.column("id")), [{ shout: "REX" }, { shout: "BO" }]);
 
   // In a subquery: the cards of workspace 1 whose lower-cased title is "bo".
-  const outer = await exposed.cardSource("c", options);
-  const inner = await exposed.cardSource("i", options);
+  const outer = await exposed.joinSource({ alias: "c" }, options);
+  const inner = await exposed.joinSource({ alias: "i" }, options);
   const matching = inner.query.select(inner.column("id")).where(exposed.lowerTitle("i"), exposed.raw("?", ["bo"]));
   assert.deepEqual(await outer.query.whereIn(outer.column("id"), matching).select({ title: outer.column("title") }), [{ title: "Bo" }]);
 });
 
-test("withoutHiddenFields strips the hidden fields from every answer of a read the owner adds, unless it runs in a system context", async (t) => {
+test("withoutHiddenFields refuses a hidden field named through any method, and strips the hidden fields from every answer of a read the owner adds, unless it runs in a system context", async (t) => {
   const { cards, resource, system, options } = await createCardFixture(t);
   // The owner's decorateRepository reads: one forwarding the caller's params, the handle's own under another name, one
-  // answering a map, one in a sub-repository and one answering at once.
+  // taking its options first, one answering a map, one in a sub-repository and one answering at once.
   const owner = Object.freeze({
     ...cards,
     queryAllIncludingArchived: (params, callOptions) => cards.queryAll(params, callOptions),
     everyCard: cards.queryAll,
+    cardsTitled: (callOptions, { title }) => cards.queryAll({ filters: { title }, fields: ["title", "publicToken"] }, callOptions),
     async cardsByTitle(callOptions) {
       const records = await cards.queryAll({ fields: ["title", "publicToken"] }, callOptions);
       return new Map(records.map((record) => [record.title, record]));
@@ -184,10 +204,15 @@ test("withoutHiddenFields strips the hidden fields from every answer of a read t
   });
   const exposed = withoutHiddenFields(owner, resource);
 
-  // Another feature's calls: no hidden field leaves, whatever the method's name or depth.
-  assert.deepEqual(await exposed.queryAllIncludingArchived({ filters: { title: "Rex" }, fields: ["title", "publicToken", "workspaceId"] }, options),
-    [{ id: "1", title: "Rex" }]);
-  assert.deepEqual(await exposed.everyCard({ filters: { title: "Bo" }, fields: ["publicToken"] }, options), [{ id: "2" }]);
+  // Another feature's calls: a hidden field its fields names is refused before the method runs, whatever the method's
+  // name, as a list or a sparse fieldset, so a method forwarding the caller's fields never answers it as undefined.
+  await assert.rejects(exposed.queryAllIncludingArchived({ filters: { title: "Rex" }, fields: ["title", "publicToken", "workspaceId"] }, options),
+    /cards\.queryAllIncludingArchived\(\) fields names the hidden field "publicToken", "workspaceId", which only the table's owner, or a system context, reads/u);
+  await assert.rejects(exposed.everyCard({ filters: { title: "Bo" }, fields: { cards: "title, sent" } }, options),
+    /cards\.everyCard\(\) fields names the hidden field "sent"/u);
+  assert.deepEqual(await exposed.queryAllIncludingArchived({ filters: { title: "Rex" }, fields: ["title"] }, options), [{ id: "1", title: "Rex" }]);
+  // No hidden field the owner's own reads name leaves, whatever the method's name or depth.
+  assert.deepEqual(await exposed.cardsTitled(options, { title: "Rex" }), [{ id: "1", title: "Rex" }]);
   assert.deepEqual([...(await exposed.cardsByTitle(options)).values()], [{ id: "1", title: "Rex" }, { id: "2", title: "Bo" }]);
   assert.deepEqual(await exposed.archive.list(options), [{ id: "1", title: "Rex" }, { id: "2", title: "Bo" }]);
   // A method that answers at once still does, and withTransaction answers the caller's own work as it is.
@@ -200,4 +225,27 @@ test("withoutHiddenFields strips the hidden fields from every answer of a read t
   assert.deepEqual(await exposed.queryAllIncludingArchived({ filters: { title: "Kit" }, fields: ["publicToken", "workspaceId"] }, { context: system }),
     [{ id: "3", publicToken: "t3", workspaceId: "2" }]);
   assert.deepEqual(await exposed.archive.list({ context: system.forWorkspace("2") }), [{ id: "3", title: "Kit", views: 1 }]);
+  // The system context counts in whichever record carries it, not only the last.
+  assert.deepEqual(await exposed.cardsTitled({ context: system.forWorkspace("2") }, { title: "Kit" }), [{ id: "3", title: "Kit", publicToken: "t3" }]);
+});
+
+test("withoutHiddenFields refuses a field one owned table hides and another shows, since it strips hidden fields by name", () => {
+  const cards = defineTableResource({
+    name: "cards",
+    tableName: "cards",
+    ownership: "public",
+    schema: { title: { type: "string" }, publicToken: { type: "string", hidden: true } }
+  });
+  const links = (hidden) => defineTableResource({
+    name: "links",
+    tableName: "links",
+    ownership: "public",
+    schema: { publicToken: { type: "string", ...(hidden ? { hidden } : {}) } }
+  });
+  const repository = Object.freeze({ get: async () => ({ id: "1", publicToken: "t1" }) });
+  assert.throws(() => withoutHiddenFields(repository, [cards, links(false)]),
+    /links\.publicToken is a visible field, but cards, which the same feature owns, hides a field of that name; its repository strips hidden fields by name from every answer/u);
+  assert.throws(() => withoutHiddenFields(repository, [links(false), cards]), /links\.publicToken is a visible field, but cards/u);
+  // Hidden in both, it is one hidden field.
+  assert.equal(typeof withoutHiddenFields(repository, [cards, links(true)]).get, "function");
 });

package/test/locks.postgres.test.js

@@ -0,0 +1,345 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
+import { retryLostInsertRace } from "@jskit-ai/database-runtime/shared";
+import { createPackageTestKnex, createTestResourceRuntime } from "../src/test/index.js";
+
+// The row locks, the many-row writes and the upserts on PostgreSQL, with real concurrent transactions: what an
+// application on PostgreSQL relies on when two requests or two servers reach the same rows at once. Each test works in
+// a schema of its own in TEST_DB_NAME (createPackageTestKnex); the other data tests of this package run here too on a
+// DB_CLIENT=pg run, and the MySQL and MariaDB races are in manyRowWrites.mysql.test.js.
+
+const skip = process.env.DB_CLIENT === "pg" ? false : "a PostgreSQL test: it runs on a DB_CLIENT=pg run, with TEST_DB_NAME";
+const WORKSPACE = Object.freeze({ context: Object.freeze({ workspace: Object.freeze({ id: "1" }) }) });
+const OTHER_WORKSPACE = Object.freeze({ context: Object.freeze({ workspace: Object.freeze({ id: "2" }) }) });
+const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const settle = (work) => work().then((value) => ({ value }), (error) => ({ error }));
+
+const slotResource = defineTableResource({
+  name: "slots",
+  tableName: "slots",
+  ownership: "workspace",
+  defaultSort: ["id"],
+  schema: {
+    id: { type: "id", primary: true },
+    workspaceId: { type: "id", nullable: true, search: true, storage: { column: "workspace_id" } },
+    code: { type: "string", nullable: true, search: true },
+    status: { type: "string", defaultTo: "open", search: true },
+    takenBy: { type: "string", nullable: true, storage: { column: "taken_by" } }
+  }
+});
+
+// A table with no owners keyed by an id the server computes, as lockOrCreate and findOrCreate take.
+const balanceResource = defineTableResource({
+  name: "balances",
+  tableName: "balances",
+  ownership: "public",
+  schema: {
+    id: { type: "string", primary: true },
+    amount: { type: "integer", defaultTo: 0 }
+  }
+});
+
+async function database(t) {
+  const knex = await createPackageTestKnex(t, { environment: process.env });
+  await knex.schema.createTable("slots", (table) => {
+    table.increments("id");
+    table.integer("workspace_id").nullable();
+    table.string("code").nullable();
+    table.string("status").notNullable().defaultTo("open");
+    table.string("taken_by").nullable();
+    table.unique(["workspace_id", "code"]);
+  });
+  await knex.schema.createTable("balances", (table) => {
+    table.string("id").primary();
+    table.integer("amount").notNullable().defaultTo(0);
+  });
+  const runtime = await createTestResourceRuntime({ knex });
+  const { slots, balances } = await runtime.own("slots.feature", [slotResource, balanceResource]);
+  return { knex, slots, balances };
+}
+
+/**
+ * Waits until `count` sessions wait, directly or behind another waiter, for the session `pid` (pg_blocking_pids).
+ * @param {import("knex").Knex} knex
+ * @param {number} pid
+ * @param {number} count
+ */
+async function waitingBehind(knex, pid, count) {
+  for (let waited = 0; waited < 10_000; waited += 25) {
+    const { rows: [{ waiting }] } = await knex.raw(
+      "WITH RECURSIVE blocked(pid) AS (" +
+      "SELECT a.pid FROM pg_stat_activity a WHERE ? = ANY(pg_blocking_pids(a.pid)) " +
+      "UNION SELECT a.pid FROM pg_stat_activity a JOIN blocked b ON b.pid = ANY(pg_blocking_pids(a.pid))" +
+      ") SELECT count(*)::int AS waiting FROM blocked",
+      [pid]
+    );
+    if (waiting >= count) return;
+    await pause(25);
+  }
+  assert.fail(`${count} sessions did not wait for session ${pid} within 10 s`);
+}
+
+/**
+ * Runs `hold(trx)` in a transaction of its own, then the writers, until all of them wait behind it; then ends it
+ * (commits, or rolls back with `rollback`) and answers each writer's outcome.
+ */
+async function raceBehind(knex, hold, writers, { rollback = false } = {}) {
+  const holder = await knex.transaction();
+  try {
+    const { rows: [{ pid }] } = await holder.raw("SELECT pg_backend_pid() AS pid");
+    await hold(holder);
+    const running = writers.map((write) => settle(write));
+    await waitingBehind(knex, pid, writers.length);
+    await (rollback ? holder.rollback() : holder.commit());
+    return await Promise.all(running);
+  } finally {
+    if (!holder.isCompleted()) await holder.rollback();
+  }
+}
+
+const noErrors = (outcomes) => assert.deepEqual(outcomes.filter(({ error }) => error).map(({ error }) => error.message), []);
+
+test("of two concurrent patchWhere calls whose filters both matched one row, exactly one writes it", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const [id] = await slots.createMany([{ status: "open" }], { ...WORKSPACE, returning: "ids" });
+  const take = (taker) => () => slots.patchWhere({ id, status: "open" }, { status: "taken", takenBy: taker }, { ...WORKSPACE, returning: "ids" });
+  const outcomes = await raceBehind(knex, (holder) => holder("slots").where({ id }).forUpdate().select("id"), [take("first"), take("second")]);
+  noErrors(outcomes);
+  assert.deepEqual(outcomes.map(({ value }) => value.length).sort(), [0, 1], "exactly one writer takes the row");
+  const row = await knex("slots").where({ id }).first("status", "taken_by");
+  assert.deepEqual([row.status, row.taken_by], ["taken", outcomes[0].value.length === 1 ? "first" : "second"]);
+});
+
+test("a deleteWhere whose filter matched a row a concurrent patchWhere changed leaves it in place", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const [id] = await slots.createMany([{ status: "open" }], { ...WORKSPACE, returning: "ids" });
+  const outcomes = await raceBehind(knex, (holder) => holder("slots").where({ id }).forUpdate().select("id"), [
+    () => slots.patchWhere({ id, status: "open" }, { status: "taken", takenBy: "first" }, { ...WORKSPACE, returning: "ids" }),
+    () => slots.deleteWhere({ id, status: "open" }, { ...WORKSPACE, returning: "ids" })
+  ]);
+  noErrors(outcomes);
+  const [patched, deleted] = outcomes.map(({ value }) => value.length);
+  const row = await knex("slots").where({ id }).first("status");
+  assert.deepEqual([patched + deleted, row ? row.status : "deleted"], [1, patched === 1 ? "taken" : "deleted"]);
+});
+
+test("a patchWhere that waited for a row changed meanwhile writes only the rows that still match", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const ids = await slots.createMany([{ status: "open" }, { status: "open" }, { status: "open" }], { ...WORKSPACE, returning: "ids" });
+  const [outcome] = await raceBehind(
+    knex,
+    (holder) => holder("slots").where({ id: ids[1] }).update({ status: "taken", taken_by: "holder" }),
+    [() => slots.patchWhere({ status: "open" }, { status: "closed" }, { ...WORKSPACE, returning: "ids" })]
+  );
+  noErrors([outcome]);
+  assert.deepEqual(outcome.value.map(String).sort(), [ids[0], ids[2]].map(String).sort());
+  assert.deepEqual((await knex("slots").orderBy("id").pluck("status")), ["closed", "taken", "closed"]);
+});
+
+test("two workers claiming at once take different rows, and skip a row another transaction holds", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const ids = (await slots.createMany([{ status: "open" }, { status: "open" }, { status: "open" }, { status: "open" }], { ...WORKSPACE, returning: "ids" })).map(String);
+  await slots.createMany([{ status: "open" }], { ...OTHER_WORKSPACE, returning: "none" });
+  const holder = await knex.transaction();
+  t.after(() => (holder.isCompleted() ? undefined : holder.rollback()));
+  await holder("slots").where({ id: ids[0] }).forUpdate().select("id");
+
+  const first = Promise.withResolvers();
+  const release = Promise.withResolvers();
+  const claimNext = (take) => (txOptions) => slots.claim({ filters: { status: "open" }, sort: ["id"], take }, txOptions);
+  const worker = slots.withTransaction(WORKSPACE, async (txOptions) => {
+    const claimed = await claimNext(1)(txOptions);
+    first.resolve(claimed.map(({ id }) => String(id)));
+    await release.promise;
+    await slots.patchWhere({ id: claimed.map(({ id }) => id) }, { status: "taken", takenBy: "first" }, txOptions);
+    return claimed;
+  });
+  assert.deepEqual(await first.promise, [ids[1]], "the first worker skips the row the holder locked");
+  // While the first worker holds its row, the second takes the rest, never the held ones nor another workspace's.
+  const second = await slots.withTransaction(WORKSPACE, async (txOptions) => (await claimNext(10)(txOptions)).map(({ id }) => String(id)));
+  assert.deepEqual(second, [ids[2], ids[3]]);
+  release.resolve();
+  await worker;
+  await holder.rollback();
+  assert.deepEqual((await knex("slots").where({ workspace_id: 1 }).orderBy("id").pluck("status")), ["open", "taken", "open", "open"]);
+});
+
+test("a claim leaves a row whose committed change stopped it matching while the claim waited, and takes the next", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const ids = (await slots.createMany([{ status: "open" }, { status: "open" }], { ...WORKSPACE, returning: "ids" })).map(String);
+  // A claim never waits (SKIP LOCKED): the held row is skipped, and once its holder commits a change, it no longer matches.
+  const holder = await knex.transaction();
+  t.after(() => (holder.isCompleted() ? undefined : holder.rollback()));
+  await holder("slots").where({ id: ids[0] }).update({ status: "taken" });
+  const during = await slots.withTransaction(WORKSPACE, async (txOptions) => (await slots.claim({ filters: { status: "open" } }, txOptions)).map(({ id }) => String(id)));
+  await holder.commit();
+  const after = await slots.withTransaction(WORKSPACE, async (txOptions) => (await slots.claim({ filters: { status: "open" }, take: 5 }, txOptions)).map(({ id }) => String(id)));
+  assert.deepEqual([during, after], [[ids[1]], [ids[1]]]);
+});
+
+test("lockMany waits for another transaction's lock, then reads the rows as it committed them, in id order", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const ids = (await slots.createMany([{ status: "open" }, { status: "open" }, { status: "open" }], { ...WORKSPACE, returning: "ids" })).map(String);
+  const [theirs] = (await slots.createMany([{ status: "open" }], { ...OTHER_WORKSPACE, returning: "ids" })).map(String);
+  const [outcome] = await raceBehind(
+    knex,
+    (holder) => holder("slots").where({ id: ids[1] }).update({ status: "taken", taken_by: "holder" }),
+    [() => slots.withTransaction(WORKSPACE, (txOptions) => slots.lockMany([ids[2], theirs, ids[1], ids[0]], { ...txOptions, missing: "skip" }))]
+  );
+  noErrors([outcome]);
+  assert.deepEqual(outcome.value.map(({ id, status }) => [String(id), status]), [[ids[0], "open"], [ids[1], "taken"], [ids[2], "open"]]);
+});
+
+test("a lockWhere that waited also locks a row that started to match while it waited", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const ids = (await slots.createMany([{ status: "open" }, { status: "taken" }, { status: "taken" }], { ...WORKSPACE, returning: "ids" })).map(String);
+  const [outcome] = await raceBehind(
+    knex,
+    (holder) => holder("slots").whereIn("id", [ids[0], ids[1]]).update({ status: "open" }),
+    [() => slots.withTransaction(WORKSPACE, (txOptions) => slots.lockWhere({ status: "open" }, txOptions))]
+  );
+  noErrors([outcome]);
+  assert.deepEqual(outcome.value.map(({ id }) => String(id)), [ids[0], ids[1]]);
+});
+
+test("two upserts of one key behind an uncommitted insert of it both succeed, and the key has one row", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  const upsert = (action, takenBy) => () => slots.createMany(
+    [{ code: "a", status: "open", takenBy }],
+    { ...WORKSPACE, onConflict: { keys: ["workspaceId", "code"], action }, returning: "ids" }
+  );
+  for (const [action, expected] of [["ignore", ["holder"]], ["merge", ["first", "second"]]]) {
+    await knex("slots").del();
+    const outcomes = await raceBehind(
+      knex,
+      (holder) => holder("slots").insert({ workspace_id: 1, code: "a", status: "open", taken_by: "holder" }),
+      [upsert(action, "first"), upsert(action, "second")],
+      { rollback: action === "merge" }
+    );
+    noErrors(outcomes);
+    const rows = await knex("slots").where({ workspace_id: 1, code: "a" }).select("taken_by");
+    assert.equal(rows.length, 1, `${action}: one row for the key`);
+    assert.ok(expected.includes(rows[0].taken_by), `${action}: kept ${rows[0].taken_by}`);
+  }
+  // The same key in another workspace is that workspace's own row.
+  await slots.createMany([{ code: "a", status: "open" }], { ...OTHER_WORKSPACE, onConflict: { keys: ["workspaceId", "code"], action: "ignore" } });
+  assert.equal((await knex("slots").where({ code: "a" })).length, 2);
+});
+
+test("two transactions that lock two rows in opposite orders deadlock, and the victim runs again and commits", { skip }, async (t) => {
+  const { slots } = await database(t);
+  const [a, b] = (await slots.createMany([{ status: "open" }, { status: "open" }], { ...WORKSPACE, returning: "ids" })).map(String);
+  const locked = { first: Promise.withResolvers(), second: Promise.withResolvers() };
+  let runs = 0;
+  const work = (mine, theirs, name, other) => slots.withTransaction(WORKSPACE, async (txOptions) => {
+    runs += 1;
+    await slots.lock(mine, txOptions);
+    locked[name].resolve();
+    await locked[other].promise;
+    await slots.lock(theirs, txOptions);
+    await slots.patch(mine, { takenBy: name }, { ...txOptions, returning: "none" });
+  });
+  noErrors(await Promise.all([settle(() => work(a, b, "first", "second")), settle(() => work(b, a, "second", "first"))]));
+  assert.equal(runs, 3, "the deadlock victim's work ran once more");
+  assert.deepEqual((await slots.queryAll({ filters: { id: [a, b] } }, WORKSPACE)).map(({ takenBy }) => takenBy), ["first", "second"]);
+});
+
+test("a transaction whose insert lost a unique race runs once more and finds the winner's row", { skip }, async (t) => {
+  const { knex, slots } = await database(t);
+  let runs = 0;
+  const record = (takenBy) => () => retryLostInsertRace(WORKSPACE, () => slots.withTransaction(WORKSPACE, async (txOptions) => {
+    runs += 1;
+    const [found] = await slots.queryAll({ filters: { code: "k" } }, txOptions);
+    return found ?? slots.create({ code: "k", status: "taken", takenBy }, txOptions);
+  }));
+  const outcomes = await raceBehind(
+    knex,
+    (holder) => holder("slots").insert({ workspace_id: 1, code: "k", status: "open", taken_by: "holder" }),
+    [record("first"), record("second")],
+    { rollback: true }
+  );
+  noErrors(outcomes);
+  const [first, second] = outcomes.map(({ value }) => value);
+  assert.equal(String(first.id), String(second.id), "both answer the one row");
+  assert.equal(runs, 3, "the loser ran once more");
+  assert.equal((await knex("slots").where({ code: "k" })).length, 1);
+});
+
+test("findOrCreate of one id at once, behind an uncommitted insert of it, answers one record to both", { skip }, async (t) => {
+  const { knex, balances } = await database(t);
+  const outcomes = await raceBehind(
+    knex,
+    (holder) => holder("balances").insert({ id: "acct-1", amount: 7 }),
+    [() => balances.findOrCreate({ id: "acct-1", amount: 1 }, WORKSPACE), () => balances.findOrCreate({ id: "acct-1", amount: 2 }, WORKSPACE)],
+    { rollback: true }
+  );
+  noErrors(outcomes);
+  const [first, second] = outcomes.map(({ value }) => value);
+  assert.deepEqual([first.id, second.id, first.amount], ["acct-1", "acct-1", second.amount]);
+  assert.deepEqual(await knex("balances").pluck("id"), ["acct-1"]);
+});
+
+test("first uses of one id at once through lockOrCreate create one record and run every work once, one at a time", { skip }, async (t) => {
+  const { balances } = await database(t);
+  let works = 0;
+  let inside = 0;
+  let overlapped = false;
+  const add = () => balances.lockOrCreate({ id: "acct-2", amount: 0 }, WORKSPACE, async (record, txOptions) => {
+    works += 1;
+    inside += 1;
+    overlapped ||= inside > 1;
+    await pause(20);
+    await balances.patch(record.id, { amount: record.amount + 1 }, { ...txOptions, returning: "none" });
+    inside -= 1;
+  });
+  noErrors(await Promise.all(Array.from({ length: 4 }, () => settle(add))));
+  assert.equal(works, 4);
+  assert.equal(overlapped, false, "the lock serializes the works");
+  assert.equal((await balances.get("acct-2", {}, WORKSPACE)).amount, 4);
+});
+
+test("a lock or patchWhere with lockMode noKeyUpdate lets another transaction add a row that references the locked one; the default lock does not", { skip }, async (t) => {
+  const { knex, slots, balances } = await database(t);
+  await knex.schema.createTable("entries", (table) => {
+    table.increments("id");
+    table.integer("slot_id").nullable().references("id").inTable("slots");
+    table.string("balance_id").nullable().references("id").inTable("balances");
+  });
+  const [slot] = await slots.createMany([{ status: "open" }], { ...WORKSPACE, returning: "ids" });
+  await balances.createMany([{ id: "acct-3", amount: 0 }], { ...WORKSPACE, returning: "none" });
+  // Another transaction adds a row that references the locked one. Its foreign key check takes FOR KEY SHARE on the
+  // locked row, which FOR NO KEY UPDATE lets through and FOR UPDATE does not; it gives up after 500 ms (55P03).
+  const addEntry = async (entry) => {
+    const { error } = await settle(() => knex.transaction(async (trx) => {
+      await trx.raw("SET LOCAL lock_timeout = '500ms'");
+      await trx("entries").insert(entry);
+    }));
+    return error ? error.code : "added";
+  };
+  const lockers = {
+    lock: (txOptions) => slots.lock(slot, txOptions),
+    lockMany: (txOptions) => slots.lockMany([slot], txOptions),
+    lockWhere: (txOptions) => slots.lockWhere({ status: "open" }, txOptions),
+    claim: (txOptions) => slots.claim({ filters: { status: "open" } }, txOptions),
+    patchWhere: (txOptions) => slots.patchWhere({ status: "open" }, { takenBy: "worker" }, txOptions)
+  };
+  const outcomes = {};
+  for (const [name, locker] of Object.entries(lockers)) {
+    for (const mode of ["noKeyUpdate", "update"]) {
+      outcomes[`${name} ${mode}`] = await slots.withTransaction(WORKSPACE, async (txOptions) => {
+        await locker(mode === "update" ? txOptions : { ...txOptions, lockMode: mode });
+        return addEntry({ slot_id: slot });
+      });
+    }
+  }
+  for (const mode of ["noKeyUpdate", "update"]) {
+    const options = mode === "update" ? WORKSPACE : { ...WORKSPACE, lockMode: mode };
+    outcomes[`lockOrCreate ${mode}`] = await balances.lockOrCreate({ id: "acct-3", amount: 0 }, options, () => addEntry({ balance_id: "acct-3" }));
+  }
+  assert.deepEqual(outcomes, Object.fromEntries([...Object.keys(lockers), "lockOrCreate"].flatMap((name) => [
+    [`${name} noKeyUpdate`, "added"],
+    [`${name} update`, "55P03"]
+  ])));
+  assert.equal(Number((await knex("entries").count({ count: "*" }).first()).count), 6);
+});

package/test/lostInsertRace.test.js

@@ -1,16 +1,17 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import knexLib from "knex";
 import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
 import { isDuplicateEntryError, retryLostInsertRace } from "@jskit-ai/database-runtime/shared";
-import { createTestResourceRuntime } from "../src/test/index.js";
+import { createPackageTestKnex, createTestResourceRuntime } from "../src/test/index.js";
+
+// SQLite in memory, or a PostgreSQL schema of its own on a DB_CLIENT=pg run (see createPackageTestKnex).
+const testKnex = (t) => createPackageTestKnex(t, { environment: process.env });
 
 const WORKSPACE_A = Object.freeze({ workspace: { id: "1" } });
 
 // Requests keyed by a request key, unique in each workspace: a repeated request finds its first answer by the key.
 async function createRequestFixture(t) {
-  const knex = knexLib({ client: "better-sqlite3", connection: { filename: ":memory:" }, useNullAsDefault: true });
-  t.after(() => knex.destroy());
+  const knex = await testKnex(t);
   await knex.schema.createTable("requests", (table) => {
     table.increments("id");
     table.integer("workspace_id").notNullable();

package/test/manyRowWrites.mysql.test.js

@@ -3,6 +3,7 @@ import test from "node:test";
 import knexLib from "knex";
 import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
 import { requireTestDatabase } from "@jskit-ai/database-runtime/server/testDatabases";
+import { UnsupportedDatabaseError } from "../src/server/index.js";
 import { createTestResourceRuntime } from "../src/test/index.js";
 
 // patchWhere and deleteWhere on MySQL or MariaDB, under two concurrent writers: each checks its filters on the row
@@ -143,3 +144,11 @@ test("a deleteWhere whose filter matched a row a concurrent patchWhere changed l
   // Whichever went first wins; the other finds the row changed (or gone) and leaves it.
   assert.deepEqual([patched + deleted, row ? row.status : "deleted"], [1, patched === 1 ? "taken" : "deleted"]);
 });
+
+test("changeHorizon() throws on MySQL and MariaDB, which stamp no row with its transaction, before any statement", async (t) => {
+  const { knex, slots } = await ownDatabase(t, "horizon");
+  const statements = [];
+  knex.on("query", ({ sql }) => statements.push(sql));
+  await assert.rejects(slots.changeHorizon({ context: WORKSPACE }), UnsupportedDatabaseError);
+  assert.deepEqual(statements, []);
+});

package/test/queryFields.test.js

@@ -1,14 +1,15 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import knexLib from "knex";
 import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
-import { createTestResourceRuntime } from "../src/test/index.js";
+import { createPackageTestKnex, createTestResourceRuntime } from "../src/test/index.js";
+
+// SQLite in memory, or a PostgreSQL schema of its own on a DB_CLIENT=pg run (see createPackageTestKnex).
+const testKnex = (t) => createPackageTestKnex(t, { environment: process.env });
 
 const WORKSPACE_A = Object.freeze({ workspace: { id: "1" } });
 
 test("a query field's select gets a value-only raw, column() and the read's filters, never Knex or its context", async (t) => {
-  const knex = knexLib({ client: "better-sqlite3", connection: { filename: ":memory:" }, useNullAsDefault: true });
-  t.after(() => knex.destroy());
+  const knex = await testKnex(t);
   await knex.schema.createTable("receivals", (table) => {
     table.increments("id");
     table.integer("workspace_id");