@jskit-ai/resource-runtime 0.2.12 → 0.2.13

package/src/server/index.js

@@ -1,6 +1,7 @@
 export { ResourceRuntimeProvider } from "./ResourceRuntimeProvider.js";
 export { DEFAULT_REQUEST_ROW_LIMIT, DEFAULT_SERVER_PAGE_SIZE } from "./resourceRuntime.js";
 export { buildJsonRestQueryParams, resolveUserScopeValue, resolveWorkspaceScopeValue } from "./resourceHost.js";
+export { checkForeignKeys } from "./foreignKeyCheck.js";
 export { createStandaloneResources } from "./standaloneResources.js";
 export { isSystemContext } from "./contexts.js";
 export { withoutHiddenFields } from "./handleHidden.js";
@@ -11,5 +12,7 @@ export {
   PageSizeError,
   RowLimitError,
   StaleRecordError,
-  UnpagedQueryError
+  SystemContextRequiredError,
+  UnpagedQueryError,
+  UnsupportedDatabaseError
 } from "./errors.js";

package/src/server/lockOrCreate.js

@@ -1,6 +1,7 @@
 import { isDeadlockError, isDuplicateEntryError, isLostDuplicateInsertRace } from "@jskit-ai/database-runtime/shared";
 import { isRecord } from "@jskit-ai/kernel/shared/support/normalize";
 import { passedOn } from "./handleGuards.js";
+import { lockModeOf } from "./handleLocks.js";
 
 // First uses of one id at once can collide on its insert before any work runs (on MySQL and MariaDB, first uses of
 // neighbouring ids too, through their gap locks): each collision runs the loser again after a short random wait.
@@ -19,7 +20,8 @@ const CREATE_IF_MISSING = Object.freeze({ keys: ["id"], action: "ignore" });
  * @template T
  * @param {Pick<import("./types.js").ResourceHandle, "name" | "withTransaction" | "lock" | "createMany">} handle
  * @param {{ id: unknown } & Record<string, unknown>} row
- * @param {{ context?: object | null, trx?: object | null }} options
+ * @param {{ context?: object | null, trx?: object | null, lockMode?: "update" | "noKeyUpdate" }} options `lockMode` is the
+ *   record's lock, as lock() takes it.
  * @param {(record: Record<string, any>, txOptions: Readonly<Record<string, any>>) => Promise<T> | T} work
  * @returns {Promise<T>}
  */
@@ -27,13 +29,15 @@ async function lockOrCreate(handle, row, options, work) {
   if (!isRecord(row) || row.id == null) {
     throw new TypeError(`${handle.name}.lockOrCreate() takes (row, options, work), the row to create holding its id.`);
   }
+  // The lock mode the caller names, for both locks of the record, refused before any transaction opens.
+  const mode = lockModeOf(handle.name, "lockOrCreate", options.lockMode);
   const locked = async (txOptions) => {
-    const existing = await handle.lock(row.id, { ...txOptions, missing: "skip" });
+    const existing = await handle.lock(row.id, { ...txOptions, ...mode, missing: "skip" });
     if (existing) {
       return existing;
     }
     await handle.createMany([row], { ...txOptions, onConflict: CREATE_IF_MISSING, returning: "none" });
-    return handle.lock(row.id, txOptions);
+    return handle.lock(row.id, { ...txOptions, ...mode });
   };
   if (options.trx) {
     const txOptions = passedOn(options);

package/src/server/resourceHandle.js

@@ -2,12 +2,13 @@ import diagnosticsChannel from "node:diagnostics_channel";
 import { RestApiRowLimitError } from "json-rest-api";
 import { retryDeadlocks } from "@jskit-ai/database-runtime/shared";
 import { isRecord } from "@jskit-ai/kernel/shared/support/normalize";
+import { readChangeHorizon } from "./changeHorizon.js";
 import { createJsonRestContext, isJobContext } from "./contexts.js";
-import { createHiddenFieldReads, markHandleFunctions } from "./handleHidden.js";
-import { createLockReads } from "./handleLocks.js";
+import { createHiddenFieldReads, createJoinSource, markHandleFunctions } from "./handleHidden.js";
+import { createLockReads, lockModeOf } from "./handleLocks.js";
 import { findOrCreateRecord, lockOrCreate } from "./lockOrCreate.js";
 import { valueOnlyRaw } from "./rawExpression.js";
-import { MissingScopeError, PageSizeError, RowLimitError, StaleRecordError, UnpagedQueryError } from "./errors.js";
+import { PageSizeError, RowLimitError, StaleRecordError, UnpagedQueryError } from "./errors.js";
 import {
   formatOf, matchesNoRow, missingOf, passedOn, refuseConditions, requireConditionValues, requireDefinedFilters, requireOwners, requirePatchConditions,
   requireReadParams, requireUpsertRows, whereSelection
@@ -154,9 +155,6 @@ function createResourceHandle({ api, resource, hooks, settings, resourceNamed })
   /** @param {{ context?: object | null }} options */
   function contextOf(options) {
     requireHandleTransaction(name, options);
-    if (!options.context && resource.ownership !== "public") {
-      throw new MissingScopeError(name);
-    }
     requireOwners(name, resource.ownership, options.context);
     return createJsonRestContext(options.context);
   }
@@ -165,6 +163,8 @@ function createResourceHandle({ api, resource, hooks, settings, resourceNamed })
   const readFields = (params, kind) => requestedFields(params, quiet[kind], () => jsonRest().vars.schemaInfo);
   // The hidden fields a plain read names, which json-rest-api leaves out (see handleHidden.js).
   const hiddenFields = createHiddenFieldReads({ resource, scopedQuery, column });
+  // The table as a join source, with its visible columns only (see handleHidden.js).
+  const joinSource = createJoinSource({ api, resource, jsonRest, scopedQuery, column });
   // lock, lockMany, lockWhere and claim (see handleLocks.js).
   const { lock, lockMany, lockWhere, claim } = createLockReads({ name, resource, jsonRest, readFields, contextOf, hiddenFields });
 
@@ -590,17 +590,19 @@ function createResourceHandle({ api, resource, hooks, settings, resourceNamed })
    * The same changes to every record the filters select (the caller's own rows only), in one statement
    * that carries the scope and filters, so a row gone meanwhile is skipped. Empty filters change every
    * row the caller owns, so they need `all: true`. `returning: "count"` answers how many changed. lockVersion is
-   * refused (see ONE_VERSION).
+   * refused (see ONE_VERSION). `lockMode` is the row lock its selection takes before the write, as the locks take it
+   * (see handleLocks.js).
    * @param {object} filters
    * @param {Record<string, unknown>} attributes
-   * @param {{ context?: object | null, trx?: object | null, returning?: "none" | "ids" | "records" | "count", onlyIf?: object, all?: boolean, chunkSize?: number }} [options]
+   * @param {{ context?: object | null, trx?: object | null, returning?: "none" | "ids" | "records" | "count", onlyIf?: object, all?: boolean, chunkSize?: number, lockMode?: "update" | "noKeyUpdate" }} [options]
    */
   async function patchWhere(filters, attributes, options = {}) {
     refuseConditions(name, "patchWhere", options, ["lockVersion"], ONE_VERSION);
     requireConditionValues(name, "patchWhere", options, ["onlyIf"]);
     const selection = whereSelection(name, "patchWhere", filters, options);
+    const mode = lockModeOf(name, "patchWhere", options.lockMode);
     return manyRowWrite("patchWhere", 0, options, (params, context) => jsonRest().patchWhere(
-      { ...params, ...selection, data: inputData(resource, attributes, "update", options.context),
+      { ...params, ...selection, ...mode, data: inputData(resource, attributes, "update", options.context),
         ...(options.onlyIf ? { onlyIf: options.onlyIf } : {}) },
       context
     ));
@@ -693,6 +695,15 @@ function createResourceHandle({ api, resource, hooks, settings, resourceNamed })
     return valueOnlyRaw(`${name}.raw()`, api.knex.instance, [sql, bindings, ...rest]);
   }
 
+  /**
+   * PostgreSQL's snapshot horizon, as decimal text, in the caller's transaction when there is one (see readChangeHorizon).
+   * @param {{ context?: object | null, trx?: object | null }} [options]
+   */
+  async function changeHorizon(options = {}) {
+    contextOf(options);
+    return readChangeHorizon(name, options.trx || api.knex.instance);
+  }
+
   markHandleFunctions({ scopedQuery, withTransaction, lockOrCreate: lockOrCreateRecord });
   return Object.freeze({
     name,
@@ -715,7 +726,9 @@ function createResourceHandle({ api, resource, hooks, settings, resourceNamed })
     lockWhere,
     lockOrCreate: lockOrCreateRecord,
     claim,
+    changeHorizon,
     scopedQuery,
+    joinSource,
     column,
     raw
   });

package/src/server/resourceHost.js

@@ -432,7 +432,8 @@ function primaryField(scopeOptions) {
 
 /**
  * json-rest-api's options for a table resource: its own keys, with `ownership`
- * as json-rest-api's autofilter preset (none for a public table),
+ * as json-rest-api's autofilter preset (none for a public or a system table,
+ * which have no owner columns),
  * `defaultExclude` as normally hidden fields, symbolic write serializers
  * resolved and collection relations derived. The owner's server-only extras
  * (row policy, id normalizer, query fields) are added here, never in the
@@ -463,7 +464,7 @@ function resourceScopeOptions(
   const scopeOptions = cloneJsonRestResourceValue(tableOptions, {
     writeSerializers: normalizeObject(writeSerializers)
   });
-  if (ownership !== "public") {
+  if (ownership !== "public" && ownership !== "system") {
     scopeOptions.autofilter = ownership;
   }
   if (isPlainJsonRestObject(searchSchema)) {

package/src/test/index.js

@@ -3,7 +3,7 @@ import { ActionProvider } from "@jskit-ai/kernel/server/actions";
 import { EventProvider } from "@jskit-ai/kernel/server/runtime";
 import { silentLogger } from "@jskit-ai/kernel/server/runtime/logger";
 import { createCapabilityRuntime, defineProvider } from "@jskit-ai/kernel/shared/capabilities";
-import { requireTestDatabase, tableNames, TestDatabaseMissingError } from "@jskit-ai/database-runtime/server/testDatabases";
+import { createPackageTestKnex, requireTestDatabase, tableNames, TestDatabaseMissingError } from "@jskit-ai/database-runtime/server/testDatabases";
 import { createResourceHost } from "../server/resourceHost.js";
 import { createResourceRuntime } from "../server/resourceRuntime.js";
 import { expectFlatQueries } from "./expectFlatQueries.js";
@@ -113,6 +113,7 @@ async function expectEveryTableOwned({ knex, runtime, infrastructureTables = [],
 
 export {
   TestDatabaseMissingError,
+  createPackageTestKnex,
   createTestResourceRuntime,
   expectEveryTableOwned,
   expectFlatQueries,

package/test/callSettings.test.js

@@ -1,9 +1,11 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import knexLib from "knex";
 import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
 import { passedOn } from "../src/server/index.js";
-import { createTestResourceRuntime } from "../src/test/index.js";
+import { createPackageTestKnex, createTestResourceRuntime } from "../src/test/index.js";
+
+// SQLite in memory, or a PostgreSQL schema of its own on a DB_CLIENT=pg run (see createPackageTestKnex).
+const testKnex = (t) => createPackageTestKnex(t, { environment: process.env });
 
 // A call's settings (returning, missing, format, lockVersion...) are for that one call. What it hands on, to the work
 // of a transaction, to an afterWrite hook or to the handle's own nested calls, is its context and transaction only.
@@ -14,8 +16,7 @@ const SETTINGS = Object.freeze({ returning: "none", missing: "skip", format: "js
 const notFound = (error) => error.code === "REST_API_RESOURCE" && error.subtype === "not_found";
 
 async function createNotes(t, hooks) {
-  const knex = knexLib({ client: "better-sqlite3", connection: { filename: ":memory:" }, useNullAsDefault: true });
-  t.after(() => knex.destroy());
+  const knex = await testKnex(t);
   await knex.schema.createTable("notes", (table) => {
     table.increments("id");
     table.integer("workspace_id").nullable();

package/test/changes.postgres.test.js

@@ -0,0 +1,141 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
+import { createPackageTestKnex, createTestResourceRuntime } from "../src/test/index.js";
+
+// What changed since a horizon, on PostgreSQL, with real concurrent transactions. Each row carries the transaction
+// that last wrote it, an xid8 change stamp that a trigger sets on every insert and update, whatever the write sent, read
+// through an xid8 field with changedSince (>=) and changedBefore (<) filters. A reader takes
+// changeHorizon() before it reads; every transaction below the horizon had finished, so a later read of the rows
+// stamped at or above it finds everything the first read could not see. Each test works in a schema of its own in
+// TEST_DB_NAME (createPackageTestKnex); other sessions on the server may hold the horizon lower, never higher, so the
+// tests check what a read must include and what it must leave out.
+
+const skip = process.env.DB_CLIENT === "pg" ? false : "a PostgreSQL test: it runs on a DB_CLIENT=pg run, with TEST_DB_NAME";
+const WORKSPACE = Object.freeze({ context: Object.freeze({ workspace: Object.freeze({ id: "1" }) }) });
+
+const messageResource = defineTableResource({
+  name: "messages",
+  tableName: "messages",
+  ownership: "workspace",
+  defaultSort: ["id"],
+  schema: {
+    id: { type: "id", primary: true },
+    workspaceId: { type: "id", nullable: true, search: true, storage: { column: "workspace_id" } },
+    body: { type: "string", required: true, search: true },
+    // The database stamps it; nothing writes it.
+    changed: { type: "xid8", nullable: true }
+  },
+  searchSchema: {
+    changedSince: { type: "xid8", actualField: "changed", filterOperator: ">=" },
+    changedBefore: { type: "xid8", actualField: "changed", filterOperator: "<" }
+  }
+});
+
+async function database(t) {
+  const knex = await createPackageTestKnex(t, { environment: process.env });
+  await knex.schema.createTable("messages", (table) => {
+    table.increments("id");
+    table.integer("workspace_id").nullable();
+    table.string("body").notNullable();
+    table.specificType("changed", "xid8").notNullable().defaultTo(knex.raw("pg_current_xact_id()"));
+  });
+  await knex.raw(
+    "create function messages_changed() returns trigger language plpgsql as $$ " +
+    "begin new.changed := pg_current_xact_id(); return new; end $$"
+  );
+  await knex.raw("create trigger messages_changed before insert or update on messages for each row execute function messages_changed()");
+  const runtime = await createTestResourceRuntime({ knex });
+  const { messages } = await runtime.own("messages.feature", [messageResource]);
+  return { knex, messages };
+}
+
+const bodies = (records) => new Map(records.map(({ id, body }) => [String(id), body]));
+
+test("a read from the change horizon finds what a transaction still open at the horizon wrote, which the highest id seen misses", { skip }, async (t) => {
+  const { knex, messages } = await database(t);
+  const [first] = (await messages.createMany([{ body: "first" }], { ...WORKSPACE, returning: "ids" })).map(String);
+
+  // A writer still open when the reader reads: it changes the first message and adds one, taking its id now.
+  const writer = await knex.transaction();
+  t.after(() => (writer.isCompleted() ? undefined : writer.rollback()));
+  await writer("messages").where({ id: first }).update({ body: "first, edited" });
+  const [{ id: added }] = await writer("messages").insert({ workspace_id: 1, body: "added by the open writer" }, ["id"]);
+  const [{ id: theirs }] = await writer("messages").insert({ workspace_id: 2, body: "another workspace's" }, ["id"]);
+  const { rows: [{ xid: writerXid }] } = await writer.raw("select pg_current_xact_id()::text as xid");
+  // A later writer adds one and commits at once, so its id is above the open writer's.
+  const [later] = (await messages.createMany([{ body: "added later" }], { ...WORKSPACE, returning: "ids" })).map(String);
+
+  const horizon = await messages.changeHorizon(WORKSPACE);
+  const page = await messages.queryAll({}, WORKSPACE);
+  assert.deepEqual([...bodies(page)], [[first, "first"], [later, "added later"]]);
+  assert.match(horizon, /^\d{1,20}$/u);
+  assert.ok(BigInt(horizon) <= BigInt(writerXid), `the open writer's transaction ${writerXid} is at or above the horizon ${horizon}`);
+
+  await writer.commit();
+  const caughtUp = bodies(await messages.queryAll({ filters: { changedSince: horizon } }, WORKSPACE));
+  assert.equal(caughtUp.get(first), "first, edited", "the change of an older message");
+  assert.equal(caughtUp.get(String(added)), "added by the open writer", "the message whose id was taken before it committed");
+  assert.equal(caughtUp.has(String(theirs)), false, "another workspace's message stays out");
+  // "Past the highest id seen" misses both: the edit kept its id, and the added message's id is below the later one's.
+  const highestSeen = Math.max(...page.map(({ id }) => Number(id)));
+  assert.deepEqual([Number(first) < highestSeen, Number(added) < highestSeen], [true, true]);
+});
+
+test("changedSince and changedBefore read the window between two horizons; what the window leaves out, the next read from its end finds", { skip }, async (t) => {
+  const { messages } = await database(t);
+  const start = await messages.changeHorizon(WORKSPACE);
+  const [a] = (await messages.createMany([{ body: "a" }], { ...WORKSPACE, returning: "ids" })).map(String);
+  const end = await messages.changeHorizon(WORKSPACE);
+  const [b] = (await messages.createMany([{ body: "b" }], { ...WORKSPACE, returning: "ids" })).map(String);
+
+  const window = bodies(await messages.queryAll({ filters: { changedSince: start, changedBefore: end } }, WORKSPACE));
+  const next = bodies(await messages.queryAll({ filters: { changedSince: end } }, WORKSPACE));
+  // b was written after the end horizon was read, so its transaction is at or above it: never in the window.
+  assert.equal(window.has(b), false);
+  assert.equal(next.get(b), "b");
+  // a is in the window or, while another session held the end horizon below a's transaction, in the next read.
+  assert.equal(window.get(a) ?? next.get(a), "a");
+  assert.ok(BigInt(start) <= BigInt(end), "the horizon never moves back");
+});
+
+test("a stamp compares as a transaction id: from a row's own stamp it is found, from the next one not, a malformed cursor is refused, and a patch moves the stamp on", { skip }, async (t) => {
+  const { knex, messages } = await database(t);
+  // The trigger stamps the writing transaction whatever the write sends.
+  const [created] = await messages.createMany([{ body: "one", changed: "1" }], { ...WORKSPACE, returning: "records" });
+  const stamp = BigInt(created.changed);
+  assert.ok(stamp > 1n, `stamped ${created.changed}`);
+  const found = async (filters) => (await messages.queryAll({ filters }, WORKSPACE)).map(({ body }) => body);
+  const next = String(stamp + 1n);
+  assert.deepEqual(
+    [await found({ changedSince: created.changed }), await found({ changedSince: next }), await found({ changedBefore: created.changed }), await found({ changedBefore: next })],
+    [["one"], [], [], ["one"]]
+  );
+  // They compare as numbers, not as text, where "9" would sort above a stamp starting with a lower digit.
+  assert.deepEqual([await found({ changedSince: "9" }), await found({ changedBefore: "9" })], [["one"], []]);
+  assert.deepEqual(await found({ changedSince: "18446744073709551615" }), []);
+  // A cursor is decimal text below 2^64, leading zeros allowed; anything else is refused before any statement.
+  assert.deepEqual(await found({ changedSince: `000${created.changed}` }), ["one"]);
+  const statements = [];
+  const counted = ({ sql }) => statements.push(sql);
+  knex.on("query", counted);
+  for (const cursor of ["not a position", "-1", "18446744073709551616", "1.5"]) {
+    await assert.rejects(found({ changedSince: cursor }), { code: "REST_API_VALIDATION" }, cursor);
+  }
+  knex.off("query", counted);
+  assert.deepEqual(statements, []);
+  await messages.patch(created.id, { body: "one, edited" }, { ...WORKSPACE, returning: "none" });
+  const [{ changed: moved }] = await knex("messages").where({ id: created.id }).select(knex.raw("changed::text as changed"));
+  assert.ok(BigInt(moved) > stamp, "a patch stamps its own transaction");
+  assert.deepEqual(await found({ changedSince: next }), ["one, edited"]);
+});
+
+test("changeHorizon() reads in the caller's transaction and checks the call's context, as every handle call does", { skip }, async (t) => {
+  const { messages } = await database(t);
+  const inside = await messages.withTransaction(WORKSPACE, async (txOptions) => {
+    await messages.create({ body: "x" }, { ...txOptions, returning: "none" });
+    return messages.changeHorizon(txOptions);
+  });
+  assert.match(inside, /^\d{1,20}$/u);
+  await assert.rejects(messages.changeHorizon({}), { code: "RESOURCE_MISSING_SCOPE" });
+});

package/test/foreignKeys.test.js

@@ -0,0 +1,94 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import knexLib from "knex";
+import { silentLogger } from "@jskit-ai/kernel/server/runtime/logger";
+import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
+import { checkForeignKeys } from "../src/server/index.js";
+import { createResourceHost } from "../src/server/resourceHost.js";
+import { createResourceRuntime } from "../src/server/resourceRuntime.js";
+
+const owners = defineTableResource({
+  name: "owners",
+  tableName: "owners",
+  ownership: "public",
+  schema: { name: { type: "string", required: true } }
+});
+
+function petsResource(foreignKeys) {
+  return defineTableResource({
+    name: "pets",
+    tableName: "pets",
+    ownership: "public",
+    schema: {
+      name: { type: "string", required: true },
+      ownerId: { type: "id", nullable: true, belongsTo: "owners", as: "owner" },
+      vetId: { type: "id", nullable: true, storage: { column: "clinic_vet" } }
+    },
+    ...(foreignKeys ? { foreignKeys } : {})
+  });
+}
+
+async function petsDatabase(t) {
+  const knex = knexLib({ client: "better-sqlite3", connection: { filename: ":memory:" }, useNullAsDefault: true });
+  t.after(() => knex.destroy());
+  await knex.schema.createTable("owners", (table) => {
+    table.increments("id");
+    table.string("name").notNullable();
+  });
+  await knex.schema.createTable("vets", (table) => table.increments("id"));
+  await knex.schema.createTable("pets", (table) => {
+    table.increments("id");
+    table.string("name").notNullable();
+    table.integer("owner_id").references("id").inTable("owners");
+    table.integer("clinic_vet");
+  });
+  return knex;
+}
+
+test("a declared foreignKeys reaches the json-rest-api resource, and a table without one declares none", async (t) => {
+  const knex = await petsDatabase(t);
+  const api = await createResourceHost({ knex, logger: silentLogger });
+  const declared = [{ columns: ["ownerId"], referencedTableName: "owners", deleteRule: "SET NULL" }];
+  const runtime = createResourceRuntime({ api });
+  const { pets } = await runtime.own("app.pets", [owners, petsResource(declared)]);
+  const context = runtime.declareSystemScope("app.pets", "The test seeds every owner's pets.")("seed a pet");
+
+  assert.deepEqual(api.resources.pets.vars.schemaInfo.foreignKeys, declared);
+  assert.deepEqual(api.resources.owners.vars.schemaInfo.foreignKeys, []);
+  const [ownerId] = await knex("owners").insert({ name: "Ada" });
+  const created = await pets.create({ name: "Rex", ownerId: String(ownerId) }, { context });
+  assert.equal(created.ownerId, String(ownerId));
+  assert.equal((await pets.get(created.id, {}, { context })).ownerId, String(ownerId));
+});
+
+test("checkForeignKeys finds a declared key the database lacks, and reports a real key nothing declares", async (t) => {
+  const knex = await petsDatabase(t);
+
+  const exact = await checkForeignKeys({
+    knex,
+    resources: [owners, petsResource([{ columns: ["ownerId"], referencedTableName: "owners" }])],
+    tables: ["owners", "pets"]
+  });
+  assert.deepEqual(exact, { missing: [], undeclared: [] });
+
+  const wrong = await checkForeignKeys({
+    knex,
+    resources: [petsResource([
+      { columns: ["ownerId"], referencedTableName: "owners", referencedColumns: ["id"] },
+      { name: "pets_vet", columns: ["vetId"], referencedTableName: "vets" }
+    ])],
+    tables: ["pets"]
+  });
+  assert.deepEqual(wrong.missing.map(({ resource, tableName, columns, referencedTableName, referencedColumns, declared }) =>
+    ({ resource, tableName, columns, referencedTableName, referencedColumns, name: declared.name })), [
+    { resource: "pets", tableName: "pets", columns: ["clinic_vet"], referencedTableName: "vets", referencedColumns: ["id"], name: "pets_vet" }
+  ]);
+  assert.deepEqual(wrong.undeclared, []);
+
+  const undeclared = await checkForeignKeys({ knex, resources: [owners, petsResource()], tables: ["pets"] });
+  assert.deepEqual(undeclared, {
+    missing: [],
+    undeclared: [{ name: "", tableName: "pets", columns: ["owner_id"], referencedTableName: "owners", referencedColumns: ["id"] }]
+  });
+  assert.deepEqual((await checkForeignKeys({ knex, resources: [petsResource()], tables: [] })).undeclared, []);
+});

package/test/hiddenFields.test.js

@@ -1,18 +1,19 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import knexLib from "knex";
 import { defineTableResource } from "@jskit-ai/resource-core/shared/tableResource";
 import { countQueries } from "@jskit-ai/database-runtime/server/testDatabases";
 import { withoutHiddenFields } from "../src/server/index.js";
-import { createTestResourceRuntime } from "../src/test/index.js";
+import { createPackageTestKnex, createTestResourceRuntime } from "../src/test/index.js";
+
+// SQLite in memory, or a PostgreSQL schema of its own on a DB_CLIENT=pg run (see createPackageTestKnex).
+const testKnex = (t) => createPackageTestKnex(t, { environment: process.env });
 
 const WORKSPACE_A = Object.freeze({ workspace: { id: "1" } });
 
 // Cards with a hidden owner, a hidden bearer token, a hidden flag, a hidden count and a hidden time: two in workspace
 // 1 and one in workspace 2.
 async function createCardFixture(t) {
-  const knex = knexLib({ client: "better-sqlite3", connection: { filename: ":memory:" }, useNullAsDefault: true });
-  t.after(() => knex.destroy());
+  const knex = await testKnex(t);
   await knex.schema.createTable("cards", (table) => {
     table.increments("id");
     table.integer("workspace_id").notNullable();
@@ -46,8 +47,8 @@ async function createCardFixture(t) {
   // The tables each SELECT reads, in order.
   const selects = async (work) => (await countQueries(knex, work)).queries
     .filter((sql) => /^select/iu.test(sql))
-    .map((sql) => [...sql.matchAll(/from `(\w+)`/gu)].map(([, table]) => table)[0]);
-  return { cards, resource, system, selects, options: { context: WORKSPACE_A } };
+    .map((sql) => [...sql.matchAll(/from [`"](\w+)[`"]/gu)].map(([, table]) => table)[0]);
+  return { knex, cards, resource, system, selects, options: { context: WORKSPACE_A } };
 }
 
 test("a handle's plain read answers the hidden fields its fields names, in one more statement, and no read without them does", async (t) => {
@@ -113,43 +114,54 @@ test("withoutHiddenFields refuses another feature's read naming a hidden field,
   assert.equal(withoutHiddenFields(cards, plain), cards);
 });
 
-test("withoutHiddenFields refuses another feature's scopedQuery, whose builder carries the hidden columns, and the owner's join source still joins", async (t) => {
-  const { cards, resource, system, options } = await createCardFixture(t);
-  // The owner's join source, a method its decorateRepository adds with the handle it receives.
+test("withoutHiddenFields refuses another feature's scopedQuery, and any builder a method answers but a joinSource()'s, which carries no hidden column", async (t) => {
+  const { knex, cards, resource, system, options } = await createCardFixture(t);
+  // Methods an owner's decorateRepository adds with the handle it receives, answering its scoped rows as scopedQuery()
+  // gives them, every column with them: wrapped, and at once.
   async function cardSource(alias, callOptions) {
     const { query } = await cards.scopedQuery({ alias }, callOptions);
     return { query, column: (field) => cards.column(field, alias) };
   }
-  const exposed = withoutHiddenFields(Object.freeze({ ...cards, cardSource }), resource);
+  const everyColumn = () => knex("cards");
+  const exposed = withoutHiddenFields(Object.freeze({ ...cards, cardSource, everyColumn }), resource);
 
   // Another feature's own builder would select the token: refused before it reads anything.
   await assert.rejects(
     (async () => (await exposed.scopedQuery({}, options)).query.select(exposed.column("publicToken")))(),
-    /cards\.scopedQuery\(\) answers every column, its hidden fields too, which only the table's owner, or a system context, reads/u
+    /cards\.scopedQuery\(\) answers every column, its hidden fields too, which only the table's owner, or a system context, reads: read the rows through the repository's reads, or join its joinSource\(\{ alias \}, options\)/u
   );
   await assert.rejects(exposed.withTransaction(options, (tx) => exposed.scopedQuery({ alias: "c" }, tx)), /cards\.scopedQuery\(\)/u);
+  await assert.rejects(exposed.cardSource("c", options),
+    /cards\.cardSource\(\) answers a query builder, which can select every column, the hidden fields too, which only the table's owner, or a system context, reads: answer the table's joinSource\(\{ alias \}, options\)/u);
+  assert.throws(() => exposed.everyColumn(), /cards\.everyColumn\(\) answers a query builder/u);
 
-  // A system context, and one it derived, still builds on it.
+  // A system context, and one it derived, still builds on them.
   const { query: everyWorkspace } = await exposed.scopedQuery({}, { context: system });
-  assert.deepEqual((await everyWorkspace.count({ cards: exposed.column("id") }))[0].cards, 3);
+  assert.deepEqual(Number((await everyWorkspace.count({ cards: exposed.column("id") }))[0].cards), 3);
   const { query: workspaceTwo } = await exposed.scopedQuery({}, { context: system.forWorkspace("2") });
   assert.deepEqual(await workspaceTwo.select({ title: exposed.column("title") }), [{ title: "Kit" }]);
+  const everyCard = await exposed.cardSource("c", { context: system });
+  assert.deepEqual(await everyCard.query.select({ token: everyCard.column("publicToken") }).orderBy(everyCard.column("id")),
+    [{ token: "t1" }, { token: "t2" }, { token: "t3" }]);
 
-  // The cross-owner report pattern: the owner's method answers its scoped rows, which the report selects from.
-  const source = await exposed.cardSource("c", options);
+  // The cross-owner report pattern: the table's joinSource() answers the caller's scoped rows, which the report
+  // selects from, with the columns a read answers and no other.
+  const source = await exposed.joinSource({ alias: "c" }, options);
   const rows = await source.query.select({ title: source.column("title") }).orderBy(source.column("id"));
   assert.deepEqual(rows, [{ title: "Rex" }, { title: "Bo" }]);
+  const { query: rex } = await exposed.joinSource({ alias: "c", filters: { title: "Rex" } }, options);
+  assert.deepEqual((await rex).map((row) => Object.keys(row)), [["id", "title"]]);
+  assert.throws(() => source.column("publicToken"), /cards\.joinSource\(\) carries no hidden field, so it has no column for "publicToken"/u);
+  const again = await exposed.joinSource({ alias: "c" }, options);
+  await assert.rejects(async () => again.query.select(cards.column("publicToken", "c")), /public_token/u);
+  await assert.rejects(exposed.joinSource({}, options), /cards\.joinSource\(\) needs the alias the report joins the cards under/u);
 });
 
-test("withoutHiddenFields answers raw(), column() and an owner's Knex values as they are, so another feature builds on them in a scoped query and a subquery", async (t) => {
+test("withoutHiddenFields answers raw(), column(), joinSource() and an owner's Knex values as they are, so another feature builds on them in a scoped query and a subquery", async (t) => {
   const { cards, resource, options } = await createCardFixture(t);
-  async function cardSource(alias, callOptions) {
-    const { query } = await cards.scopedQuery({ alias }, callOptions);
-    return { query, column: (field) => cards.column(field, alias) };
-  }
   // An owner's method answering a Knex value at once: an expression over its own column.
   const lowerTitle = (alias) => cards.raw("LOWER(??)", [cards.column("title", alias)]);
-  const exposed = withoutHiddenFields(Object.freeze({ ...cards, cardSource, lowerTitle }), resource);
+  const exposed = withoutHiddenFields(Object.freeze({ ...cards, lowerTitle }), resource);
 
   assert.equal(exposed.column("title", "c"), cards.column("title", "c"));
   const shout = exposed.raw("UPPER(??)", [exposed.column("title", "c")]);
@@ -157,24 +169,25 @@ test("withoutHiddenFields answers raw(), column() and an owner's Knex values as
   assert.equal(typeof exposed.lowerTitle("i").toSQL, "function");
 
   // In the owner's scoped rows, as a select.
-  const source = await exposed.cardSource("c", options);
+  const source = await exposed.joinSource({ alias: "c" }, options);
   assert.deepEqual(await source.query.select({ shout }).orderBy(source.column("id")), [{ shout: "REX" }, { shout: "BO" }]);
 
   // In a subquery: the cards of workspace 1 whose lower-cased title is "bo".
-  const outer = await exposed.cardSource("c", options);
-  const inner = await exposed.cardSource("i", options);
+  const outer = await exposed.joinSource({ alias: "c" }, options);
+  const inner = await exposed.joinSource({ alias: "i" }, options);
   const matching = inner.query.select(inner.column("id")).where(exposed.lowerTitle("i"), exposed.raw("?", ["bo"]));
   assert.deepEqual(await outer.query.whereIn(outer.column("id"), matching).select({ title: outer.column("title") }), [{ title: "Bo" }]);
 });
 
-test("withoutHiddenFields strips the hidden fields from every answer of a read the owner adds, unless it runs in a system context", async (t) => {
+test("withoutHiddenFields refuses a hidden field named through any method, and strips the hidden fields from every answer of a read the owner adds, unless it runs in a system context", async (t) => {
   const { cards, resource, system, options } = await createCardFixture(t);
   // The owner's decorateRepository reads: one forwarding the caller's params, the handle's own under another name, one
-  // answering a map, one in a sub-repository and one answering at once.
+  // taking its options first, one answering a map, one in a sub-repository and one answering at once.
   const owner = Object.freeze({
     ...cards,
     queryAllIncludingArchived: (params, callOptions) => cards.queryAll(params, callOptions),
     everyCard: cards.queryAll,
+    cardsTitled: (callOptions, { title }) => cards.queryAll({ filters: { title }, fields: ["title", "publicToken"] }, callOptions),
     async cardsByTitle(callOptions) {
       const records = await cards.queryAll({ fields: ["title", "publicToken"] }, callOptions);
       return new Map(records.map((record) => [record.title, record]));
@@ -184,10 +197,15 @@ test("withoutHiddenFields strips the hidden fields from every answer of a read t
   });
   const exposed = withoutHiddenFields(owner, resource);
 
-  // Another feature's calls: no hidden field leaves, whatever the method's name or depth.
-  assert.deepEqual(await exposed.queryAllIncludingArchived({ filters: { title: "Rex" }, fields: ["title", "publicToken", "workspaceId"] }, options),
-    [{ id: "1", title: "Rex" }]);
-  assert.deepEqual(await exposed.everyCard({ filters: { title: "Bo" }, fields: ["publicToken"] }, options), [{ id: "2" }]);
+  // Another feature's calls: a hidden field its fields names is refused before the method runs, whatever the method's
+  // name, as a list or a sparse fieldset, so a method forwarding the caller's fields never answers it as undefined.
+  await assert.rejects(exposed.queryAllIncludingArchived({ filters: { title: "Rex" }, fields: ["title", "publicToken", "workspaceId"] }, options),
+    /cards\.queryAllIncludingArchived\(\) fields names the hidden field "publicToken", "workspaceId", which only the table's owner, or a system context, reads/u);
+  await assert.rejects(exposed.everyCard({ filters: { title: "Bo" }, fields: { cards: "title, sent" } }, options),
+    /cards\.everyCard\(\) fields names the hidden field "sent"/u);
+  assert.deepEqual(await exposed.queryAllIncludingArchived({ filters: { title: "Rex" }, fields: ["title"] }, options), [{ id: "1", title: "Rex" }]);
+  // No hidden field the owner's own reads name leaves, whatever the method's name or depth.
+  assert.deepEqual(await exposed.cardsTitled(options, { title: "Rex" }), [{ id: "1", title: "Rex" }]);
   assert.deepEqual([...(await exposed.cardsByTitle(options)).values()], [{ id: "1", title: "Rex" }, { id: "2", title: "Bo" }]);
   assert.deepEqual(await exposed.archive.list(options), [{ id: "1", title: "Rex" }, { id: "2", title: "Bo" }]);
   // A method that answers at once still does, and withTransaction answers the caller's own work as it is.
@@ -200,4 +218,27 @@ test("withoutHiddenFields strips the hidden fields from every answer of a read t
   assert.deepEqual(await exposed.queryAllIncludingArchived({ filters: { title: "Kit" }, fields: ["publicToken", "workspaceId"] }, { context: system }),
     [{ id: "3", publicToken: "t3", workspaceId: "2" }]);
   assert.deepEqual(await exposed.archive.list({ context: system.forWorkspace("2") }), [{ id: "3", title: "Kit", views: 1 }]);
+  // The system context counts in whichever record carries it, not only the last.
+  assert.deepEqual(await exposed.cardsTitled({ context: system.forWorkspace("2") }, { title: "Kit" }), [{ id: "3", title: "Kit", publicToken: "t3" }]);
+});
+
+test("withoutHiddenFields refuses a field one owned table hides and another shows, since it strips hidden fields by name", () => {
+  const cards = defineTableResource({
+    name: "cards",
+    tableName: "cards",
+    ownership: "public",
+    schema: { title: { type: "string" }, publicToken: { type: "string", hidden: true } }
+  });
+  const links = (hidden) => defineTableResource({
+    name: "links",
+    tableName: "links",
+    ownership: "public",
+    schema: { publicToken: { type: "string", ...(hidden ? { hidden } : {}) } }
+  });
+  const repository = Object.freeze({ get: async () => ({ id: "1", publicToken: "t1" }) });
+  assert.throws(() => withoutHiddenFields(repository, [cards, links(false)]),
+    /links\.publicToken is a visible field, but cards, which the same feature owns, hides a field of that name; its repository strips hidden fields by name from every answer/u);
+  assert.throws(() => withoutHiddenFields(repository, [links(false), cards]), /links\.publicToken is a visible field, but cards/u);
+  // Hidden in both, it is one hidden field.
+  assert.equal(typeof withoutHiddenFields(repository, [cards, links(true)]).get, "function");
 });