@protobase/server 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/batch/run-batch.js +2 -1
- package/dist/changes/actor.js +12 -0
- package/dist/changes/capture-plan.js +62 -0
- package/dist/changes/capture-schema.d.ts +18 -0
- package/dist/changes/capture-schema.js +65 -0
- package/dist/changes/capture-sql.js +108 -0
- package/dist/changes/coalesce.d.ts +38 -0
- package/dist/changes/coalesce.js +69 -0
- package/dist/changes/events-options.d.ts +10 -0
- package/dist/changes/events-options.js +10 -0
- package/dist/changes/match-events.d.ts +9 -0
- package/dist/changes/match-events.js +55 -0
- package/dist/changes/resource-change.d.ts +27 -0
- package/dist/changes/resource-change.js +43 -0
- package/dist/create-admin.d.ts +7 -1
- package/dist/create-admin.js +7 -2
- package/dist/functions/require-env.d.ts +1 -1
- package/dist/index.d.ts +13 -2
- package/dist/index.js +10 -1
- package/dist/jobs/current.d.ts +8 -0
- package/dist/jobs/current.js +18 -0
- package/dist/jobs/database.js +36 -0
- package/dist/jobs/define-job.d.ts +8 -0
- package/dist/jobs/define-job.js +38 -0
- package/dist/jobs/drain-route.d.ts +21 -0
- package/dist/jobs/drain-route.js +28 -0
- package/dist/jobs/failed-jobs.js +67 -0
- package/dist/jobs/functions-sql.js +126 -0
- package/dist/jobs/job-types.d.ts +73 -0
- package/dist/jobs/options.js +44 -0
- package/dist/jobs/run-job.d.ts +9 -0
- package/dist/jobs/run-job.js +78 -0
- package/dist/jobs/runtime.d.ts +46 -0
- package/dist/jobs/runtime.js +156 -0
- package/dist/jobs/schedule.js +25 -0
- package/dist/jobs/schedules.js +41 -0
- package/dist/jobs/schema.d.ts +15 -0
- package/dist/jobs/schema.js +63 -0
- package/dist/project-config.d.ts +6 -1
- package/dist/project-config.js +18 -3
- package/dist/transactions.js +10 -3
- package/dist/write-pipeline.js +2 -1
- package/package.json +6 -4
package/dist/batch/run-batch.js
CHANGED
|
@@ -2,6 +2,7 @@ import { HttpProblem, badRequest, notFound } from "../problem.js";
|
|
|
2
2
|
import { problemOf } from "../error-handler.js";
|
|
3
3
|
import { requestAccess } from "../request-access.js";
|
|
4
4
|
import { writeTransaction } from "../transactions.js";
|
|
5
|
+
import { actorSettings } from "../changes/actor.js";
|
|
5
6
|
import { keyOf } from "../write-shared.js";
|
|
6
7
|
import { writeIn } from "../write-pipeline.js";
|
|
7
8
|
import { resolveData, resolveKey } from "./refs.js";
|
|
@@ -99,7 +100,7 @@ const runBatch = async (deps, session, operations) => {
|
|
|
99
100
|
throw atOperation(error, index, operation);
|
|
100
101
|
}
|
|
101
102
|
return results;
|
|
102
|
-
});
|
|
103
|
+
}, actorSettings(deps, session));
|
|
103
104
|
};
|
|
104
105
|
//#endregion
|
|
105
106
|
export { runBatch };
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
//#region src/changes/actor.ts
|
|
2
|
+
/** The setting a write transaction sets so the change capture records who wrote, when a resource has events. */
|
|
3
|
+
const actorSettings = (deps, session) => {
|
|
4
|
+
if (!deps.actorSetting) return {};
|
|
5
|
+
const actor = {
|
|
6
|
+
id: session.user.id,
|
|
7
|
+
roles: session.user.roles
|
|
8
|
+
};
|
|
9
|
+
return { [deps.actorSetting]: JSON.stringify(actor) };
|
|
10
|
+
};
|
|
11
|
+
//#endregion
|
|
12
|
+
export { actorSettings };
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { captureBody, captureFunctionSql, createTriggerSql, dropTriggerSql, outboxSql, quoteIdent, resourceTriggers, tableSql } from "./capture-sql.js";
|
|
2
|
+
//#region src/changes/capture-plan.ts
|
|
3
|
+
const sameTrigger = (found, wanted) => found.type === wanted.type && found.args.length === wanted.args.length && found.args.every((arg, index) => arg === wanted.args[index]);
|
|
4
|
+
const tableOf = (state, model) => {
|
|
5
|
+
const table = state.tables[model.name];
|
|
6
|
+
if (!table) throw new Error(`The table ${tableSql(model)} of resource "${model.name}" does not exist`);
|
|
7
|
+
return table;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Compares the database with the resources that declare events. Fails on what it cannot change safely: a missing
|
|
11
|
+
* table, two resources with events on one table, or a trigger of the same name that calls another function.
|
|
12
|
+
*/
|
|
13
|
+
const capturePlan = (schema, models, state) => {
|
|
14
|
+
const withEvents = models.filter((model) => (model.events ?? []).length > 0);
|
|
15
|
+
const capture = state.capture;
|
|
16
|
+
const current = capture !== void 0 && capture.definer && capture.source === captureBody(schema);
|
|
17
|
+
const wanted = withEvents.flatMap((model) => resourceTriggers(model).map((trigger) => ({
|
|
18
|
+
model,
|
|
19
|
+
trigger,
|
|
20
|
+
table: tableOf(state, model)
|
|
21
|
+
})));
|
|
22
|
+
const tables = /* @__PURE__ */ new Map();
|
|
23
|
+
for (const { model, table } of wanted) {
|
|
24
|
+
const other = tables.get(table);
|
|
25
|
+
if (other && other !== model.name) throw new Error(`Not implemented: resources "${other}" and "${model.name}" both have events on table ${tableSql(model)}; declare them on one`);
|
|
26
|
+
tables.set(table, model.name);
|
|
27
|
+
}
|
|
28
|
+
const create = wanted.filter(({ model, trigger, table }) => {
|
|
29
|
+
const found = state.triggers.find((candidate) => candidate.table === table && candidate.name === trigger.name);
|
|
30
|
+
if (found && found.function !== capture?.oid) throw new Error(`Table ${tableSql(model)} has a trigger "${trigger.name}" of its own; rename it, since events need that name`);
|
|
31
|
+
return !found || !sameTrigger(found, trigger);
|
|
32
|
+
});
|
|
33
|
+
const drop = state.triggers.filter((found) => found.function === capture?.oid).filter((found) => !wanted.some(({ trigger, table }) => table === found.table && trigger.name === found.name));
|
|
34
|
+
return {
|
|
35
|
+
setup: withEvents.length > 0 && (!state.outbox || !current),
|
|
36
|
+
create: create.map(({ model, trigger }) => ({
|
|
37
|
+
model,
|
|
38
|
+
trigger
|
|
39
|
+
})),
|
|
40
|
+
drop: drop.map(({ name, tableSql }) => ({
|
|
41
|
+
name,
|
|
42
|
+
tableSql
|
|
43
|
+
}))
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
/** The plan as SQL statements, each made to apply over what exists already; `undefined` when there is nothing to do. */
|
|
47
|
+
const planSql = (schema, plan) => {
|
|
48
|
+
const statements = [
|
|
49
|
+
...plan.setup ? [
|
|
50
|
+
`create schema if not exists ${quoteIdent(schema)}`,
|
|
51
|
+
outboxSql(schema),
|
|
52
|
+
captureFunctionSql(schema)
|
|
53
|
+
] : [],
|
|
54
|
+
...plan.drop.map(({ name, tableSql }) => dropTriggerSql(name, tableSql)),
|
|
55
|
+
...plan.create.flatMap(({ model, trigger }) => [dropTriggerSql(trigger.name, tableSql(model)), createTriggerSql(schema, trigger, tableSql(model))])
|
|
56
|
+
];
|
|
57
|
+
return statements.length === 0 ? void 0 : `${statements.join(";\n\n")};`;
|
|
58
|
+
};
|
|
59
|
+
/** The resources whose changes would not be captured as the config declares: what a server refuses to start with. */
|
|
60
|
+
const uncaptured = (plan, models) => plan.setup ? models.filter((model) => (model.events ?? []).length > 0).map((model) => model.name) : [...new Set(plan.create.map(({ model }) => model.name))];
|
|
61
|
+
//#endregion
|
|
62
|
+
export { capturePlan, planSql, uncaptured };
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { EventsOptions } from "./events-options.js";
|
|
2
|
+
import { ResourceModel } from "@protobase/schema";
|
|
3
|
+
import { Db } from "@protobase/query";
|
|
4
|
+
//#region src/changes/capture-schema.d.ts
|
|
5
|
+
/** The command that writes what the config needs as a migration file of the project. */
|
|
6
|
+
export declare const migrationCommand = "protobase migration";
|
|
7
|
+
/**
|
|
8
|
+
* The SQL that brings the database to the capture the resources' events need: the schema with the outbox and
|
|
9
|
+
* `capture()`, and a trigger pair per resource with events, recreated when its fields changed; triggers of resources
|
|
10
|
+
* without events any more are dropped. `undefined` when the database is current. Tables are never dropped.
|
|
11
|
+
*/
|
|
12
|
+
export declare const changesMigration: (db: Db, models: readonly ResourceModel[], options?: EventsOptions) => Promise<string | undefined>;
|
|
13
|
+
/**
|
|
14
|
+
* Refuses to go on while a resource's events would not be captured as declared: a missing or out-of-date schema, or
|
|
15
|
+
* trigger. The server never changes the schema itself: the project's migrations do, before the new version serves.
|
|
16
|
+
*/
|
|
17
|
+
export declare const checkChanges: (db: Db, models: readonly ResourceModel[], options?: EventsOptions) => Promise<void>;
|
|
18
|
+
//#endregion
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { changesSchema } from "./events-options.js";
|
|
2
|
+
import { quoteIdent, tableSql } from "./capture-sql.js";
|
|
3
|
+
import { capturePlan, planSql, uncaptured } from "./capture-plan.js";
|
|
4
|
+
import { sql } from "kysely";
|
|
5
|
+
//#region src/changes/capture-schema.ts
|
|
6
|
+
/** The command that writes what the config needs as a migration file of the project. */
|
|
7
|
+
const migrationCommand = "protobase migration";
|
|
8
|
+
const triggerArgs = (hex) => {
|
|
9
|
+
const bytes = new Uint8Array((hex.match(/../g) ?? []).map((pair) => parseInt(pair, 16)));
|
|
10
|
+
return new TextDecoder().decode(bytes).split("\0").slice(0, -1);
|
|
11
|
+
};
|
|
12
|
+
const rowsOf = async (db, query) => (await query.execute(db)).rows;
|
|
13
|
+
const readCaptureState = async (db, schema, models) => {
|
|
14
|
+
const [outbox] = await rowsOf(db, sql`select to_regclass(${`${quoteIdent(schema)}."outbox"`}) is not null as found`);
|
|
15
|
+
const [capture] = await rowsOf(db, sql`select p.oid::text as oid, p.prosrc as source, p.prosecdef as definer from pg_proc p join pg_namespace n on n.oid = p.pronamespace where n.nspname = ${schema} and p.proname = 'capture' and p.pronargs = 0`);
|
|
16
|
+
const tables = {};
|
|
17
|
+
for (const model of models) {
|
|
18
|
+
const [found] = await rowsOf(db, sql`select to_regclass(${tableSql(model)})::oid::text as oid`);
|
|
19
|
+
tables[model.name] = found?.oid ?? null;
|
|
20
|
+
}
|
|
21
|
+
const oids = Object.values(tables).filter((oid) => oid !== null);
|
|
22
|
+
const triggers = await rowsOf(db, sql`select t.tgrelid::text as "table", format('%I.%I', ns.nspname, c.relname) as "tableSql", t.tgname as name,
|
|
23
|
+
t.tgfoid::text as "function", t.tgtype::int as type, encode(t.tgargs, 'hex') as args
|
|
24
|
+
from pg_trigger t join pg_class c on c.oid = t.tgrelid join pg_namespace ns on ns.oid = c.relnamespace
|
|
25
|
+
where not t.tgisinternal and (t.tgfoid::text = ${capture?.oid ?? ""} or t.tgrelid::text = any(${oids}::text[]))`);
|
|
26
|
+
return {
|
|
27
|
+
outbox: outbox?.found ?? false,
|
|
28
|
+
...capture && { capture },
|
|
29
|
+
triggers: triggers.map((trigger) => ({
|
|
30
|
+
...trigger,
|
|
31
|
+
args: triggerArgs(trigger.args)
|
|
32
|
+
})),
|
|
33
|
+
tables
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
const plan = async (db, models, options) => {
|
|
37
|
+
const schema = changesSchema(options);
|
|
38
|
+
const withEvents = models.filter((model) => (model.events ?? []).length > 0);
|
|
39
|
+
return {
|
|
40
|
+
schema,
|
|
41
|
+
plan: capturePlan(schema, models, await readCaptureState(db, schema, withEvents))
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* The SQL that brings the database to the capture the resources' events need: the schema with the outbox and
|
|
46
|
+
* `capture()`, and a trigger pair per resource with events, recreated when its fields changed; triggers of resources
|
|
47
|
+
* without events any more are dropped. `undefined` when the database is current. Tables are never dropped.
|
|
48
|
+
*/
|
|
49
|
+
const changesMigration = async (db, models, options) => {
|
|
50
|
+
const { schema, plan: found } = await plan(db, models, options);
|
|
51
|
+
return planSql(schema, found);
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Refuses to go on while a resource's events would not be captured as declared: a missing or out-of-date schema, or
|
|
55
|
+
* trigger. The server never changes the schema itself: the project's migrations do, before the new version serves.
|
|
56
|
+
*/
|
|
57
|
+
const checkChanges = async (db, models, options) => {
|
|
58
|
+
if (!models.some((model) => model.events)) return;
|
|
59
|
+
const { plan: found } = await plan(db, models, options);
|
|
60
|
+
const missing = uncaptured(found, models);
|
|
61
|
+
if (missing.length === 0) return;
|
|
62
|
+
throw new Error(`The changes of ${missing.join(", ")} are not captured as their events need: their triggers or the changes schema are missing or out of date. Write the change as a migration with \`${migrationCommand}\`, apply it with the project's migrations, then start again.`);
|
|
63
|
+
};
|
|
64
|
+
//#endregion
|
|
65
|
+
export { changesMigration, checkChanges, migrationCommand };
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
//#region src/changes/capture-sql.ts
|
|
2
|
+
const quoteIdent = (name) => `"${name.replaceAll("\"", "\"\"")}"`;
|
|
3
|
+
const quoteLiteral = (text) => `'${text.replaceAll("'", "''")}'`;
|
|
4
|
+
const qualified = ({ schema, name }) => schema ? `${quoteIdent(schema)}.${quoteIdent(name)}` : quoteIdent(name);
|
|
5
|
+
/** A Postgres array literal of text, every element quoted, so the same columns always give the same text. */
|
|
6
|
+
const textArray = (items) => `{${items.map((item) => `"${item.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"")}"`).join(",")}}`;
|
|
7
|
+
const identifier = (name) => {
|
|
8
|
+
const encoder = new TextEncoder();
|
|
9
|
+
let kept = "";
|
|
10
|
+
for (const char of name) {
|
|
11
|
+
if (encoder.encode(kept + char).length > 63) break;
|
|
12
|
+
kept += char;
|
|
13
|
+
}
|
|
14
|
+
return kept;
|
|
15
|
+
};
|
|
16
|
+
/** The table as SQL names it, qualified when the resource names a schema. */
|
|
17
|
+
const tableSql = (model) => qualified(model.table);
|
|
18
|
+
/**
|
|
19
|
+
* The outbox: one row per changed row, written in the writer's transaction. `transaction` is the writer's transaction
|
|
20
|
+
* id; `dispatch_after` holds back a change that the process which wrote it is delivering itself.
|
|
21
|
+
*/
|
|
22
|
+
const outboxSql = (schema) => `create table if not exists ${quoteIdent(schema)}."outbox" (
|
|
23
|
+
"id" bigint generated always as identity primary key,
|
|
24
|
+
"transaction" bigint not null,
|
|
25
|
+
"resource" text not null,
|
|
26
|
+
"op" text not null,
|
|
27
|
+
"old" jsonb,
|
|
28
|
+
"new" jsonb,
|
|
29
|
+
"changed" text[],
|
|
30
|
+
"actor" jsonb,
|
|
31
|
+
"dispatch_after" timestamptz not null default now(),
|
|
32
|
+
"created_at" timestamptz not null default now()
|
|
33
|
+
)`;
|
|
34
|
+
/**
|
|
35
|
+
* The body of `capture()`: its arguments are the resource, the columns to keep and the sensitive columns, whose names
|
|
36
|
+
* it lists when they change but whose values it never keeps. It skips a transaction that set `<schema>.capture` to
|
|
37
|
+
* `off`, and an update that changed none of those columns.
|
|
38
|
+
*/
|
|
39
|
+
const captureBody = (schema) => `
|
|
40
|
+
declare
|
|
41
|
+
kept text[] := TG_ARGV[1]::text[];
|
|
42
|
+
hidden text[] := TG_ARGV[2]::text[];
|
|
43
|
+
old_row jsonb;
|
|
44
|
+
new_row jsonb;
|
|
45
|
+
changed text[];
|
|
46
|
+
begin
|
|
47
|
+
if current_setting('${schema}.capture', true) = 'off' then
|
|
48
|
+
return null;
|
|
49
|
+
end if;
|
|
50
|
+
if TG_OP <> 'INSERT' then
|
|
51
|
+
old_row := to_jsonb(OLD);
|
|
52
|
+
end if;
|
|
53
|
+
if TG_OP <> 'DELETE' then
|
|
54
|
+
new_row := to_jsonb(NEW);
|
|
55
|
+
end if;
|
|
56
|
+
if TG_OP = 'UPDATE' then
|
|
57
|
+
select array_agg(c.name order by c.position) into changed
|
|
58
|
+
from unnest(kept || hidden) with ordinality as c(name, position)
|
|
59
|
+
where old_row -> c.name is distinct from new_row -> c.name;
|
|
60
|
+
if changed is null then
|
|
61
|
+
return null;
|
|
62
|
+
end if;
|
|
63
|
+
end if;
|
|
64
|
+
insert into ${quoteIdent(schema)}."outbox" ("transaction", "resource", "op", "old", "new", "changed", "actor")
|
|
65
|
+
values (
|
|
66
|
+
pg_current_xact_id()::text::bigint,
|
|
67
|
+
TG_ARGV[0],
|
|
68
|
+
TG_OP,
|
|
69
|
+
(select jsonb_object_agg(o.key, o.value) from jsonb_each(old_row) as o where o.key = any(kept)),
|
|
70
|
+
(select jsonb_object_agg(n.key, n.value) from jsonb_each(new_row) as n where n.key = any(kept)),
|
|
71
|
+
changed,
|
|
72
|
+
nullif(current_setting('${schema}.actor', true), '')::jsonb
|
|
73
|
+
);
|
|
74
|
+
return null;
|
|
75
|
+
end
|
|
76
|
+
`;
|
|
77
|
+
/**
|
|
78
|
+
* `capture()` runs as its owner, with a fixed search path: a writer needs no rights on the schema, so another
|
|
79
|
+
* service writing as its own role keeps working once a resource has events.
|
|
80
|
+
*/
|
|
81
|
+
const captureFunctionSql = (schema) => `create or replace function ${quoteIdent(schema)}."capture"() returns trigger language plpgsql security definer set search_path = pg_catalog, pg_temp as $capture$${captureBody(schema)}$capture$`;
|
|
82
|
+
/** The two triggers of a resource with events: inserts and deletes, and updates that change the row. */
|
|
83
|
+
const resourceTriggers = (model) => {
|
|
84
|
+
const fields = Object.values(model.fields);
|
|
85
|
+
const args = [
|
|
86
|
+
model.name,
|
|
87
|
+
textArray(fields.filter((field) => !field.sensitive).map((field) => field.column)),
|
|
88
|
+
textArray(fields.filter((field) => field.sensitive).map((field) => field.column))
|
|
89
|
+
];
|
|
90
|
+
return [{
|
|
91
|
+
name: identifier(`${model.table.name}_changes`),
|
|
92
|
+
type: 13,
|
|
93
|
+
args
|
|
94
|
+
}, {
|
|
95
|
+
name: identifier(`${model.table.name}_changes_update`),
|
|
96
|
+
type: 17,
|
|
97
|
+
args
|
|
98
|
+
}];
|
|
99
|
+
};
|
|
100
|
+
const dropTriggerSql = (name, table) => `drop trigger if exists ${quoteIdent(name)} on ${table}`;
|
|
101
|
+
const createTriggerSql = (schema, trigger, table) => {
|
|
102
|
+
const timing = trigger.type & 16 ? "after update" : "after insert or delete";
|
|
103
|
+
const when = trigger.type & 16 ? " when (old.* is distinct from new.*)" : "";
|
|
104
|
+
const args = trigger.args.map(quoteLiteral).join(", ");
|
|
105
|
+
return `create trigger ${quoteIdent(trigger.name)} ${timing} on ${table} for each row${when} execute function ${quoteIdent(schema)}."capture"(${args})`;
|
|
106
|
+
};
|
|
107
|
+
//#endregion
|
|
108
|
+
export { captureBody, captureFunctionSql, createTriggerSql, dropTriggerSql, outboxSql, quoteIdent, resourceTriggers, tableSql };
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { Row } from "../types.js";
|
|
2
|
+
import { EventActor } from "@protobase/schema";
|
|
3
|
+
//#region src/changes/coalesce.d.ts
|
|
4
|
+
/** A row of the outbox: one row's insert, update or delete, its columns as `changes.capture()` kept them. */
|
|
5
|
+
export type CapturedRow = {
|
|
6
|
+
/** The outbox id: commit order for the changes of one record. */
|
|
7
|
+
id: string;
|
|
8
|
+
transaction: string;
|
|
9
|
+
/** The resource whose table changed, which its trigger names. */
|
|
10
|
+
resource: string;
|
|
11
|
+
op: 'INSERT' | 'UPDATE' | 'DELETE';
|
|
12
|
+
old: Row | null;
|
|
13
|
+
new: Row | null;
|
|
14
|
+
/** The columns an update changed, sensitive ones included; `null` for inserts and deletes. */
|
|
15
|
+
changed: string[] | null;
|
|
16
|
+
actor: EventActor | null;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* One record's changes in one transaction, as one: its columns at the start (`before`, none when it was inserted) and
|
|
20
|
+
* at commit (`after`, none when it was deleted), and for an update the columns that differ, sensitive ones included.
|
|
21
|
+
*/
|
|
22
|
+
export type CoalescedChange = {
|
|
23
|
+
transaction: string;
|
|
24
|
+
resource: string;
|
|
25
|
+
before: Row | null;
|
|
26
|
+
after: Row | null;
|
|
27
|
+
changed: string[];
|
|
28
|
+
actor: EventActor | null;
|
|
29
|
+
/** The id of its last outbox row. */
|
|
30
|
+
sequence: string;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* The outbox rows of one or more transactions, in id order, as one change per (transaction, record). An insert then a
|
|
34
|
+
* delete is nothing, and so is an update that ends where it started; a delete then an insert of the same key is an
|
|
35
|
+
* update. `keyColumns` names a resource's primary key columns, which identify its records.
|
|
36
|
+
*/
|
|
37
|
+
export declare const coalesce: (rows: readonly CapturedRow[], keyColumns: (resource: string) => readonly string[]) => CoalescedChange[];
|
|
38
|
+
//#endregion
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
//#region src/changes/coalesce.ts
|
|
2
|
+
const sameValue = (a, b) => a === b || JSON.stringify(a) === JSON.stringify(b);
|
|
3
|
+
const recordKey = (row, columns, keyColumns) => JSON.stringify([
|
|
4
|
+
row.transaction,
|
|
5
|
+
row.resource,
|
|
6
|
+
...keyColumns.map((column) => columns?.[column] ?? null)
|
|
7
|
+
]);
|
|
8
|
+
const changedColumns = (before, after, listed) => [.../* @__PURE__ */ new Set([
|
|
9
|
+
...listed,
|
|
10
|
+
...Object.keys(before),
|
|
11
|
+
...Object.keys(after)
|
|
12
|
+
])].filter((column) => {
|
|
13
|
+
return !(column in before || column in after) || !sameValue(before[column], after[column]);
|
|
14
|
+
});
|
|
15
|
+
const merged = ({ first, last, actor, listed }) => {
|
|
16
|
+
const base = {
|
|
17
|
+
transaction: first.transaction,
|
|
18
|
+
resource: first.resource,
|
|
19
|
+
actor,
|
|
20
|
+
sequence: last.id
|
|
21
|
+
};
|
|
22
|
+
const before = first.op === "INSERT" ? null : first.old;
|
|
23
|
+
const after = last.op === "DELETE" ? null : last.new;
|
|
24
|
+
if (!before && !after) return [];
|
|
25
|
+
if (!before || !after) return [{
|
|
26
|
+
...base,
|
|
27
|
+
before,
|
|
28
|
+
after,
|
|
29
|
+
changed: []
|
|
30
|
+
}];
|
|
31
|
+
const changed = changedColumns(before, after, listed);
|
|
32
|
+
return changed.length > 0 ? [{
|
|
33
|
+
...base,
|
|
34
|
+
before,
|
|
35
|
+
after,
|
|
36
|
+
changed
|
|
37
|
+
}] : [];
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* The outbox rows of one or more transactions, in id order, as one change per (transaction, record). An insert then a
|
|
41
|
+
* delete is nothing, and so is an update that ends where it started; a delete then an insert of the same key is an
|
|
42
|
+
* update. `keyColumns` names a resource's primary key columns, which identify its records.
|
|
43
|
+
*/
|
|
44
|
+
const coalesce = (rows, keyColumns) => {
|
|
45
|
+
const current = /* @__PURE__ */ new Map();
|
|
46
|
+
const groups = [];
|
|
47
|
+
for (const row of rows) {
|
|
48
|
+
const columns = keyColumns(row.resource);
|
|
49
|
+
const key = recordKey(row, row.op === "INSERT" ? row.new : row.old, columns);
|
|
50
|
+
let group = current.get(key);
|
|
51
|
+
if (!group) {
|
|
52
|
+
group = {
|
|
53
|
+
first: row,
|
|
54
|
+
last: row,
|
|
55
|
+
actor: row.actor,
|
|
56
|
+
listed: /* @__PURE__ */ new Set()
|
|
57
|
+
};
|
|
58
|
+
groups.push(group);
|
|
59
|
+
}
|
|
60
|
+
group.last = row;
|
|
61
|
+
group.actor = row.actor ?? group.actor;
|
|
62
|
+
for (const column of row.changed ?? []) group.listed.add(column);
|
|
63
|
+
current.delete(key);
|
|
64
|
+
current.set(row.op === "DELETE" ? key : recordKey(row, row.new, columns), group);
|
|
65
|
+
}
|
|
66
|
+
return groups.flatMap(merged);
|
|
67
|
+
};
|
|
68
|
+
//#endregion
|
|
69
|
+
export { coalesce };
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
//#region src/changes/events-options.d.ts
|
|
2
|
+
/** Where captured changes live in the app's database (`events` in `protobase.config.ts`). */
|
|
3
|
+
export type EventsOptions = {
|
|
4
|
+
/**
|
|
5
|
+
* The schema holding the outbox and the capture function, and the prefix of their transaction settings
|
|
6
|
+
* (`changes.actor`, `changes.capture`); default `changes`. Rename it when the database already has a schema by that name.
|
|
7
|
+
*/
|
|
8
|
+
schema?: string;
|
|
9
|
+
};
|
|
10
|
+
//#endregion
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
//#region src/changes/events-options.ts
|
|
2
|
+
const schemaPattern = /^[a-z_][a-z0-9_]*$/;
|
|
3
|
+
/** The schema of the options, checked: it also prefixes setting names, which take plain lowercase names only. */
|
|
4
|
+
const changesSchema = (options = {}) => {
|
|
5
|
+
const schema = options.schema ?? "changes";
|
|
6
|
+
if (!schemaPattern.test(schema)) throw new Error(`Invalid events schema "${schema}": use lowercase letters, digits and "_"`);
|
|
7
|
+
return schema;
|
|
8
|
+
};
|
|
9
|
+
//#endregion
|
|
10
|
+
export { changesSchema };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { ResourceChange } from "./resource-change.js";
|
|
2
|
+
import { EventMatch, ResourceModel } from "@protobase/schema";
|
|
3
|
+
//#region src/changes/match-events.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* The names of the resource's events a change raises, in declaration order. The filters are checked once, here; `matches`
|
|
6
|
+
* are the events' code predicates by event name (`eventMatches()` of the resource).
|
|
7
|
+
*/
|
|
8
|
+
export declare const eventMatcher: (model: ResourceModel, matches?: Record<string, EventMatch<any>>) => (change: ResourceChange) => string[];
|
|
9
|
+
//#endregion
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { checkEventFilter, evaluateFilter } from "@protobase/schema";
|
|
2
|
+
//#region src/changes/match-events.ts
|
|
3
|
+
const filterOf = (model, source) => {
|
|
4
|
+
const { filter, problem } = checkEventFilter(model, source);
|
|
5
|
+
if (!filter) throw new Error(`Resource "${model.name}": event filter "${source}" ${problem}`);
|
|
6
|
+
return filter;
|
|
7
|
+
};
|
|
8
|
+
const holds = (filter, record) => record !== void 0 && evaluateFilter(filter, record) === true;
|
|
9
|
+
const contextOf = (change) => ({
|
|
10
|
+
operation: change.operation,
|
|
11
|
+
...change.before && { before: change.before },
|
|
12
|
+
...change.after && { after: change.after },
|
|
13
|
+
changed: change.changed,
|
|
14
|
+
actor: change.actor,
|
|
15
|
+
source: change.source,
|
|
16
|
+
...change.tenant !== void 0 && { tenant: change.tenant },
|
|
17
|
+
transaction: change.transaction
|
|
18
|
+
});
|
|
19
|
+
const eventTest = (model, event, match) => {
|
|
20
|
+
const tests = [(change) => event.operations.includes(change.operation)];
|
|
21
|
+
if (event.when !== void 0) {
|
|
22
|
+
const when = filterOf(model, event.when);
|
|
23
|
+
tests.push((change) => holds(when, change.after));
|
|
24
|
+
}
|
|
25
|
+
if (event.from !== void 0) {
|
|
26
|
+
const from = filterOf(model, event.from);
|
|
27
|
+
tests.push((change) => holds(from, change.before));
|
|
28
|
+
}
|
|
29
|
+
if (event.becomes !== void 0) {
|
|
30
|
+
const condition = filterOf(model, event.becomes);
|
|
31
|
+
tests.push((change) => holds(condition, change.after) && !holds(condition, change.before));
|
|
32
|
+
}
|
|
33
|
+
if (event.changed) {
|
|
34
|
+
const fields = event.changed;
|
|
35
|
+
tests.push((change) => change.changed.some((name) => fields.includes(name)));
|
|
36
|
+
}
|
|
37
|
+
if (event.match) {
|
|
38
|
+
if (!match) throw new Error(`Resource "${model.name}": event "${event.name}" has no match function`);
|
|
39
|
+
tests.push((change) => match(contextOf(change)));
|
|
40
|
+
}
|
|
41
|
+
return (change) => tests.every((test) => test(change));
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* The names of the resource's events a change raises, in declaration order. The filters are checked once, here; `matches`
|
|
45
|
+
* are the events' code predicates by event name (`eventMatches()` of the resource).
|
|
46
|
+
*/
|
|
47
|
+
const eventMatcher = (model, matches = {}) => {
|
|
48
|
+
const tests = (model.events ?? []).map((event) => ({
|
|
49
|
+
name: event.name,
|
|
50
|
+
test: eventTest(model, event, matches[event.name])
|
|
51
|
+
}));
|
|
52
|
+
return (change) => tests.filter(({ test }) => test(change)).map(({ name }) => name);
|
|
53
|
+
};
|
|
54
|
+
//#endregion
|
|
55
|
+
export { eventMatcher };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { Row, TenantValue } from "../types.js";
|
|
2
|
+
import { CoalescedChange } from "./coalesce.js";
|
|
3
|
+
import { EventActor, EventOperation, ResourceModel } from "@protobase/schema";
|
|
4
|
+
//#region src/changes/resource-change.d.ts
|
|
5
|
+
/** One record's change in one transaction, by field name: what events are matched against and payloads are built from. */
|
|
6
|
+
export type ResourceChange = {
|
|
7
|
+
resource: string;
|
|
8
|
+
/** The record's key as its URL has it. */
|
|
9
|
+
key: string;
|
|
10
|
+
operation: EventOperation;
|
|
11
|
+
/** The record before and after the change, without sensitive fields. */
|
|
12
|
+
before?: Row;
|
|
13
|
+
after?: Row;
|
|
14
|
+
/** The fields an update changed, sensitive ones included. */
|
|
15
|
+
changed: string[];
|
|
16
|
+
actor: EventActor | null;
|
|
17
|
+
source: 'api' | 'database';
|
|
18
|
+
tenant?: TenantValue;
|
|
19
|
+
transaction: string;
|
|
20
|
+
/** Grows with every change of a record, so receivers can drop stale events. */
|
|
21
|
+
sequence: string;
|
|
22
|
+
};
|
|
23
|
+
/** A captured row by field name, fields without a captured column left out. */
|
|
24
|
+
export declare const recordOf: (model: ResourceModel, columns: Row) => Row;
|
|
25
|
+
/** A coalesced change of the resource's table, by field name, with deletes and undeletes told apart from updates. */
|
|
26
|
+
export declare const resourceChange: (model: ResourceModel, change: CoalescedChange) => ResourceChange;
|
|
27
|
+
//#endregion
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { encodeKey } from "@protobase/schema";
|
|
2
|
+
//#region src/changes/resource-change.ts
|
|
3
|
+
const fieldValue = (type, value) => {
|
|
4
|
+
if (value === null || value === void 0) return value;
|
|
5
|
+
if (type === "bigint" || type === "decimal") return String(value);
|
|
6
|
+
if (type === "integer") return Number(value);
|
|
7
|
+
return value;
|
|
8
|
+
};
|
|
9
|
+
/** A captured row by field name, fields without a captured column left out. */
|
|
10
|
+
const recordOf = (model, columns) => Object.fromEntries(Object.values(model.fields).filter((field) => Object.hasOwn(columns, field.column)).map((field) => [field.name, fieldValue(field.type, columns[field.column])]));
|
|
11
|
+
const operationOf = (model, before, after) => {
|
|
12
|
+
if (!before) return "create";
|
|
13
|
+
if (!after) return "delete";
|
|
14
|
+
if (!model.softDelete) return "update";
|
|
15
|
+
const wasDeleted = before[model.softDelete] != null;
|
|
16
|
+
const isDeleted = after[model.softDelete] != null;
|
|
17
|
+
if (!wasDeleted && isDeleted) return "delete";
|
|
18
|
+
if (wasDeleted && !isDeleted) return "undelete";
|
|
19
|
+
return "update";
|
|
20
|
+
};
|
|
21
|
+
const fieldsOf = (model, columns) => Object.values(model.fields).filter((field) => columns.includes(field.column)).map((field) => field.name);
|
|
22
|
+
/** A coalesced change of the resource's table, by field name, with deletes and undeletes told apart from updates. */
|
|
23
|
+
const resourceChange = (model, change) => {
|
|
24
|
+
const before = change.before ? recordOf(model, change.before) : void 0;
|
|
25
|
+
const after = change.after ? recordOf(model, change.after) : void 0;
|
|
26
|
+
const record = after ?? before;
|
|
27
|
+
const tenant = model.tenant === void 0 ? void 0 : record[model.tenant];
|
|
28
|
+
return {
|
|
29
|
+
resource: model.name,
|
|
30
|
+
key: encodeKey(model.primaryKey.map((name) => record[name])),
|
|
31
|
+
operation: operationOf(model, before, after),
|
|
32
|
+
...before && { before },
|
|
33
|
+
...after && { after },
|
|
34
|
+
changed: fieldsOf(model, change.changed),
|
|
35
|
+
actor: change.actor,
|
|
36
|
+
source: change.actor ? "api" : "database",
|
|
37
|
+
...tenant != null && { tenant },
|
|
38
|
+
transaction: change.transaction,
|
|
39
|
+
sequence: change.sequence
|
|
40
|
+
};
|
|
41
|
+
};
|
|
42
|
+
//#endregion
|
|
43
|
+
export { recordOf, resourceChange };
|
package/dist/create-admin.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { AdminAuth } from "./better-auth/create-auth.js";
|
|
2
|
+
import { EventsOptions } from "./changes/events-options.js";
|
|
2
3
|
import { FilesOptions } from "./files/options.js";
|
|
3
4
|
import { AdminEnv, Authenticator, PageSource, ViewSource } from "./types.js";
|
|
4
5
|
import { ResourceSource } from "./resource-source.js";
|
|
5
6
|
import { FunctionSource } from "./functions/define-function.js";
|
|
6
7
|
import { AdminOptions } from "./admin-options.js";
|
|
8
|
+
import { JobDefinition } from "./jobs/job-types.js";
|
|
7
9
|
import { Hono } from "hono";
|
|
8
10
|
import { UserMenuSource } from "@protobase/schema";
|
|
9
11
|
import { Db } from "@protobase/query";
|
|
@@ -23,8 +25,12 @@ export type CreateAdminInput = {
|
|
|
23
25
|
functions?: Record<string, FunctionSource>;
|
|
24
26
|
/** Where file fields keep their bytes (see `FilesOptions`). */
|
|
25
27
|
files?: FilesOptions;
|
|
28
|
+
/** Where the changes that resources' events are raised from are captured (see `EventsOptions`). */
|
|
29
|
+
events?: EventsOptions;
|
|
30
|
+
/** The app's jobs (see `defineJob`): with any, admins get the failed jobs page (`failedJobs`). */
|
|
31
|
+
jobs?: JobDefinition[];
|
|
26
32
|
options?: AdminOptions;
|
|
27
33
|
};
|
|
28
34
|
/** The admin REST API as a Hono app. It uses only Web APIs, so it runs on Node, Workers and anywhere else Hono does. */
|
|
29
|
-
export declare const createAdmin: ({ resources, views, pages, userMenu, db, authenticate, auth, functions, files: filesOptions, options }: CreateAdminInput) => Hono<AdminEnv, import("hono/types").BlankSchema, "/">;
|
|
35
|
+
export declare const createAdmin: ({ resources: appResources, views: appViews, pages, userMenu, db, authenticate, auth, functions, files: filesOptions, events, jobs, options }: CreateAdminInput) => Hono<AdminEnv, import("hono/types").BlankSchema, "/">;
|
|
30
36
|
//#endregion
|
package/dist/create-admin.js
CHANGED
|
@@ -6,6 +6,7 @@ import { consoleAuditQueue } from "./audit/console-queue.js";
|
|
|
6
6
|
import { authBasePath } from "./better-auth/create-auth.js";
|
|
7
7
|
import { organizationRoutes } from "./better-auth/organizations/admin-routes.js";
|
|
8
8
|
import { statusRoute } from "./better-auth/status-route.js";
|
|
9
|
+
import { changesSchema } from "./changes/events-options.js";
|
|
9
10
|
import { checkShell } from "./check-shell.js";
|
|
10
11
|
import { checkViews } from "./check-views.js";
|
|
11
12
|
import { createFilesRuntime } from "./files/runtime.js";
|
|
@@ -22,12 +23,15 @@ import { docsRoutes } from "./routes/docs.js";
|
|
|
22
23
|
import { metaRoutes } from "./routes/meta.js";
|
|
23
24
|
import { recordRoutes } from "./routes/record.js";
|
|
24
25
|
import { createScanGuard } from "./scan-guard.js";
|
|
26
|
+
import { failedJobs, failedJobsView } from "./jobs/failed-jobs.js";
|
|
25
27
|
import { Hono } from "hono";
|
|
26
28
|
import { checkPages } from "@protobase/layout";
|
|
27
29
|
//#region src/create-admin.ts
|
|
28
30
|
/** The admin REST API as a Hono app. It uses only Web APIs, so it runs on Node, Workers and anywhere else Hono does. */
|
|
29
|
-
const createAdmin = ({ resources, views = [], pages = [], userMenu, db, authenticate, auth, functions = {}, files: filesOptions, options = {} }) => {
|
|
31
|
+
const createAdmin = ({ resources: appResources, views: appViews = [], pages = [], userMenu, db, authenticate, auth, functions = {}, files: filesOptions, events, jobs = [], options = {} }) => {
|
|
30
32
|
const basePath = options.basePath ?? "/api/v1";
|
|
33
|
+
const resources = jobs.length > 0 ? [...appResources, failedJobs] : appResources;
|
|
34
|
+
const views = jobs.length > 0 ? [...appViews, failedJobsView] : appViews;
|
|
31
35
|
const registry = buildRegistry(resources);
|
|
32
36
|
const apiFunctions = Object.fromEntries(Object.entries(functions).map(([name, source]) => [name, toApiFunction(name, source)]));
|
|
33
37
|
const viewModels = views.map((view) => view.toModel());
|
|
@@ -67,7 +71,8 @@ const createAdmin = ({ resources, views = [], pages = [], userMenu, db, authenti
|
|
|
67
71
|
rowPermissions: (resource) => typeof options.rowPermissions === "object" ? !options.rowPermissions.except.includes(resource) : options.rowPermissions !== false,
|
|
68
72
|
...assistant && { assistant: assistant.kind === "external" ? assistant.url : `${systemPath}/assistant` },
|
|
69
73
|
...runtime && { runtime },
|
|
70
|
-
...files && { files }
|
|
74
|
+
...files && { files },
|
|
75
|
+
...registry.entries.some((entry) => entry.model.events) && { actorSetting: `${changesSchema(events)}.actor` }
|
|
71
76
|
};
|
|
72
77
|
const meta = createMeta(deps);
|
|
73
78
|
const onError = createErrorHandler(options.onUnhandledError);
|
|
@@ -3,5 +3,5 @@
|
|
|
3
3
|
* An environment variable the function cannot work without, such as an API key: `protobase dev` reads it from `.env`,
|
|
4
4
|
* `protobase serve` from its environment. A missing one is an error, so the call is a 500 the app reports.
|
|
5
5
|
*/
|
|
6
|
-
export declare const requireEnv: (name: string) =>
|
|
6
|
+
export declare const requireEnv: (name: string) => string;
|
|
7
7
|
//#endregion
|