@protobase/server 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/batch/run-batch.js +2 -1
- package/dist/changes/actor.js +12 -0
- package/dist/changes/capture-plan.js +62 -0
- package/dist/changes/capture-schema.d.ts +18 -0
- package/dist/changes/capture-schema.js +65 -0
- package/dist/changes/capture-sql.js +108 -0
- package/dist/changes/coalesce.d.ts +38 -0
- package/dist/changes/coalesce.js +69 -0
- package/dist/changes/events-options.d.ts +10 -0
- package/dist/changes/events-options.js +10 -0
- package/dist/changes/match-events.d.ts +9 -0
- package/dist/changes/match-events.js +55 -0
- package/dist/changes/resource-change.d.ts +27 -0
- package/dist/changes/resource-change.js +43 -0
- package/dist/create-admin.d.ts +7 -1
- package/dist/create-admin.js +7 -2
- package/dist/functions/require-env.d.ts +1 -1
- package/dist/index.d.ts +13 -2
- package/dist/index.js +10 -1
- package/dist/jobs/current.d.ts +8 -0
- package/dist/jobs/current.js +18 -0
- package/dist/jobs/database.js +36 -0
- package/dist/jobs/define-job.d.ts +8 -0
- package/dist/jobs/define-job.js +38 -0
- package/dist/jobs/drain-route.d.ts +21 -0
- package/dist/jobs/drain-route.js +28 -0
- package/dist/jobs/failed-jobs.js +67 -0
- package/dist/jobs/functions-sql.js +126 -0
- package/dist/jobs/job-types.d.ts +73 -0
- package/dist/jobs/options.js +44 -0
- package/dist/jobs/run-job.d.ts +9 -0
- package/dist/jobs/run-job.js +78 -0
- package/dist/jobs/runtime.d.ts +46 -0
- package/dist/jobs/runtime.js +156 -0
- package/dist/jobs/schedule.js +25 -0
- package/dist/jobs/schedules.js +41 -0
- package/dist/jobs/schema.d.ts +15 -0
- package/dist/jobs/schema.js +63 -0
- package/dist/project-config.d.ts +6 -1
- package/dist/project-config.js +18 -3
- package/dist/transactions.js +10 -3
- package/dist/write-pipeline.js +2 -1
- package/package.json +6 -4
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
//#region src/jobs/run-job.ts
|
|
2
|
+
/** Thrown from a job's `run`, it fails the job now: the attempts it has left are not used. */
|
|
3
|
+
var FatalJobError = class extends Error {
|
|
4
|
+
name = "FatalJobError";
|
|
5
|
+
};
|
|
6
|
+
/** The payload as the job's `payload` schema returns it, or why it does not pass. */
|
|
7
|
+
const validatePayload = async (definition, payload) => {
|
|
8
|
+
const schema = definition.options.payload;
|
|
9
|
+
if (!schema) return { value: payload };
|
|
10
|
+
const result = await schema["~standard"].validate(payload);
|
|
11
|
+
if (!result.issues) return { value: result.value };
|
|
12
|
+
const issues = result.issues.map((issue) => {
|
|
13
|
+
const field = (issue.path ?? []).map((part) => String(typeof part === "object" ? part.key : part)).join(".");
|
|
14
|
+
return field ? `${field}: ${issue.message}` : issue.message;
|
|
15
|
+
});
|
|
16
|
+
return { invalid: `Job "${definition.name}" has an invalid payload: ${issues.join("; ")}` };
|
|
17
|
+
};
|
|
18
|
+
const checkPayload = async (definition, payload) => {
|
|
19
|
+
const checked = await validatePayload(definition, payload);
|
|
20
|
+
if ("invalid" in checked) throw new FatalJobError(checked.invalid);
|
|
21
|
+
return checked.value;
|
|
22
|
+
};
|
|
23
|
+
const readOnly = (error) => error?.code === "25006";
|
|
24
|
+
const pause = (ms, signal) => new Promise((resolve) => {
|
|
25
|
+
if (signal.aborted) return resolve();
|
|
26
|
+
const timer = setTimeout(resolve, ms);
|
|
27
|
+
signal.addEventListener("abort", () => {
|
|
28
|
+
clearTimeout(timer);
|
|
29
|
+
resolve();
|
|
30
|
+
}, { once: true });
|
|
31
|
+
});
|
|
32
|
+
/** Runs `run` again every `waitMs` while the database is read-only, until the signal ends the try. */
|
|
33
|
+
const throughWriteFreeze = async (run, signal, waitMs = 5e3) => {
|
|
34
|
+
for (;;) try {
|
|
35
|
+
return await run();
|
|
36
|
+
} catch (error) {
|
|
37
|
+
if (!readOnly(error) || signal.aborted) throw error;
|
|
38
|
+
await pause(waitMs, signal);
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
/** What a failed try keeps as the job's output, which the admin app shows as its last error. */
|
|
42
|
+
const errorOutput = (error) => error instanceof Error ? {
|
|
43
|
+
name: error.name,
|
|
44
|
+
message: error.message,
|
|
45
|
+
...error.stack && { stack: error.stack }
|
|
46
|
+
} : { message: String(error) };
|
|
47
|
+
/**
|
|
48
|
+
* One try of one job: completed when `run` returns, failed (to be retried while attempts are left) when it throws, and
|
|
49
|
+
* failed for good on a `FatalJobError` or an invalid payload.
|
|
50
|
+
*/
|
|
51
|
+
const runJob = async (definition, job, db) => {
|
|
52
|
+
try {
|
|
53
|
+
const payload = await checkPayload(definition, job.data ?? void 0);
|
|
54
|
+
const context = {
|
|
55
|
+
db,
|
|
56
|
+
job: {
|
|
57
|
+
id: job.id,
|
|
58
|
+
name: job.name,
|
|
59
|
+
attempt: job.retryCount + 1,
|
|
60
|
+
maxAttempts: job.retryLimit + 1
|
|
61
|
+
},
|
|
62
|
+
signal: job.signal
|
|
63
|
+
};
|
|
64
|
+
await throughWriteFreeze(() => definition.options.run(payload, context), job.signal);
|
|
65
|
+
return {
|
|
66
|
+
id: job.id,
|
|
67
|
+
status: "completed"
|
|
68
|
+
};
|
|
69
|
+
} catch (error) {
|
|
70
|
+
return {
|
|
71
|
+
id: job.id,
|
|
72
|
+
status: error instanceof FatalJobError ? "deadletter" : "failed",
|
|
73
|
+
output: errorOutput(error)
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
//#endregion
|
|
78
|
+
export { FatalJobError, runJob, throughWriteFreeze, validatePayload };
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { JobDefinition } from "./job-types.js";
|
|
2
|
+
import { JobsEnqueuer } from "./current.js";
|
|
3
|
+
import { Db } from "@protobase/query";
|
|
4
|
+
//#region src/jobs/runtime.d.ts
|
|
5
|
+
export type StartJobsOptions = {
|
|
6
|
+
db: Db;
|
|
7
|
+
jobs: JobDefinition[];
|
|
8
|
+
/** Runs the jobs in this process; `false` only enqueues them, for a process whose jobs another one runs. Default true. */
|
|
9
|
+
work?: boolean;
|
|
10
|
+
/** Seconds between a worker's checks for due jobs; an enqueue outside a transaction wakes this process's worker at once. Default 2. */
|
|
11
|
+
pollSeconds?: number;
|
|
12
|
+
/** Seconds running jobs get to finish when the jobs stop, before they are failed and retried later. Default 30. */
|
|
13
|
+
shutdownSeconds?: number;
|
|
14
|
+
/** Gets every error the workers and the queue maintenance meet outside a job's own `run`. */
|
|
15
|
+
onError: (error: unknown) => void;
|
|
16
|
+
/** The time recurring jobs are scheduled from; tests pass their own. */
|
|
17
|
+
now?: () => Date;
|
|
18
|
+
};
|
|
19
|
+
/** What a drain left: whether no job is due or running any more, how many are, and when the next one is due. */
|
|
20
|
+
export type DrainResult = {
|
|
21
|
+
drained: boolean;
|
|
22
|
+
remaining: number;
|
|
23
|
+
nextDue: Date | null;
|
|
24
|
+
};
|
|
25
|
+
export type JobsRuntime = JobsEnqueuer & {
|
|
26
|
+
/** Whether this process runs the jobs, or only enqueues them. */
|
|
27
|
+
readonly works: boolean;
|
|
28
|
+
/** Takes the project's jobs again, as `protobase dev` does when their files change. */
|
|
29
|
+
update(jobs: JobDefinition[]): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Runs the jobs that are due until none is due or running, or `seconds` have passed: for a host that wakes a sleeping
|
|
32
|
+
* app to run its jobs. Needs this process to run jobs.
|
|
33
|
+
*/
|
|
34
|
+
drain(seconds: number): Promise<DrainResult>;
|
|
35
|
+
/** When the next job is due, or `null` when none waits (`jobs.next_due()`). */
|
|
36
|
+
nextDue(): Promise<Date | null>;
|
|
37
|
+
/** Stops taking jobs, lets running ones finish within `shutdownSeconds`, then fails and releases the rest. */
|
|
38
|
+
stop(): Promise<void>;
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Starts the app's jobs in this process: their queues, a worker per job unless `work` is false, and the maintenance
|
|
42
|
+
* that retries jobs whose worker died. It reads the jobs schema, never changes it: check it first with
|
|
43
|
+
* `checkJobsSchema`. Enqueueing through `defineJob(...).enqueue` reaches the runtime started here.
|
|
44
|
+
*/
|
|
45
|
+
export declare const startJobs: ({ db, jobs, work, pollSeconds, shutdownSeconds, onError, now }: StartJobsOptions) => Promise<JobsRuntime>;
|
|
46
|
+
//#endregion
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { jobsSchema } from "./schema.js";
|
|
2
|
+
import { clearCurrentJobs, setCurrentJobs } from "./current.js";
|
|
3
|
+
import "./schedule.js";
|
|
4
|
+
import { runJob, validatePayload } from "./run-job.js";
|
|
5
|
+
import { jobsDatabase } from "./database.js";
|
|
6
|
+
import { queueOptions, queueSettingsDiffer, sendOptions } from "./options.js";
|
|
7
|
+
import { createSchedules } from "./schedules.js";
|
|
8
|
+
import { sql } from "kysely";
|
|
9
|
+
import { PgBoss } from "pg-boss";
|
|
10
|
+
//#region src/jobs/runtime.ts
|
|
11
|
+
const createBoss = (db, work) => new PgBoss({
|
|
12
|
+
db: jobsDatabase(db),
|
|
13
|
+
schema: jobsSchema,
|
|
14
|
+
migrate: false,
|
|
15
|
+
createSchema: false,
|
|
16
|
+
schedule: false,
|
|
17
|
+
supervise: work,
|
|
18
|
+
registerInstance: false,
|
|
19
|
+
reindex: false,
|
|
20
|
+
monitorVacuum: false,
|
|
21
|
+
useListenNotify: false
|
|
22
|
+
});
|
|
23
|
+
/**
|
|
24
|
+
* Starts the app's jobs in this process: their queues, a worker per job unless `work` is false, and the maintenance
|
|
25
|
+
* that retries jobs whose worker died. It reads the jobs schema, never changes it: check it first with
|
|
26
|
+
* `checkJobsSchema`. Enqueueing through `defineJob(...).enqueue` reaches the runtime started here.
|
|
27
|
+
*/
|
|
28
|
+
const startJobs = async ({ db, jobs, work = true, pollSeconds = 2, shutdownSeconds = 30, onError, now = () => /* @__PURE__ */ new Date() }) => {
|
|
29
|
+
const boss = createBoss(db, work);
|
|
30
|
+
boss.on("error", onError);
|
|
31
|
+
await boss.start();
|
|
32
|
+
const schedules = createSchedules(boss, db, now);
|
|
33
|
+
const definitions = /* @__PURE__ */ new Map();
|
|
34
|
+
const workers = /* @__PURE__ */ new Map();
|
|
35
|
+
const ensureQueue = async (definition) => {
|
|
36
|
+
const wanted = queueOptions(definition.options);
|
|
37
|
+
const queue = await boss.getQueue(definition.name);
|
|
38
|
+
if (!queue) return boss.createQueue(definition.name, wanted);
|
|
39
|
+
if (queue.policy !== wanted.policy) throw new Error(`Job "${definition.name}" ${definition.options.ordered ? "became" : "is no longer"} ordered, but its queue keeps the old order. Let its jobs finish or delete them, drop the queue with \`select ${jobsSchema}.delete_queue('${definition.name}')\`, then start again.`);
|
|
40
|
+
if (queueSettingsDiffer(wanted, queue)) {
|
|
41
|
+
const { policy, ...settings } = wanted;
|
|
42
|
+
await boss.updateQueue(definition.name, settings);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
const startWorker = async (definition) => {
|
|
46
|
+
const concurrency = definition.options.concurrency ?? 1;
|
|
47
|
+
const running = workers.get(definition.name);
|
|
48
|
+
if (running?.concurrency === concurrency) return;
|
|
49
|
+
if (running) await boss.offWork(definition.name);
|
|
50
|
+
const id = await boss.work(definition.name, {
|
|
51
|
+
batchSize: 1,
|
|
52
|
+
localConcurrency: concurrency,
|
|
53
|
+
pollingIntervalSeconds: pollSeconds,
|
|
54
|
+
includeMetadata: true,
|
|
55
|
+
perJobResults: true
|
|
56
|
+
}, async ([job]) => {
|
|
57
|
+
const current = definitions.get(definition.name) ?? definition;
|
|
58
|
+
if (current.options.schedule !== void 0 && job.singletonKey === "protobase.schedule") await schedules.following(current, job.startAfter);
|
|
59
|
+
return [await runJob(current, job, db)];
|
|
60
|
+
});
|
|
61
|
+
workers.set(definition.name, {
|
|
62
|
+
id,
|
|
63
|
+
concurrency
|
|
64
|
+
});
|
|
65
|
+
};
|
|
66
|
+
const update = async (next) => {
|
|
67
|
+
for (const definition of next) await ensureQueue(definition);
|
|
68
|
+
for (const name of definitions.keys()) {
|
|
69
|
+
if (next.some((definition) => definition.name === name)) continue;
|
|
70
|
+
definitions.delete(name);
|
|
71
|
+
if (workers.delete(name)) await boss.offWork(name);
|
|
72
|
+
}
|
|
73
|
+
for (const definition of next) definitions.set(definition.name, definition);
|
|
74
|
+
if (!work) return;
|
|
75
|
+
for (const definition of next) await startWorker(definition);
|
|
76
|
+
await schedules.sync(next);
|
|
77
|
+
};
|
|
78
|
+
const enqueue = async (definition, target, payload, options = {}) => {
|
|
79
|
+
const { name } = definition;
|
|
80
|
+
if (!definitions.has(name)) throw new Error(`Job "${name}" is not among the app's jobs; export it from the module protobase.config.ts passes as \`jobs\``);
|
|
81
|
+
const checked = await validatePayload(definition, payload);
|
|
82
|
+
if ("invalid" in checked) throw new Error(checked.invalid);
|
|
83
|
+
const data = checked.value ?? null;
|
|
84
|
+
const send = {
|
|
85
|
+
...sendOptions(definition.options, options),
|
|
86
|
+
db: jobsDatabase(target)
|
|
87
|
+
};
|
|
88
|
+
const keyed = async (key) => {
|
|
89
|
+
if (options.mode !== "replace") {
|
|
90
|
+
const [waiting] = await boss.findJobs(name, {
|
|
91
|
+
key,
|
|
92
|
+
queued: true,
|
|
93
|
+
db: send.db
|
|
94
|
+
});
|
|
95
|
+
if (waiting) return waiting.id;
|
|
96
|
+
}
|
|
97
|
+
return (await boss.upsert(name, data, send)).jobs[0];
|
|
98
|
+
};
|
|
99
|
+
const id = options.key === void 0 ? await boss.send(name, data, send) : await keyed(options.key);
|
|
100
|
+
if (!id) throw new Error(`Job "${name}" was not enqueued`);
|
|
101
|
+
if (!target.isTransaction) {
|
|
102
|
+
const worker = workers.get(name);
|
|
103
|
+
if (worker) boss.notifyWorker(worker.id);
|
|
104
|
+
}
|
|
105
|
+
return id;
|
|
106
|
+
};
|
|
107
|
+
const nextDue = async () => (await sql`select ${sql.id(jobsSchema, "next_due")}() as due`.execute(db)).rows[0].due;
|
|
108
|
+
const pending = async () => {
|
|
109
|
+
const job = sql.id(jobsSchema, "job");
|
|
110
|
+
const { rows } = await sql`
|
|
111
|
+
select count(*)::int as count from ${job} j where j.name = any(${[...definitions.keys()]}::text[]) and (
|
|
112
|
+
(j.state < 'active' and not j.blocked and j.start_after <= now()
|
|
113
|
+
and not (j.policy = 'key_strict_fifo' and exists (
|
|
114
|
+
select 1 from ${job} h where h.name = j.name and h.singleton_key = j.singleton_key and h.state = 'failed')))
|
|
115
|
+
or (j.state = 'active' and j.started_on + j.expire_seconds * interval '1s' > now()))`.execute(db);
|
|
116
|
+
return rows[0].count;
|
|
117
|
+
};
|
|
118
|
+
const drain = async (seconds) => {
|
|
119
|
+
if (!work) throw new Error("This process runs no jobs, so it cannot drain them");
|
|
120
|
+
const deadline = Date.now() + seconds * 1e3;
|
|
121
|
+
for (;;) {
|
|
122
|
+
for (const worker of workers.values()) boss.notifyWorker(worker.id);
|
|
123
|
+
const remaining = await pending();
|
|
124
|
+
if (remaining === 0 || Date.now() >= deadline) return {
|
|
125
|
+
drained: remaining === 0,
|
|
126
|
+
remaining,
|
|
127
|
+
nextDue: await nextDue()
|
|
128
|
+
};
|
|
129
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
const runtime = {
|
|
133
|
+
works: work,
|
|
134
|
+
enqueue,
|
|
135
|
+
update,
|
|
136
|
+
drain,
|
|
137
|
+
nextDue,
|
|
138
|
+
stop: async () => {
|
|
139
|
+
await boss.stop({
|
|
140
|
+
graceful: true,
|
|
141
|
+
timeout: shutdownSeconds * 1e3
|
|
142
|
+
});
|
|
143
|
+
clearCurrentJobs(runtime);
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
try {
|
|
147
|
+
await update(jobs);
|
|
148
|
+
setCurrentJobs(runtime);
|
|
149
|
+
} catch (error) {
|
|
150
|
+
await boss.stop({ graceful: false });
|
|
151
|
+
throw error;
|
|
152
|
+
}
|
|
153
|
+
return runtime;
|
|
154
|
+
};
|
|
155
|
+
//#endregion
|
|
156
|
+
export { startJobs };
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { CronExpressionParser } from "cron-parser";
|
|
3
|
+
//#region src/jobs/schedule.ts
|
|
4
|
+
/**
|
|
5
|
+
* The key every run of a recurring job waits under. Each run is a job of its own, enqueued for its time: it is what
|
|
6
|
+
* `jobs.next_due()` sees while the app sleeps, so a host that wakes the app for due jobs wakes it for these too.
|
|
7
|
+
*/
|
|
8
|
+
const scheduleKey = "protobase.schedule";
|
|
9
|
+
/** The first run of `schedule` strictly after `after`, in `timezone`; throws on an expression cron-parser refuses. */
|
|
10
|
+
const nextRun = (schedule, timezone, after) => CronExpressionParser.parse(schedule, {
|
|
11
|
+
currentDate: after,
|
|
12
|
+
tz: timezone
|
|
13
|
+
}).next().toDate();
|
|
14
|
+
/** Whether `at` is a run of `schedule`, so a run left from an earlier schedule can be told apart. */
|
|
15
|
+
const isRun = (schedule, timezone, at) => nextRun(schedule, timezone, /* @__PURE__ */ new Date(at.getTime() - 1)).getTime() === at.getTime();
|
|
16
|
+
/**
|
|
17
|
+
* The id of the run of job `name` at `at`: the same in every process, so however many workers enqueue a run, the
|
|
18
|
+
* database keeps one (its primary key), and the run happens once.
|
|
19
|
+
*/
|
|
20
|
+
const runId = (name, at) => {
|
|
21
|
+
const hex = createHash("sha256").update(`${name}@${at.toISOString()}`).digest("hex");
|
|
22
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-8${hex.slice(13, 16)}-${(parseInt(hex[16], 16) & 3 | 8).toString(16)}${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
|
|
23
|
+
};
|
|
24
|
+
//#endregion
|
|
25
|
+
export { isRun, nextRun, runId, scheduleKey };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { jobsSchema } from "./schema.js";
|
|
2
|
+
import { isRun, nextRun, runId, scheduleKey } from "./schedule.js";
|
|
3
|
+
import { jobsDatabase } from "./database.js";
|
|
4
|
+
import { sql } from "kysely";
|
|
5
|
+
//#region src/jobs/schedules.ts
|
|
6
|
+
const job = sql.id(jobsSchema, "job");
|
|
7
|
+
/** The recurring jobs' runs: enqueued ahead, one per time, and kept in step with their schedules. */
|
|
8
|
+
const createSchedules = (boss, db, now) => {
|
|
9
|
+
const timezone = (definition) => definition.options.timezone ?? "UTC";
|
|
10
|
+
const enqueueRun = (definition, at) => boss.send(definition.name, null, {
|
|
11
|
+
id: runId(definition.name, at),
|
|
12
|
+
startAfter: at,
|
|
13
|
+
singletonKey: scheduleKey,
|
|
14
|
+
db: jobsDatabase(db)
|
|
15
|
+
});
|
|
16
|
+
/**
|
|
17
|
+
* Makes the waiting runs match `definitions`: drops those of jobs without a schedule now, and those at a time the
|
|
18
|
+
* schedule no longer has, then enqueues each schedule's next run. A run that is due stays, to run as soon as it can.
|
|
19
|
+
*/
|
|
20
|
+
const sync = async (definitions) => {
|
|
21
|
+
const scheduled = definitions.filter((definition) => definition.options.schedule !== void 0);
|
|
22
|
+
await sql`delete from ${job} where singleton_key = ${scheduleKey} and state < 'active' and not (name = any(${scheduled.map(({ name }) => name)}::text[]))`.execute(db);
|
|
23
|
+
for (const definition of scheduled) {
|
|
24
|
+
const stale = (await sql`
|
|
25
|
+
select id, start_after from ${job} where name = ${definition.name} and singleton_key = ${scheduleKey} and state < 'active'`.execute(db)).rows.filter((run) => !isRun(definition.options.schedule, timezone(definition), new Date(run.start_after))).map((run) => run.id);
|
|
26
|
+
if (stale.length > 0) await sql`delete from ${job} where name = ${definition.name} and id = any(${stale}::uuid[])`.execute(db);
|
|
27
|
+
await enqueueRun(definition, nextRun(definition.options.schedule, timezone(definition), now()));
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
/** As a run starts, the one after it: from its own time, or from now when it ran late, so missed times are skipped. */
|
|
31
|
+
const following = async (definition, ranAt) => {
|
|
32
|
+
const from = new Date(Math.max(ranAt.getTime(), now().getTime()));
|
|
33
|
+
await enqueueRun(definition, nextRun(definition.options.schedule, timezone(definition), from));
|
|
34
|
+
};
|
|
35
|
+
return {
|
|
36
|
+
sync,
|
|
37
|
+
following
|
|
38
|
+
};
|
|
39
|
+
};
|
|
40
|
+
//#endregion
|
|
41
|
+
export { createSchedules };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { Db } from "@protobase/query";
|
|
2
|
+
//#region src/jobs/schema.d.ts
|
|
3
|
+
/** The schema the job queue lives in: pg-boss's tables and Protobase's functions. */
|
|
4
|
+
export declare const jobsSchema = "jobs";
|
|
5
|
+
/**
|
|
6
|
+
* The SQL that brings the jobs schema to what this version of Protobase runs: pg-boss's own construction plan when it
|
|
7
|
+
* is missing, or its migration plan from the version installed, then Protobase's functions. `undefined` when current.
|
|
8
|
+
*/
|
|
9
|
+
export declare const jobsSchemaMigration: (db: Db) => Promise<string | undefined>;
|
|
10
|
+
/**
|
|
11
|
+
* Refuses to go on while the jobs schema is missing or not the version this Protobase runs. The server never creates
|
|
12
|
+
* or migrates it: the project's migrations do, before the new version serves.
|
|
13
|
+
*/
|
|
14
|
+
export declare const checkJobsSchema: (db: Db) => Promise<void>;
|
|
15
|
+
//#endregion
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { functionNames, functionsComment, functionsSql } from "./functions-sql.js";
|
|
2
|
+
import { CompiledQuery } from "kysely";
|
|
3
|
+
import { getConstructionPlans, getMigrationPlans } from "pg-boss";
|
|
4
|
+
//#region src/jobs/schema.ts
|
|
5
|
+
/** The schema the job queue lives in: pg-boss's tables and Protobase's functions. */
|
|
6
|
+
const jobsSchema = "jobs";
|
|
7
|
+
/** The command that writes what the database lacks as the project's next migration. */
|
|
8
|
+
const migrationCommand = "protobase migration";
|
|
9
|
+
/**
|
|
10
|
+
* A pg-boss plan as a migration the project applies in its own transaction: without its BEGIN and COMMIT, without the
|
|
11
|
+
* timeouts it sets for that transaction, which would hold for the project's later migrations too, and with index
|
|
12
|
+
* builds that are not `CONCURRENTLY`, which Postgres refuses in a transaction. Migrations run while nothing else
|
|
13
|
+
* writes, so a plain index build only costs a lock nobody waits on.
|
|
14
|
+
*/
|
|
15
|
+
const migrationStatements = (plan) => {
|
|
16
|
+
const match = /^\s*BEGIN;\n\s*SET LOCAL lock_timeout = \d+;\n\s*SET LOCAL idle_in_transaction_session_timeout = \d+;\n([\s\S]*?)\n\s*COMMIT;\n?([\s\S]*)$/.exec(plan);
|
|
17
|
+
if (!match) throw new Error(`Unexpected pg-boss plan:\n${plan.slice(0, 200)}`);
|
|
18
|
+
return `${match[1].trim()}\n${match[2]}`.replace(/\b(CREATE (?:UNIQUE )?INDEX|DROP INDEX|REINDEX INDEX) CONCURRENTLY\b/gi, "$1").trim();
|
|
19
|
+
};
|
|
20
|
+
const rows = async (db, text, parameters = []) => (await db.executeQuery(CompiledQuery.raw(text, parameters))).rows;
|
|
21
|
+
/** What the database holds of the jobs schema: pg-boss's version, when installed, and whether Protobase's functions are current. */
|
|
22
|
+
const readJobsSchema = async (db) => {
|
|
23
|
+
const [{ installed }] = await rows(db, `select to_regclass('${jobsSchema}.version') is not null as installed`);
|
|
24
|
+
if (!installed) return {
|
|
25
|
+
pgBoss: void 0,
|
|
26
|
+
functionsCurrent: false
|
|
27
|
+
};
|
|
28
|
+
const [version] = await rows(db, `select version from ${jobsSchema}.version`);
|
|
29
|
+
const functions = await rows(db, `select p.proname as name, obj_description(p.oid, 'pg_proc') as comment from pg_proc p
|
|
30
|
+
join pg_namespace n on n.oid = p.pronamespace where n.nspname = $1 and p.proname = any($2)`, [jobsSchema, functionNames]);
|
|
31
|
+
const functionsCurrent = functionNames.every((name) => functions.some((found) => found.name === name && found.comment === functionsComment));
|
|
32
|
+
return {
|
|
33
|
+
pgBoss: version === void 0 ? void 0 : Number(version.version),
|
|
34
|
+
functionsCurrent
|
|
35
|
+
};
|
|
36
|
+
};
|
|
37
|
+
const newer = (version) => /* @__PURE__ */ new Error(`The jobs schema is at pg-boss schema version ${version}, newer than the 45 this version of Protobase runs; upgrade Protobase`);
|
|
38
|
+
/**
|
|
39
|
+
* The SQL that brings the jobs schema to what this version of Protobase runs: pg-boss's own construction plan when it
|
|
40
|
+
* is missing, or its migration plan from the version installed, then Protobase's functions. `undefined` when current.
|
|
41
|
+
*/
|
|
42
|
+
const jobsSchemaMigration = async (db) => {
|
|
43
|
+
const { pgBoss, functionsCurrent } = await readJobsSchema(db);
|
|
44
|
+
if (pgBoss !== void 0 && pgBoss > 45) throw newer(pgBoss);
|
|
45
|
+
if (pgBoss === 45 && functionsCurrent) return void 0;
|
|
46
|
+
const parts = [];
|
|
47
|
+
if (pgBoss === void 0) parts.push(migrationStatements(getConstructionPlans(jobsSchema)));
|
|
48
|
+
else if (pgBoss < 45) parts.push(migrationStatements(getMigrationPlans(jobsSchema, pgBoss)));
|
|
49
|
+
parts.push(functionsSql(jobsSchema));
|
|
50
|
+
return parts.join("\n\n");
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* Refuses to go on while the jobs schema is missing or not the version this Protobase runs. The server never creates
|
|
54
|
+
* or migrates it: the project's migrations do, before the new version serves.
|
|
55
|
+
*/
|
|
56
|
+
const checkJobsSchema = async (db) => {
|
|
57
|
+
const { pgBoss, functionsCurrent } = await readJobsSchema(db);
|
|
58
|
+
if (pgBoss !== void 0 && pgBoss > 45) throw newer(pgBoss);
|
|
59
|
+
if (pgBoss === 45 && functionsCurrent) return;
|
|
60
|
+
throw new Error(`The jobs schema is ${pgBoss === void 0 ? "missing" : "behind this version of Protobase"}. Write it as a migration with \`${migrationCommand}\`, apply it with the project's migrations, then start again.`);
|
|
61
|
+
};
|
|
62
|
+
//#endregion
|
|
63
|
+
export { checkJobsSchema, jobsSchema, jobsSchemaMigration, migrationCommand, migrationStatements, readJobsSchema };
|
package/dist/project-config.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AdminAuth } from "./better-auth/create-auth.js";
|
|
2
|
+
import { EventsOptions } from "./changes/events-options.js";
|
|
2
3
|
import { FilesOptions } from "./files/options.js";
|
|
3
4
|
import { Authenticator } from "./types.js";
|
|
4
5
|
import { FunctionSource } from "./functions/define-function.js";
|
|
@@ -21,13 +22,17 @@ export type ProjectConfig = {
|
|
|
21
22
|
functions?: Record<string, FunctionSource>;
|
|
22
23
|
/** Where file fields keep their bytes: providers by name, retention and the signing secret (see `FilesOptions`). */
|
|
23
24
|
files?: FilesOptions;
|
|
25
|
+
/** Where the changes that resources' events are raised from are captured (see `EventsOptions`). */
|
|
26
|
+
events?: EventsOptions;
|
|
27
|
+
/** The module of the app's jobs, e.g. `import * as jobs from './config/jobs'`: every `defineJob` it exports. */
|
|
28
|
+
jobs?: Record<string, unknown>;
|
|
24
29
|
options?: AdminOptions;
|
|
25
30
|
};
|
|
26
31
|
/** Typed identity, for `export default defineConfig({ extends: [...], config, auth, ... })` in `protobase.config.ts`. */
|
|
27
32
|
export declare const defineConfig: (config: ProjectConfig) => ProjectConfig;
|
|
28
33
|
/**
|
|
29
34
|
* One config from several, each after the ones it `extends`, the last being the app's own. Named things are all kept:
|
|
30
|
-
* resources, views and pages of the config modules, functions, assistant tools, file providers; one name defined by two configs is an error naming
|
|
35
|
+
* resources, views and pages of the config modules, functions, jobs, assistant tools, file providers; one name defined by two configs is an error naming
|
|
31
36
|
* both. Write hooks run in the same order. Every other value is the last config's that sets it, so the app's own wins.
|
|
32
37
|
*/
|
|
33
38
|
export declare const mergeConfig: (...configs: ProjectConfig[]) => ProjectConfig;
|
package/dist/project-config.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { jobDefinitions } from "./jobs/define-job.js";
|
|
1
2
|
import { isPageDefinition } from "@protobase/layout";
|
|
2
3
|
//#region src/project-config.ts
|
|
3
4
|
/** Typed identity, for `export default defineConfig({ extends: [...], config, auth, ... })` in `protobase.config.ts`. */
|
|
@@ -104,6 +105,16 @@ const mergedFiles = (named) => {
|
|
|
104
105
|
...secret !== void 0 && { secret }
|
|
105
106
|
};
|
|
106
107
|
};
|
|
108
|
+
const mergedJobs = (named) => {
|
|
109
|
+
const all = named.flatMap(({ config, label }) => jobDefinitions(config.jobs ?? {}).map((value) => ({
|
|
110
|
+
name: value.name,
|
|
111
|
+
value,
|
|
112
|
+
label
|
|
113
|
+
})));
|
|
114
|
+
if (all.length === 0) return void 0;
|
|
115
|
+
checkUnique("The job", all);
|
|
116
|
+
return Object.fromEntries(all.map(({ name, value }) => [name, value]));
|
|
117
|
+
};
|
|
107
118
|
const mergedOptions = (named) => {
|
|
108
119
|
const all = named.flatMap(({ config }) => config.options ? [config.options] : []);
|
|
109
120
|
if (all.length === 0) return void 0;
|
|
@@ -117,19 +128,21 @@ const mergedOptions = (named) => {
|
|
|
117
128
|
};
|
|
118
129
|
/**
|
|
119
130
|
* One config from several, each after the ones it `extends`, the last being the app's own. Named things are all kept:
|
|
120
|
-
* resources, views and pages of the config modules, functions, assistant tools, file providers; one name defined by two configs is an error naming
|
|
131
|
+
* resources, views and pages of the config modules, functions, jobs, assistant tools, file providers; one name defined by two configs is an error naming
|
|
121
132
|
* both. Write hooks run in the same order. Every other value is the last config's that sets it, so the app's own wins.
|
|
122
133
|
*/
|
|
123
134
|
const mergeConfig = (...configs) => {
|
|
124
135
|
const named = configs.flatMap((config, index) => flatten(config, config.name ?? `config ${index + 1}`));
|
|
125
|
-
const { db, authenticate, auth } = lastSet(named.map(({ config }) => ({
|
|
136
|
+
const { db, authenticate, auth, events } = lastSet(named.map(({ config }) => ({
|
|
126
137
|
db: config.db,
|
|
127
138
|
authenticate: config.authenticate,
|
|
128
|
-
auth: config.auth
|
|
139
|
+
auth: config.auth,
|
|
140
|
+
events: config.events
|
|
129
141
|
})));
|
|
130
142
|
const module = mergedModule(named);
|
|
131
143
|
const functions = mergedFunctions(named);
|
|
132
144
|
const files = mergedFiles(named);
|
|
145
|
+
const jobs = mergedJobs(named);
|
|
133
146
|
const options = mergedOptions(named);
|
|
134
147
|
const name = configs.at(-1)?.name;
|
|
135
148
|
return {
|
|
@@ -140,6 +153,8 @@ const mergeConfig = (...configs) => {
|
|
|
140
153
|
...auth && { auth },
|
|
141
154
|
...functions && { functions },
|
|
142
155
|
...files && { files },
|
|
156
|
+
...events && { events },
|
|
157
|
+
...jobs && { jobs },
|
|
143
158
|
...options && { options }
|
|
144
159
|
};
|
|
145
160
|
};
|
package/dist/transactions.js
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
import { sql } from "kysely";
|
|
2
2
|
//#region src/transactions.ts
|
|
3
|
-
/**
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
/**
|
|
4
|
+
* One transaction whose statements Postgres cancels after `timeoutMs` (SET LOCAL statement_timeout), with `settings`
|
|
5
|
+
* set for it as well, all in one statement.
|
|
6
|
+
*/
|
|
7
|
+
const writeTransaction = (db, timeoutMs, run, settings = {}) => db.transaction().execute(async (trx) => {
|
|
8
|
+
const all = Object.entries({
|
|
9
|
+
statement_timeout: String(timeoutMs),
|
|
10
|
+
...settings
|
|
11
|
+
});
|
|
12
|
+
await sql`select ${sql.join(all.map(([name, value]) => sql`set_config(${name}, ${value}, true)`))}`.execute(trx);
|
|
6
13
|
return run(trx);
|
|
7
14
|
});
|
|
8
15
|
/** Same, and Postgres refuses any write inside it. */
|
package/dist/write-pipeline.js
CHANGED
|
@@ -4,6 +4,7 @@ import { tenantScope } from "./tenant.js";
|
|
|
4
4
|
import { requestAccess, requireOperation, targetFilter, writableFields } from "./request-access.js";
|
|
5
5
|
import { writeTransaction } from "./transactions.js";
|
|
6
6
|
import { attachFiles } from "./files/attach.js";
|
|
7
|
+
import { actorSettings } from "./changes/actor.js";
|
|
7
8
|
import { validateBody } from "./validate-body.js";
|
|
8
9
|
import { assertWritten, checkPrecondition, decide, denied, firstKeyField, missing, notifier, shown } from "./write-shared.js";
|
|
9
10
|
import { applyUpdate, planUpdate } from "./write-update.js";
|
|
@@ -89,7 +90,7 @@ const writeIn = async (trx, deps, access, request) => {
|
|
|
89
90
|
const runWrite = async (deps, entry, session, request) => {
|
|
90
91
|
const access = await requestAccess(deps, entry, session);
|
|
91
92
|
requireOperation(access, request.operation === "undelete" ? "update" : request.operation);
|
|
92
|
-
return writeTransaction(deps.db, deps.statementTimeoutMs, (trx) => writeIn(trx, deps, access, request));
|
|
93
|
+
return writeTransaction(deps.db, deps.statementTimeoutMs, (trx) => writeIn(trx, deps, access, request), actorSettings(deps, session));
|
|
93
94
|
};
|
|
94
95
|
//#endregion
|
|
95
96
|
export { runWrite, writeIn };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@protobase/server",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"description": "The Protobase REST API as a Hono app, with Better Auth sign-in.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -30,18 +30,20 @@
|
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
32
|
"@better-auth/passkey": "^1.7.7",
|
|
33
|
-
"@protobase/layout": "0.8.
|
|
34
|
-
"@protobase/query": "0.8.
|
|
35
|
-
"@protobase/schema": "0.8.
|
|
33
|
+
"@protobase/layout": "0.8.2",
|
|
34
|
+
"@protobase/query": "0.8.2",
|
|
35
|
+
"@protobase/schema": "0.8.2",
|
|
36
36
|
"@scalar/hono-api-reference": "^0.12.10",
|
|
37
37
|
"@simplewebauthn/server": "^13.3.1",
|
|
38
38
|
"better-auth": "^1.7.7",
|
|
39
|
+
"cron-parser": "^5.10.1",
|
|
39
40
|
"file-type": "^22.1.1",
|
|
40
41
|
"hono": "^4.13.13",
|
|
41
42
|
"image-size": "^2.0.4",
|
|
42
43
|
"jose": "^6.2.12",
|
|
43
44
|
"kysely": "^0.29.6",
|
|
44
45
|
"nodemailer": "^10.0.16",
|
|
46
|
+
"pg-boss": "12.37.1",
|
|
45
47
|
"zod": "^4.6.5"
|
|
46
48
|
}
|
|
47
49
|
}
|