@podosoft/podokit 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/deploy-checks.d.ts +16 -5
- package/dist/deploy-checks.js +17 -9
- package/dist/deploy-render.js +28 -3
- package/dist/deploy-schema.d.ts +7 -3
- package/dist/deploy-schema.js +16 -5
- package/dist/deploy.d.ts +9 -0
- package/dist/deploy.js +83 -17
- package/dist/index.js +9 -2
- package/dist/templates/fullstack/AGENTS.md +3 -1
- package/dist/templates/fullstack/apps/api/package.json +1 -1
- package/dist/templates/fullstack/apps/api/scripts/verify-api-contract.ts +2 -1
- package/dist/templates/fullstack/apps/api/src/build.spec.ts +5 -2
- package/dist/templates/fullstack/apps/api/src/core/api-contract.spec.ts +43 -4
- package/dist/templates/fullstack/apps/api/src/core/api-contract.ts +39 -4
- package/dist/templates/fullstack/dot-agents/skills/podokit-deploy/SKILL.md +1 -1
- package/dist/templates/fullstack/dot-agents/skills/podokit-deploy/references/profile.md +5 -2
- package/dist/templates/fullstack/tests/README.md +20 -10
- package/dist/templates/fullstack/tests/helpers/paths.ts +13 -2
- package/dist/templates/fullstack/tests/helpers/seeded-admin.ts +15 -0
- package/dist/templates/fullstack/tests/package.json +3 -3
- package/dist/templates/fullstack/tests/playwright.config.ts +8 -6
- package/dist/templates/fullstack/tests/playwright.extensions.ts +73 -12
- package/dist/templates/fullstack/tests/run-e2e.ts +28 -0
- package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/admin/nav.test.ts +17 -3
- package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/components/theme-toggle.svelte +1 -1
- package/dist/templates/modules/admin-dashboard/files/tests/api/audit.api.spec.ts +96 -7
- package/dist/templates/modules/admin-dashboard/files/tests/api/auth.api.spec.ts +11 -0
- package/dist/templates/modules/admin-dashboard/files/tests/helpers/hydration.ts +13 -4
- package/dist/templates/modules/admin-dashboard/files/tests/playwright.config.ts +8 -6
- package/dist/templates/modules/admin-dashboard/files/tests/ui/theme-mode.ui.spec.ts +16 -10
- package/dist/templates/modules/audit-log/files/apps/api/src/audit/README.md +14 -2
- package/dist/templates/modules/audit-log/files/apps/api/src/audit/audit-hook.spec.ts +385 -0
- package/dist/templates/modules/audit-log/files/apps/api/src/audit/audit-hook.ts +132 -43
- package/dist/templates/modules/audit-log/module.manifest.json +2 -2
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth.ts +3 -1
- package/dist/templates/modules/auth/files/apps/api/src/auth/db.ts +13 -1
- package/dist/templates/modules/auth/files/apps/api/src/auth/empty-lists.spec.ts +67 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/empty-lists.ts +54 -0
- package/dist/templates/modules/auth/module.manifest.json +1 -0
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.spec.ts +50 -0
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.ts +23 -3
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.spec.ts +87 -0
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.ts +60 -3
- package/dist/templates/modules/object-storage-local/files/tests/api/storage-local.api.spec.ts +3 -6
- package/dist/templates/modules/object-storage-s3/files/tests/api/storage.api.spec.ts +3 -6
- package/dist/templates/modules/rate-limit/files/tests/api/rate-limit.api.spec.ts +163 -62
- package/dist/templates/modules/sse/files/tests/api/events.api.spec.ts +3 -6
- package/dist/update.d.ts +25 -5
- package/dist/update.js +86 -10
- package/package.json +2 -2
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { Database as SqliteDatabase } from "bun:sqlite";
|
|
2
2
|
import type { SQL } from "bun";
|
|
3
|
+
import { Kysely, PostgresDialect } from "kysely";
|
|
3
4
|
import type { AuthConfigRow, ConfigVersions } from "@podosoft/podokit-auth";
|
|
4
5
|
import type { DatabaseProviderName } from "@podosoft/podokit-runtime";
|
|
5
6
|
import { validateEnv } from "../config/env.validation";
|
|
6
7
|
import { PROVIDERS } from "../config/providers";
|
|
7
8
|
import { databaseUrl, sqliteDatabasePath } from "../database/database";
|
|
8
9
|
import { answersWithin, sharedProbe, type DatabaseProbe } from "./database-outage";
|
|
10
|
+
import { EmptyListPlugin } from "./empty-lists";
|
|
9
11
|
import { isMaintenanceCommand } from "./maintenance";
|
|
10
12
|
import { AuthPostgresPool } from "./postgres-pool";
|
|
11
13
|
|
|
@@ -30,7 +32,17 @@ export const sqliteDatabase = provider === "sqlite"
|
|
|
30
32
|
? new SqliteDatabase(sqliteDatabasePath(url), { create: true, strict: true })
|
|
31
33
|
: null;
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
/** The Kysely instance Better Auth queries PostgreSQL through, or null on SQLite. */
|
|
36
|
+
export const postgresAuthQueries = postgresPool
|
|
37
|
+
? new Kysely<object>({
|
|
38
|
+
dialect: new PostgresDialect({ pool: postgresPool }),
|
|
39
|
+
plugins: [new EmptyListPlugin()],
|
|
40
|
+
})
|
|
41
|
+
: null;
|
|
42
|
+
|
|
43
|
+
export const authDatabase = postgresAuthQueries
|
|
44
|
+
? { db: postgresAuthQueries, type: "postgres" as const, transaction: true }
|
|
45
|
+
: sqliteDatabase;
|
|
34
46
|
if (!authDatabase) throw new Error("No authentication database is configured");
|
|
35
47
|
|
|
36
48
|
/** Whether the authentication database answers `SELECT 1` within the readiness check timeout. */
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { DummyDriver, Kysely, PostgresAdapter, PostgresIntrospector, PostgresQueryCompiler } from "kysely";
|
|
3
|
+
import type { DatabaseProviderName } from "@podosoft/podokit-runtime";
|
|
4
|
+
import { PROVIDERS } from "../config/providers";
|
|
5
|
+
import { postgresAuthQueries } from "./db";
|
|
6
|
+
import { EmptyListPlugin } from "./empty-lists";
|
|
7
|
+
|
|
8
|
+
function databaseProvider(): DatabaseProviderName {
|
|
9
|
+
return PROVIDERS.database;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// A type alias, not an interface: withTables() needs an index signature.
|
|
13
|
+
type Tables = {
|
|
14
|
+
user: { id: string; email: string };
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const db = new Kysely<Tables>({
|
|
18
|
+
dialect: {
|
|
19
|
+
createAdapter: () => new PostgresAdapter(),
|
|
20
|
+
createDriver: () => new DummyDriver(),
|
|
21
|
+
createIntrospector: (kysely) => new PostgresIntrospector(kysely),
|
|
22
|
+
createQueryCompiler: () => new PostgresQueryCompiler(),
|
|
23
|
+
},
|
|
24
|
+
plugins: [new EmptyListPlugin()],
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
describe("empty lists in authentication queries", () => {
|
|
28
|
+
it("sends an empty in list as a condition that never holds", () => {
|
|
29
|
+
const query = db.selectFrom("user").selectAll().where("id", "in", []).limit(0).compile();
|
|
30
|
+
expect(query.sql).toBe('select * from "user" where 1 = 0 limit $1');
|
|
31
|
+
expect(query.parameters).toEqual([0]);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("sends an empty not in list as a condition that always holds", () => {
|
|
35
|
+
const query = db
|
|
36
|
+
.selectFrom("user")
|
|
37
|
+
.select("id")
|
|
38
|
+
.where("email", "=", "a@example.com")
|
|
39
|
+
.where("id", "not in", [])
|
|
40
|
+
.compile();
|
|
41
|
+
expect(query.sql).toBe('select "id" from "user" where "email" = $1 and 1 = 1');
|
|
42
|
+
expect(query.parameters).toEqual(["a@example.com"]);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it("leaves lists with values and other operators as they are", () => {
|
|
46
|
+
const query = db
|
|
47
|
+
.deleteFrom("user")
|
|
48
|
+
.where((eb) => eb.or([eb("id", "in", ["a", "b"]), eb("email", "=", "c@example.com")]))
|
|
49
|
+
.compile();
|
|
50
|
+
expect(query.sql).toBe('delete from "user" where ("id" in ($1, $2) or "email" = $3)');
|
|
51
|
+
expect(query.parameters).toEqual(["a", "b", "c@example.com"]);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("applies to the queries Better Auth sends to PostgreSQL", () => {
|
|
55
|
+
if (databaseProvider() !== "postgres") {
|
|
56
|
+
expect(postgresAuthQueries).toBeNull();
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const query = postgresAuthQueries
|
|
60
|
+
?.withTables<Tables>()
|
|
61
|
+
.selectFrom("user")
|
|
62
|
+
.selectAll()
|
|
63
|
+
.where("id", "in", [])
|
|
64
|
+
.compile();
|
|
65
|
+
expect(query?.sql).toBe('select * from "user" where 1 = 0');
|
|
66
|
+
});
|
|
67
|
+
});
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BinaryOperationNode,
|
|
3
|
+
OperationNodeTransformer,
|
|
4
|
+
OperatorNode,
|
|
5
|
+
PrimitiveValueListNode,
|
|
6
|
+
ValueListNode,
|
|
7
|
+
ValueNode,
|
|
8
|
+
type KyselyPlugin,
|
|
9
|
+
type PluginTransformQueryArgs,
|
|
10
|
+
type PluginTransformResultArgs,
|
|
11
|
+
type QueryId,
|
|
12
|
+
type QueryResult,
|
|
13
|
+
type RootOperationNode,
|
|
14
|
+
type UnknownRow,
|
|
15
|
+
} from "kysely";
|
|
16
|
+
|
|
17
|
+
function constantComparison(holds: boolean): BinaryOperationNode {
|
|
18
|
+
return BinaryOperationNode.create(
|
|
19
|
+
ValueNode.createImmediate(1),
|
|
20
|
+
OperatorNode.create("="),
|
|
21
|
+
ValueNode.createImmediate(holds ? 1 : 0),
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
class EmptyListTransformer extends OperationNodeTransformer {
|
|
26
|
+
protected override transformBinaryOperation(node: BinaryOperationNode, queryId?: QueryId): BinaryOperationNode {
|
|
27
|
+
const list = node.rightOperand;
|
|
28
|
+
const empty = (PrimitiveValueListNode.is(list) || ValueListNode.is(list)) && list.values.length === 0;
|
|
29
|
+
if (empty && OperatorNode.is(node.operator)) {
|
|
30
|
+
if (node.operator.operator === "in") return constantComparison(false);
|
|
31
|
+
if (node.operator.operator === "not in") return constantComparison(true);
|
|
32
|
+
}
|
|
33
|
+
return super.transformBinaryOperation(node, queryId);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Better Auth's Kysely adapter sends an `in` filter on an empty array as
|
|
39
|
+
* `IN ()`, which PostgreSQL rejects as a syntax error. The organization
|
|
40
|
+
* plugin's list-members endpoint looks the members' users up that way, so a
|
|
41
|
+
* filter or page that matched no member answered 500. This plugin sends an
|
|
42
|
+
* empty `in` list as `1 = 0` and an empty `not in` list as `1 = 1`.
|
|
43
|
+
*/
|
|
44
|
+
export class EmptyListPlugin implements KyselyPlugin {
|
|
45
|
+
private readonly transformer = new EmptyListTransformer();
|
|
46
|
+
|
|
47
|
+
transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
|
|
48
|
+
return this.transformer.transformNode(args.node, args.queryId);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
transformResult(args: PluginTransformResultArgs): Promise<QueryResult<UnknownRow>> {
|
|
52
|
+
return Promise.resolve(args.result);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -136,6 +136,9 @@ describe("BullMqJobQueue schedules", () => {
|
|
|
136
136
|
await expect(queue.schedule("bad", "report", {}, { cron: "61 * * * *" })).rejects.toThrow();
|
|
137
137
|
await expect(queue.schedule("bad", "report", {}, { cron: "0 3 * * *", timezone: "Mars/Olympus" })).rejects.toThrow();
|
|
138
138
|
await expect(queue.schedule("bad", "report", {}, { everyMs: 10 })).rejects.toThrow("at least 1000");
|
|
139
|
+
await expect(queue.schedule("bad", "report", {}, { everyMs: 1_000 }, { keepFailed: 1.5 })).rejects.toThrow(
|
|
140
|
+
"keepFailed must be a whole number of at least 0",
|
|
141
|
+
);
|
|
139
142
|
} finally {
|
|
140
143
|
await queue.close();
|
|
141
144
|
}
|
|
@@ -183,4 +186,51 @@ describe.skipIf(!process.env.REDIS_URL && !process.env.REDIS_HOST)("BullMqJobQue
|
|
|
183
186
|
await inspect.close();
|
|
184
187
|
}
|
|
185
188
|
});
|
|
189
|
+
|
|
190
|
+
it("limits the finished jobs a schedule's runs leave, 100 completed and 1000 failed by default", async () => {
|
|
191
|
+
const name = `schedule-test-${crypto.randomUUID()}`;
|
|
192
|
+
const queue = new BullMqJobQueue(producerConnection(REDIS));
|
|
193
|
+
const inspect = new Queue(name, { connection: redisConnection(REDIS) });
|
|
194
|
+
try {
|
|
195
|
+
await queue.connect();
|
|
196
|
+
await queue.schedule("default", name, {}, { cron: "0 3 * * *" });
|
|
197
|
+
await queue.schedule("custom", name, {}, { everyMs: 3_600_000 }, { keepCompleted: 0, keepFailed: 20 });
|
|
198
|
+
expect((await inspect.getJobScheduler("default"))?.template?.opts).toMatchObject({
|
|
199
|
+
removeOnComplete: { count: 100 },
|
|
200
|
+
removeOnFail: { count: 1_000 },
|
|
201
|
+
});
|
|
202
|
+
expect((await inspect.getJobScheduler("custom"))?.template?.opts).toMatchObject({
|
|
203
|
+
removeOnComplete: { count: 0 },
|
|
204
|
+
removeOnFail: { count: 20 },
|
|
205
|
+
});
|
|
206
|
+
} finally {
|
|
207
|
+
await queue.close();
|
|
208
|
+
await inspect.obliterate({ force: true });
|
|
209
|
+
await inspect.close();
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
it("removes finished runs beyond the schedule's limits", async () => {
|
|
214
|
+
const name = `schedule-test-${crypto.randomUUID()}`;
|
|
215
|
+
const queue = new BullMqJobQueue(producerConnection(REDIS));
|
|
216
|
+
const inspect = new Queue(name, { connection: redisConnection(REDIS) });
|
|
217
|
+
let runs = 0;
|
|
218
|
+
try {
|
|
219
|
+
await queue.connect();
|
|
220
|
+
await queue.process(name, () => {
|
|
221
|
+
runs += 1;
|
|
222
|
+
if (runs % 2 === 0) throw new Error("expected failure");
|
|
223
|
+
});
|
|
224
|
+
await queue.schedule("tick", name, {}, { everyMs: 1_000 }, { keepCompleted: 1, keepFailed: 1 });
|
|
225
|
+
for (let waited = 0; runs < 4 && waited < 10_000; waited += 100) await Bun.sleep(100);
|
|
226
|
+
expect(runs).toBeGreaterThanOrEqual(4);
|
|
227
|
+
// The last run's job moves to its finished set just after the handler returns.
|
|
228
|
+
await Bun.sleep(200);
|
|
229
|
+
expect(await inspect.getJobCounts("completed", "failed")).toEqual({ completed: 1, failed: 1 });
|
|
230
|
+
} finally {
|
|
231
|
+
await queue.close();
|
|
232
|
+
await inspect.obliterate({ force: true });
|
|
233
|
+
await inspect.close();
|
|
234
|
+
}
|
|
235
|
+
}, 15_000);
|
|
186
236
|
});
|
|
@@ -10,6 +10,9 @@ import {
|
|
|
10
10
|
type JobSnapshot,
|
|
11
11
|
type JobStatus,
|
|
12
12
|
type RepeatOptions,
|
|
13
|
+
type ScheduleOptions,
|
|
14
|
+
type ScheduleRetention,
|
|
15
|
+
scheduleRetention,
|
|
13
16
|
type Unsubscribe,
|
|
14
17
|
} from "@podosoft/podokit-runtime";
|
|
15
18
|
import type { SupervisedConnection } from "../core/services";
|
|
@@ -56,6 +59,7 @@ interface Schedule {
|
|
|
56
59
|
name: string;
|
|
57
60
|
payload: unknown;
|
|
58
61
|
repeat: RepeatOptions;
|
|
62
|
+
retention: ScheduleRetention;
|
|
59
63
|
}
|
|
60
64
|
|
|
61
65
|
/** Rejects a schedule BullMQ could not run, before it is kept. */
|
|
@@ -141,13 +145,20 @@ export class BullMqJobQueue implements JobQueue, SupervisedConnection {
|
|
|
141
145
|
};
|
|
142
146
|
}
|
|
143
147
|
|
|
144
|
-
async schedule(
|
|
148
|
+
async schedule(
|
|
149
|
+
id: string,
|
|
150
|
+
name: string,
|
|
151
|
+
payload: unknown,
|
|
152
|
+
repeat: RepeatOptions,
|
|
153
|
+
options?: ScheduleOptions,
|
|
154
|
+
): Promise<void> {
|
|
145
155
|
assertSchedule(repeat);
|
|
156
|
+
const retention = scheduleRetention(options);
|
|
146
157
|
const previous = this.schedules.get(id);
|
|
147
158
|
// A scheduler lives in its queue, so a schedule that moves leaves the old one.
|
|
148
159
|
if (previous && previous.name !== name) this.removals.set(id, previous.name);
|
|
149
160
|
if (this.removals.get(id) === name) this.removals.delete(id);
|
|
150
|
-
this.schedules.set(id, { name, payload, repeat });
|
|
161
|
+
this.schedules.set(id, { name, payload, repeat, retention });
|
|
151
162
|
await this.whenConnected(async (client) => {
|
|
152
163
|
if (this.removals.has(id)) await this.removeSchedule(client, id);
|
|
153
164
|
await this.applySchedule(client, id);
|
|
@@ -196,10 +207,19 @@ export class BullMqJobQueue implements JobQueue, SupervisedConnection {
|
|
|
196
207
|
const schedule = this.schedules.get(id);
|
|
197
208
|
if (!schedule) return;
|
|
198
209
|
const { cron, everyMs, timezone } = schedule.repeat;
|
|
210
|
+
// A count above 0 trims the queue's whole completed or failed set to that
|
|
211
|
+
// many jobs; 0 removes only the finished run.
|
|
199
212
|
await this.queue(schedule.name, client).upsertJobScheduler(
|
|
200
213
|
id,
|
|
201
214
|
cron !== undefined ? { pattern: cron, tz: timezone ?? "UTC" } : { every: everyMs },
|
|
202
|
-
{
|
|
215
|
+
{
|
|
216
|
+
name: schedule.name,
|
|
217
|
+
data: schedule.payload,
|
|
218
|
+
opts: {
|
|
219
|
+
removeOnComplete: { count: schedule.retention.keepCompleted },
|
|
220
|
+
removeOnFail: { count: schedule.retention.keepFailed },
|
|
221
|
+
},
|
|
222
|
+
},
|
|
203
223
|
);
|
|
204
224
|
}
|
|
205
225
|
|
package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.spec.ts
CHANGED
|
@@ -149,6 +149,93 @@ describe("local job schedules", () => {
|
|
|
149
149
|
await expect(queue.schedule("bad", "report", {}, { cron: "61 * * * *" })).rejects.toThrow();
|
|
150
150
|
await expect(queue.schedule("bad", "report", {}, { cron: "0 3 * * *", timezone: "Mars/Olympus" })).rejects.toThrow();
|
|
151
151
|
await expect(queue.schedule("bad", "report", {}, { everyMs: 10 })).rejects.toThrow("at least 1000");
|
|
152
|
+
await expect(queue.schedule("bad", "report", {}, { everyMs: 1_000 }, { keepCompleted: -1 })).rejects.toThrow(
|
|
153
|
+
"keepCompleted must be a whole number of at least 0",
|
|
154
|
+
);
|
|
155
|
+
queue.close();
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
/** Moves the clock to each time and runs the job the schedule added then. */
|
|
160
|
+
async function runScheduled(queue: LocalJobQueue, time: ReturnType<typeof manualTime>, times: number[]): Promise<void> {
|
|
161
|
+
for (const at of times) {
|
|
162
|
+
time.advance(at);
|
|
163
|
+
// The schedule adds its job without waiting; let the insert finish.
|
|
164
|
+
await Bun.sleep(1);
|
|
165
|
+
expect(await queue.runOnce()).toBe(true);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async function finishedJobs(database: SQL): Promise<string[]> {
|
|
170
|
+
const rows = await database<Array<{ id: string; status: string }>>`
|
|
171
|
+
SELECT "id", "status" FROM "podokit_job" ORDER BY "updatedAt", "id"
|
|
172
|
+
`;
|
|
173
|
+
return rows.map((row) => `${row.id} ${row.status}`);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
describe("local job schedule cleanup", () => {
|
|
177
|
+
it("keeps the newest finished jobs of a scheduled run's name up to the schedule's limits", async () => {
|
|
178
|
+
sql = new SQL(":memory:");
|
|
179
|
+
const time = manualTime(0);
|
|
180
|
+
const queue = new LocalJobQueue(sql, { pollIntervalMs: 60_000, now: time.now, timer: time.timer });
|
|
181
|
+
await queue.process("report", (_payload, job) => {
|
|
182
|
+
if (job.id.endsWith(":3000") || job.id.endsWith(":5000")) throw new Error("expected failure");
|
|
183
|
+
});
|
|
184
|
+
await queue.process("other", () => undefined);
|
|
185
|
+
const other = await queue.enqueue("other", {});
|
|
186
|
+
const adHoc = await queue.enqueue("report", {});
|
|
187
|
+
while (await queue.runOnce());
|
|
188
|
+
await queue.schedule("tick", "report", {}, { everyMs: 1_000 }, { keepCompleted: 2, keepFailed: 1 });
|
|
189
|
+
await queue.start();
|
|
190
|
+
await Bun.sleep(5);
|
|
191
|
+
|
|
192
|
+
await runScheduled(queue, time, [1_000, 2_000, 3_000, 4_000, 5_000, 6_000]);
|
|
193
|
+
|
|
194
|
+
// As BullMQ trims a queue: the job enqueue() added goes too, another name's stays.
|
|
195
|
+
expect(await finishedJobs(sql)).toEqual([
|
|
196
|
+
`${other.id} completed`,
|
|
197
|
+
"report:schedule:tick:4000 completed",
|
|
198
|
+
"report:schedule:tick:5000 failed",
|
|
199
|
+
"report:schedule:tick:6000 completed",
|
|
200
|
+
]);
|
|
201
|
+
expect(await queue.get(adHoc.id)).toBeNull();
|
|
202
|
+
queue.close();
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
it("removes only the run itself when a limit is 0", async () => {
|
|
206
|
+
sql = new SQL(":memory:");
|
|
207
|
+
const time = manualTime(0);
|
|
208
|
+
const queue = new LocalJobQueue(sql, { pollIntervalMs: 60_000, now: time.now, timer: time.timer });
|
|
209
|
+
await queue.process("report", (_payload, job) => {
|
|
210
|
+
if (job.id.endsWith(":2000")) throw new Error("expected failure");
|
|
211
|
+
});
|
|
212
|
+
const adHoc = await queue.enqueue("report", {});
|
|
213
|
+
while (await queue.runOnce());
|
|
214
|
+
await queue.schedule("tick", "report", {}, { everyMs: 1_000 }, { keepCompleted: 0, keepFailed: 0 });
|
|
215
|
+
await queue.start();
|
|
216
|
+
await Bun.sleep(5);
|
|
217
|
+
|
|
218
|
+
await runScheduled(queue, time, [1_000, 2_000, 3_000]);
|
|
219
|
+
|
|
220
|
+
expect(await finishedJobs(sql)).toEqual([`${adHoc.id} completed`]);
|
|
221
|
+
queue.close();
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
it("keeps 100 completed runs of a schedule by default", async () => {
|
|
225
|
+
sql = new SQL(":memory:");
|
|
226
|
+
const time = manualTime(0);
|
|
227
|
+
const queue = new LocalJobQueue(sql, { pollIntervalMs: 60_000, now: time.now, timer: time.timer });
|
|
228
|
+
await queue.process("report", () => undefined);
|
|
229
|
+
await queue.schedule("tick", "report", {}, { everyMs: 1_000 });
|
|
230
|
+
await queue.start();
|
|
231
|
+
await Bun.sleep(5);
|
|
232
|
+
|
|
233
|
+
await runScheduled(queue, time, Array.from({ length: 105 }, (_, index) => (index + 1) * 1_000));
|
|
234
|
+
|
|
235
|
+
const kept = await finishedJobs(sql);
|
|
236
|
+
expect(kept).toHaveLength(100);
|
|
237
|
+
expect(kept[0]).toBe("report:schedule:tick:6000 completed");
|
|
238
|
+
expect(kept.at(-1)).toBe("report:schedule:tick:105000 completed");
|
|
152
239
|
queue.close();
|
|
153
240
|
});
|
|
154
241
|
});
|
|
@@ -9,6 +9,9 @@ import {
|
|
|
9
9
|
type JobSnapshot,
|
|
10
10
|
type JobStatus,
|
|
11
11
|
type RepeatOptions,
|
|
12
|
+
type ScheduleOptions,
|
|
13
|
+
type ScheduleRetention,
|
|
14
|
+
scheduleRetention,
|
|
12
15
|
type Unsubscribe,
|
|
13
16
|
} from "@podosoft/podokit-runtime";
|
|
14
17
|
|
|
@@ -42,9 +45,15 @@ interface Schedule {
|
|
|
42
45
|
name: string;
|
|
43
46
|
payload: unknown;
|
|
44
47
|
repeat: RepeatOptions;
|
|
48
|
+
retention: ScheduleRetention;
|
|
45
49
|
cancel?: () => void;
|
|
46
50
|
}
|
|
47
51
|
|
|
52
|
+
type FinishedStatus = "completed" | "failed";
|
|
53
|
+
|
|
54
|
+
// A scheduled job's deduplication key: this prefix, the schedule id, ":", and the time.
|
|
55
|
+
const SCHEDULE_KEY = "schedule:";
|
|
56
|
+
|
|
48
57
|
// setTimeout fires at once for a delay above 2^31 - 1 ms, so longer waits are split.
|
|
49
58
|
const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;
|
|
50
59
|
|
|
@@ -160,13 +169,20 @@ export class LocalJobQueue implements JobQueue {
|
|
|
160
169
|
});
|
|
161
170
|
}
|
|
162
171
|
|
|
163
|
-
async schedule(
|
|
172
|
+
async schedule(
|
|
173
|
+
id: string,
|
|
174
|
+
name: string,
|
|
175
|
+
payload: unknown,
|
|
176
|
+
repeat: RepeatOptions,
|
|
177
|
+
options?: ScheduleOptions,
|
|
178
|
+
): Promise<void> {
|
|
164
179
|
assertRepeatOptions(repeat);
|
|
180
|
+
const retention = scheduleRetention(options);
|
|
165
181
|
// Rejects an invalid cron expression or time zone now rather than at the first run.
|
|
166
182
|
nextScheduledTime(repeat, this.now());
|
|
167
183
|
if (JSON.stringify(payload) === undefined) throw new Error("Job payload must be JSON serializable");
|
|
168
184
|
this.schedules.get(id)?.cancel?.();
|
|
169
|
-
const schedule: Schedule = { id, name, payload, repeat };
|
|
185
|
+
const schedule: Schedule = { id, name, payload, repeat, retention };
|
|
170
186
|
this.schedules.set(id, schedule);
|
|
171
187
|
if (this.started) this.arm(schedule);
|
|
172
188
|
}
|
|
@@ -205,6 +221,7 @@ export class LocalJobQueue implements JobQueue {
|
|
|
205
221
|
SET "status" = 'completed', "updatedAt" = ${completedAt}, "error" = NULL
|
|
206
222
|
WHERE "id" = ${row.id}
|
|
207
223
|
`;
|
|
224
|
+
await this.removeFinished(row, "completed");
|
|
208
225
|
} catch (error) {
|
|
209
226
|
await this.fail(row, error);
|
|
210
227
|
}
|
|
@@ -227,7 +244,7 @@ export class LocalJobQueue implements JobQueue {
|
|
|
227
244
|
const at = nextScheduledTime(schedule.repeat, after);
|
|
228
245
|
schedule.cancel = this.waitUntil(at, () => {
|
|
229
246
|
if (!this.started || this.schedules.get(schedule.id) !== schedule) return;
|
|
230
|
-
void this.enqueue(schedule.name, schedule.payload, { deduplicationKey:
|
|
247
|
+
void this.enqueue(schedule.name, schedule.payload, { deduplicationKey: `${SCHEDULE_KEY}${schedule.id}:${at}` })
|
|
231
248
|
.catch((error: unknown) => {
|
|
232
249
|
const message = error instanceof Error ? error.message : String(error);
|
|
233
250
|
process.stderr.write(`Add scheduled job failed: ${schedule.id} ${message}\n`);
|
|
@@ -269,6 +286,45 @@ export class LocalJobQueue implements JobQueue {
|
|
|
269
286
|
CREATE INDEX IF NOT EXISTS "podokit_job_pending"
|
|
270
287
|
ON "podokit_job" ("status", "availableAt", "createdAt")
|
|
271
288
|
`;
|
|
289
|
+
await this.sql`
|
|
290
|
+
CREATE INDEX IF NOT EXISTS "podokit_job_finished"
|
|
291
|
+
ON "podokit_job" ("name", "status", "updatedAt")
|
|
292
|
+
`;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** The schedule whose job id `row` has, while that schedule is kept. */
|
|
296
|
+
private scheduleOf(row: JobRow): Schedule | undefined {
|
|
297
|
+
for (const schedule of this.schedules.values()) {
|
|
298
|
+
const prefix = `${schedule.name}:${SCHEDULE_KEY}${schedule.id}:`;
|
|
299
|
+
if (schedule.name === row.name && row.id.startsWith(prefix) && /^\d+$/.test(row.id.slice(prefix.length))) {
|
|
300
|
+
return schedule;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return undefined;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* After a scheduled run finishes, keeps the newest finished jobs of its name
|
|
308
|
+
* up to the schedule's limit, as BullMQ trims a queue, or removes only the
|
|
309
|
+
* run when the limit is 0.
|
|
310
|
+
*/
|
|
311
|
+
private async removeFinished(row: JobRow, status: FinishedStatus): Promise<void> {
|
|
312
|
+
const schedule = this.scheduleOf(row);
|
|
313
|
+
if (!schedule) return;
|
|
314
|
+
const keep = status === "completed" ? schedule.retention.keepCompleted : schedule.retention.keepFailed;
|
|
315
|
+
if (keep === 0) {
|
|
316
|
+
await this.sql`DELETE FROM "podokit_job" WHERE "id" = ${row.id}`;
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
await this.sql`
|
|
320
|
+
DELETE FROM "podokit_job"
|
|
321
|
+
WHERE "name" = ${row.name} AND "status" = ${status} AND "id" NOT IN (
|
|
322
|
+
SELECT "id" FROM "podokit_job"
|
|
323
|
+
WHERE "name" = ${row.name} AND "status" = ${status}
|
|
324
|
+
ORDER BY "updatedAt" DESC, "id" DESC
|
|
325
|
+
LIMIT ${keep}
|
|
326
|
+
)
|
|
327
|
+
`;
|
|
272
328
|
}
|
|
273
329
|
|
|
274
330
|
private poll(delay: number): void {
|
|
@@ -296,6 +352,7 @@ export class LocalJobQueue implements JobQueue {
|
|
|
296
352
|
"error" = ${message.slice(0, 2_000)}
|
|
297
353
|
WHERE "id" = ${row.id}
|
|
298
354
|
`;
|
|
355
|
+
if (!retry) await this.removeFinished(row, "failed");
|
|
299
356
|
}
|
|
300
357
|
|
|
301
358
|
private snapshot(row: JobRow): JobSnapshot {
|
package/dist/templates/modules/object-storage-local/files/tests/api/storage-local.api.spec.ts
CHANGED
|
@@ -1,13 +1,8 @@
|
|
|
1
1
|
import { expect, test, type APIRequestContext, type PlaywrightWorkerArgs } from "@playwright/test";
|
|
2
|
+
import { seededAdmin } from "../helpers/seeded-admin";
|
|
2
3
|
|
|
3
4
|
const base = process.env.E2E_BASE_URL ?? "http://localhost:5001";
|
|
4
5
|
const origin = { origin: base };
|
|
5
|
-
// The administrator seeded by the admin-dashboard Playwright setup. Inlined so this
|
|
6
|
-
// spec also compiles in apps that do not install that module.
|
|
7
|
-
const admin = {
|
|
8
|
-
email: process.env.E2E_ADMIN_EMAIL || "admin@example.com",
|
|
9
|
-
password: process.env.E2E_ADMIN_PASSWORD || "Podokit3e-Str0ng!pw",
|
|
10
|
-
};
|
|
11
6
|
|
|
12
7
|
type Playwright = PlaywrightWorkerArgs["playwright"];
|
|
13
8
|
|
|
@@ -36,6 +31,8 @@ async function signUp(ctx: APIRequestContext): Promise<boolean> {
|
|
|
36
31
|
|
|
37
32
|
/** Resolves false when no administrator session is available. */
|
|
38
33
|
async function signInAdmin(ctx: APIRequestContext): Promise<boolean> {
|
|
34
|
+
const admin = await seededAdmin();
|
|
35
|
+
if (!admin) return false;
|
|
39
36
|
const response = await ctx.post("/api/auth/sign-in/email", { data: admin });
|
|
40
37
|
return response.ok() && ((await sessionRoles(ctx)) ?? []).includes("admin");
|
|
41
38
|
}
|
|
@@ -1,13 +1,8 @@
|
|
|
1
1
|
import { expect, test, type APIRequestContext, type PlaywrightWorkerArgs } from "@playwright/test";
|
|
2
|
+
import { seededAdmin } from "../helpers/seeded-admin";
|
|
2
3
|
|
|
3
4
|
const base = process.env.E2E_BASE_URL ?? "http://localhost:5001";
|
|
4
5
|
const origin = { origin: base };
|
|
5
|
-
// The administrator seeded by the admin-dashboard Playwright setup. Inlined so this
|
|
6
|
-
// spec also compiles in apps that do not install that module.
|
|
7
|
-
const admin = {
|
|
8
|
-
email: process.env.E2E_ADMIN_EMAIL || "admin@example.com",
|
|
9
|
-
password: process.env.E2E_ADMIN_PASSWORD || "Podokit3e-Str0ng!pw",
|
|
10
|
-
};
|
|
11
6
|
|
|
12
7
|
type Playwright = PlaywrightWorkerArgs["playwright"];
|
|
13
8
|
|
|
@@ -36,6 +31,8 @@ async function signUp(ctx: APIRequestContext): Promise<boolean> {
|
|
|
36
31
|
|
|
37
32
|
/** Resolves false when no administrator session is available. */
|
|
38
33
|
async function signInAdmin(ctx: APIRequestContext): Promise<boolean> {
|
|
34
|
+
const admin = await seededAdmin();
|
|
35
|
+
if (!admin) return false;
|
|
39
36
|
const response = await ctx.post("/api/auth/sign-in/email", { data: admin });
|
|
40
37
|
return response.ok() && ((await sessionRoles(ctx)) ?? []).includes("admin");
|
|
41
38
|
}
|