@podosoft/podokit 1.0.2 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -5
- package/dist/add.d.ts +14 -1
- package/dist/add.js +42 -5
- package/dist/create.d.ts +4 -0
- package/dist/create.js +9 -0
- package/dist/deploy-compose-profile.d.ts +3 -0
- package/dist/deploy-compose-profile.js +74 -8
- package/dist/deploy-compose-render.d.ts +1 -0
- package/dist/deploy-compose-render.js +46 -1
- package/dist/deploy-compose.js +2 -1
- package/dist/deploy-profile.js +18 -8
- package/dist/deploy.js +157 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +41 -0
- package/dist/lib.d.ts +2 -0
- package/dist/lib.js +8 -2
- package/dist/lockfile.d.ts +6 -1
- package/dist/lockfile.js +22 -4
- package/dist/provider.d.ts +24 -0
- package/dist/provider.js +142 -0
- package/dist/remove.js +6 -3
- package/dist/templates/fullstack/AGENTS.md +5 -4
- package/dist/templates/fullstack/README.md +10 -3
- package/dist/templates/fullstack/apps/api/package.json +3 -2
- package/dist/templates/fullstack/apps/api/src/config/env.validation.ts +1 -0
- package/dist/templates/fullstack/apps/api/src/config/providers.ts +9 -0
- package/dist/templates/fullstack/apps/api/src/core/services.ts +7 -2
- package/dist/templates/fullstack/apps/api/src/database/data-source.ts +30 -8
- package/dist/templates/fullstack/apps/api/src/database/database.spec.ts +58 -0
- package/dist/templates/fullstack/apps/api/src/database/database.ts +58 -2
- package/dist/templates/fullstack/apps/api/src/database/migrate.ts +24 -2
- package/dist/templates/fullstack/apps/api/src/database/sqlite-migrator.spec.ts +50 -0
- package/dist/templates/fullstack/apps/api/src/database/sqlite-migrator.ts +134 -0
- package/dist/templates/fullstack/compose.dev.yaml +2 -2
- package/dist/templates/fullstack/dot-agents/skills/podokit-deploy/references/profile.md +10 -3
- package/dist/templates/fullstack/dot-env.example +2 -0
- package/dist/templates/modules/admin-dashboard/files/apps/api/src/admin.module.ts +2 -2
- package/dist/templates/modules/admin-dashboard/files/apps/api/src/profile-image/profile-image.service.spec.ts +10 -4
- package/dist/templates/modules/admin-dashboard/files/apps/api/src/profile-image/profile-image.service.ts +10 -4
- package/dist/templates/modules/admin-dashboard/files/apps/api/src/site-settings/site-settings.service.ts +10 -4
- package/dist/templates/modules/admin-dashboard/module.manifest.json +3 -5
- package/dist/templates/modules/audit-log/files/apps/api/src/audit/audit-enabled.ts +2 -5
- package/dist/templates/modules/audit-log/files/apps/api/src/audit/audit.migration.spec.ts +33 -0
- package/dist/templates/modules/audit-log/files/apps/api/src/audit/audit.service.spec.ts +13 -0
- package/dist/templates/modules/audit-log/files/apps/api/src/audit/audit.service.ts +17 -8
- package/dist/templates/modules/audit-log/files/apps/api/src/migrations/1720300000000-InitAuditLogs.ts +5 -4
- package/dist/templates/modules/audit-log/module.manifest.json +1 -1
- package/dist/templates/modules/auth/files/apps/api/src/auth/account-issuer-migration.spec.ts +170 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/account-issuer-migration.ts +260 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth-config-store.ts +2 -2
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth.module.ts +4 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth.service.ts +2 -2
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth.ts +4 -4
- package/dist/templates/modules/auth/files/apps/api/src/auth/db.ts +55 -10
- package/dist/templates/modules/auth/files/apps/api/src/auth/feature-gate.ts +10 -14
- package/dist/templates/modules/auth/files/apps/api/src/migrate.ts +22 -6
- package/dist/templates/modules/auth/files/apps/api/src/migrations/1720400000000-InitAppSettings.ts +2 -1
- package/dist/templates/modules/auth/files/apps/api/src/migrations/1720500000000-InitAuthConfig.ts +4 -3
- package/dist/templates/modules/auth/files/apps/api/src/settings/app-settings.migration.spec.ts +15 -4
- package/dist/templates/modules/auth/files/apps/api/src/settings/auth-config.migration.spec.ts +30 -0
- package/dist/templates/modules/auth/module.manifest.json +1 -1
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.spec.ts +10 -0
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.ts +107 -0
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/jobs.module.ts +12 -11
- package/dist/templates/modules/bullmq/files/apps/api/src/main-worker.ts +5 -0
- package/dist/templates/modules/bullmq/module.manifest.json +7 -0
- package/dist/templates/modules/cache-memory/files/apps/api/src/cache-memory/cache-memory.module.spec.ts +16 -0
- package/dist/templates/modules/cache-memory/files/apps/api/src/cache-memory/cache-memory.module.ts +12 -0
- package/dist/templates/modules/cache-memory/module.manifest.json +30 -0
- package/dist/templates/modules/events-memory/files/apps/api/src/events-provider-memory/events-memory.module.ts +14 -0
- package/dist/templates/modules/events-memory/module.manifest.json +30 -0
- package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/events-redis.module.ts +16 -0
- package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/redis-event.bus.ts +75 -0
- package/dist/templates/modules/events-redis/module.manifest.json +32 -0
- package/dist/templates/modules/file-upload/files/apps/api/src/files/files.module.ts +9 -5
- package/dist/templates/modules/file-upload/files/tests/api/files.api.spec.ts +1 -1
- package/dist/templates/modules/file-upload/module.manifest.json +4 -3
- package/dist/templates/modules/job-progress/files/apps/api/src/progress/job-progress.module.ts +36 -14
- package/dist/templates/modules/job-progress/files/apps/api/src/progress/progress.bridge.spec.ts +9 -32
- package/dist/templates/modules/job-progress/files/apps/api/src/progress/progress.processor.ts +16 -11
- package/dist/templates/modules/job-progress/module.manifest.json +7 -8
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/jobs-local.module.ts +42 -0
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.spec.ts +45 -0
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.ts +212 -0
- package/dist/templates/modules/jobs-local/module.manifest.json +34 -0
- package/dist/templates/modules/mailer/files/apps/api/src/mail/mailer.ts +7 -3
- package/dist/templates/modules/mailer/module.manifest.json +2 -4
- package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/storage-local.module.spec.ts +25 -0
- package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/storage-local.module.ts +60 -0
- package/dist/templates/modules/object-storage-local/module.manifest.json +34 -0
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-object.store.ts +50 -0
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage.module.ts +8 -0
- package/dist/templates/modules/object-storage-s3/files/infra/docker/minio.compose.yml +4 -4
- package/dist/templates/modules/object-storage-s3/files/tests/api/storage.api.spec.ts +20 -12
- package/dist/templates/modules/object-storage-s3/module.manifest.json +4 -2
- package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.module.spec.ts +14 -1
- package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.module.ts +18 -37
- package/dist/templates/modules/rate-limit/files/tests/api/rate-limit.api.spec.ts +12 -0
- package/dist/templates/modules/rate-limit/module.manifest.json +8 -6
- package/dist/templates/modules/redis/files/apps/api/src/redis/redis-cache.store.ts +82 -0
- package/dist/templates/modules/redis/files/apps/api/src/redis/redis.module.ts +6 -0
- package/dist/templates/modules/redis/module.manifest.json +2 -0
- package/dist/templates/modules/sse/files/apps/api/src/events/events.module.ts +5 -6
- package/dist/templates/modules/sse/files/apps/api/src/events/events.service.spec.ts +4 -4
- package/dist/templates/modules/sse/files/apps/api/src/events/events.service.ts +6 -5
- package/dist/templates/modules/sse/module.manifest.json +4 -6
- package/dist/templates/todo/AGENTS.md +1 -1
- package/dist/templates/todo/README.md +8 -3
- package/package.json +2 -1
- package/dist/templates/modules/job-progress/files/apps/api/src/progress/progress.bridge.ts +0 -32
- package/dist/templates/modules/mailer/files/apps/api/src/mail/db.ts +0 -13
- package/dist/templates/modules/sse/files/apps/api/src/events/events.transport.ts +0 -111
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "job-progress",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
3
|
+
"manifestVersion": 3,
|
|
4
|
+
"description": "Live job progress over the selected jobs and events providers, streamed to clients with SSE.",
|
|
5
|
+
"requires": ["sse"],
|
|
6
|
+
"requiresCapabilities": ["jobs", "events"],
|
|
5
7
|
"targetApp": "api",
|
|
6
8
|
"env": [],
|
|
7
9
|
"inject": [
|
|
8
10
|
{ "file": "apps/api/src/app.ts", "marker": "// podokit:end:imports", "text": "import { jobProgressModule } from \"./progress/job-progress.module\";" },
|
|
9
|
-
{ "file": "apps/api/src/app.ts", "marker": "// podokit:end:modules", "text": "jobProgressModule," }
|
|
10
|
-
{ "file": "apps/api/src/jobs/worker.module.ts", "marker": "// podokit:end:worker-imports", "text": "import { processProgressJob } from \"../progress/progress.processor\";" },
|
|
11
|
-
{ "file": "apps/api/src/jobs/worker.module.ts", "marker": "// podokit:end:worker-processors", "text": "{ queue: \"progress\", processor: processProgressJob }," }
|
|
11
|
+
{ "file": "apps/api/src/app.ts", "marker": "// podokit:end:modules", "text": "jobProgressModule," }
|
|
12
12
|
],
|
|
13
13
|
"instructions": [
|
|
14
|
-
"
|
|
15
|
-
"Run API and worker in separate terminals: {{rootRun}} dev / {{apiRun}} dev:worker{{apiWorkspaceArg}}",
|
|
14
|
+
"Configure the selected jobs and events providers before starting the API.",
|
|
16
15
|
"Watch progress: curl -N localhost:5002/events/stream",
|
|
17
16
|
"Start a job: curl -XPOST localhost:5002/progress -H 'content-type: application/json' -d '{\"steps\":5}'",
|
|
18
|
-
"The stream shows job-progress events 20 -> 40 -> ... -> 100
|
|
17
|
+
"The stream shows job-progress events 20 -> 40 -> ... -> 100 through the selected events provider."
|
|
19
18
|
]
|
|
20
19
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { AppException } from "@podosoft/podokit-contracts";
|
|
2
|
+
import { JOBS } from "@podosoft/podokit-runtime";
|
|
3
|
+
import { Elysia, t } from "elysia";
|
|
4
|
+
import { PROVIDERS } from "../config/providers";
|
|
5
|
+
import { DATABASE, type AppPlugin, type PodokitModule } from "../core/services";
|
|
6
|
+
import { LocalJobQueue } from "./local-job.queue";
|
|
7
|
+
|
|
8
|
+
const localJobsPlugin: AppPlugin = ({ services }) => {
|
|
9
|
+
if (PROVIDERS.jobs !== "local") return new Elysia({ name: "podokit.jobs-local.inactive" });
|
|
10
|
+
const queue = services.resolve(JOBS);
|
|
11
|
+
return new Elysia({ name: "podokit.jobs-local" })
|
|
12
|
+
.post("/jobs", async ({ body, set }) => {
|
|
13
|
+
const job = await queue.enqueue("demo", { text: body.text });
|
|
14
|
+
set.status = 201;
|
|
15
|
+
return { id: job.id };
|
|
16
|
+
}, {
|
|
17
|
+
body: t.Object({ text: t.String({ minLength: 1 }) }),
|
|
18
|
+
detail: { tags: ["jobs"], summary: "Enqueue a job" },
|
|
19
|
+
})
|
|
20
|
+
.get("/jobs/:id", async ({ params }) => {
|
|
21
|
+
const job = await queue.get(params.id);
|
|
22
|
+
if (!job) throw new AppException("JOB_NOT_FOUND", `Job ${params.id} not found`, 404);
|
|
23
|
+
return { id: job.id, state: job.status, error: job.error ?? null };
|
|
24
|
+
}, {
|
|
25
|
+
params: t.Object({ id: t.String({ minLength: 1 }) }),
|
|
26
|
+
detail: { tags: ["jobs"], summary: "Read job status" },
|
|
27
|
+
});
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export const jobsLocalModule: PodokitModule = {
|
|
31
|
+
name: "jobs-local",
|
|
32
|
+
configure: (_env, services): void => {
|
|
33
|
+
if (PROVIDERS.jobs !== "local") return;
|
|
34
|
+
const queue = new LocalJobQueue(services.resolve(DATABASE).sql, {
|
|
35
|
+
pollIntervalMs: Number(process.env.LOCAL_JOBS_POLL_INTERVAL_MS ?? 250),
|
|
36
|
+
});
|
|
37
|
+
void queue.process("demo", () => undefined);
|
|
38
|
+
services.register(JOBS, queue, () => queue.close());
|
|
39
|
+
services.onStart(() => queue.start());
|
|
40
|
+
},
|
|
41
|
+
plugin: localJobsPlugin,
|
|
42
|
+
};
|
package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.spec.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import { SQL } from "bun";
|
|
3
|
+
import { LocalJobQueue } from "./local-job.queue";
|
|
4
|
+
|
|
5
|
+
let sql: SQL | undefined;
|
|
6
|
+
|
|
7
|
+
afterEach(async () => {
|
|
8
|
+
await sql?.close();
|
|
9
|
+
sql = undefined;
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
describe("local job queue", () => {
|
|
13
|
+
it("persists and processes a job", async () => {
|
|
14
|
+
sql = new SQL(":memory:");
|
|
15
|
+
const queue = new LocalJobQueue(sql, { pollIntervalMs: 10 });
|
|
16
|
+
const payloads: unknown[] = [];
|
|
17
|
+
await queue.process("demo", (payload) => { payloads.push(payload); });
|
|
18
|
+
const job = await queue.enqueue("demo", { text: "hello" });
|
|
19
|
+
expect(await queue.runOnce()).toBe(true);
|
|
20
|
+
expect(payloads).toEqual([{ text: "hello" }]);
|
|
21
|
+
await expect(queue.get(job.id)).resolves.toMatchObject({
|
|
22
|
+
status: "completed",
|
|
23
|
+
attempts: 1,
|
|
24
|
+
});
|
|
25
|
+
queue.close();
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it("retries failures and records the final error", async () => {
|
|
29
|
+
sql = new SQL(":memory:");
|
|
30
|
+
let now = 1_000;
|
|
31
|
+
const queue = new LocalJobQueue(sql, { pollIntervalMs: 10, now: () => now });
|
|
32
|
+
await queue.process("fail", () => { throw new Error("expected failure"); });
|
|
33
|
+
const job = await queue.enqueue("fail", {}, { attempts: 2 });
|
|
34
|
+
expect(await queue.runOnce()).toBe(true);
|
|
35
|
+
expect((await queue.get(job.id))?.status).toBe("waiting");
|
|
36
|
+
now += 10;
|
|
37
|
+
expect(await queue.runOnce()).toBe(true);
|
|
38
|
+
await expect(queue.get(job.id)).resolves.toMatchObject({
|
|
39
|
+
status: "failed",
|
|
40
|
+
attempts: 2,
|
|
41
|
+
error: "expected failure",
|
|
42
|
+
});
|
|
43
|
+
queue.close();
|
|
44
|
+
});
|
|
45
|
+
});
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import type { SQL } from "bun";
|
|
2
|
+
import {
|
|
3
|
+
type EnqueueOptions,
|
|
4
|
+
type JobHandler,
|
|
5
|
+
type JobQueue,
|
|
6
|
+
type JobReference,
|
|
7
|
+
type JobSnapshot,
|
|
8
|
+
type JobStatus,
|
|
9
|
+
type Unsubscribe,
|
|
10
|
+
} from "@podosoft/podokit-runtime";
|
|
11
|
+
|
|
12
|
+
interface JobRow {
|
|
13
|
+
id: string;
|
|
14
|
+
name: string;
|
|
15
|
+
payload: string;
|
|
16
|
+
status: JobStatus;
|
|
17
|
+
attempts: number;
|
|
18
|
+
maxAttempts: number;
|
|
19
|
+
availableAt: number;
|
|
20
|
+
createdAt: number;
|
|
21
|
+
updatedAt: number;
|
|
22
|
+
error: string | null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface LocalJobQueueOptions {
|
|
26
|
+
pollIntervalMs?: number;
|
|
27
|
+
now?: () => number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export class LocalJobQueue implements JobQueue {
|
|
31
|
+
private readonly handlers = new Map<string, JobHandler>();
|
|
32
|
+
private readonly pollIntervalMs: number;
|
|
33
|
+
private readonly now: () => number;
|
|
34
|
+
private timer?: ReturnType<typeof setTimeout>;
|
|
35
|
+
private started = false;
|
|
36
|
+
private initialized?: Promise<void>;
|
|
37
|
+
|
|
38
|
+
constructor(
|
|
39
|
+
private readonly sql: SQL,
|
|
40
|
+
options: LocalJobQueueOptions = {},
|
|
41
|
+
) {
|
|
42
|
+
this.pollIntervalMs = options.pollIntervalMs ?? 250;
|
|
43
|
+
if (!Number.isSafeInteger(this.pollIntervalMs) || this.pollIntervalMs < 10) {
|
|
44
|
+
throw new Error("pollIntervalMs must be an integer of at least 10");
|
|
45
|
+
}
|
|
46
|
+
this.now = options.now ?? Date.now;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
initialize(): Promise<void> {
|
|
50
|
+
this.initialized ??= (async () => {
|
|
51
|
+
await this.sql`
|
|
52
|
+
CREATE TABLE IF NOT EXISTS "podokit_job" (
|
|
53
|
+
"id" text PRIMARY KEY,
|
|
54
|
+
"name" text NOT NULL,
|
|
55
|
+
"payload" text NOT NULL,
|
|
56
|
+
"status" text NOT NULL,
|
|
57
|
+
"attempts" integer NOT NULL,
|
|
58
|
+
"maxAttempts" integer NOT NULL,
|
|
59
|
+
"availableAt" bigint NOT NULL,
|
|
60
|
+
"createdAt" bigint NOT NULL,
|
|
61
|
+
"updatedAt" bigint NOT NULL,
|
|
62
|
+
"error" text
|
|
63
|
+
)
|
|
64
|
+
`;
|
|
65
|
+
await this.sql`
|
|
66
|
+
CREATE INDEX IF NOT EXISTS "podokit_job_pending"
|
|
67
|
+
ON "podokit_job" ("status", "availableAt", "createdAt")
|
|
68
|
+
`;
|
|
69
|
+
})();
|
|
70
|
+
return this.initialized;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async start(): Promise<void> {
|
|
74
|
+
if (this.started) return;
|
|
75
|
+
await this.initialize();
|
|
76
|
+
this.started = true;
|
|
77
|
+
this.schedule(0);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async enqueue(
|
|
81
|
+
name: string,
|
|
82
|
+
payload: unknown,
|
|
83
|
+
options: EnqueueOptions = {},
|
|
84
|
+
): Promise<JobReference> {
|
|
85
|
+
await this.initialize();
|
|
86
|
+
const encoded = JSON.stringify(payload);
|
|
87
|
+
if (encoded === undefined) throw new Error("Job payload must be JSON serializable");
|
|
88
|
+
const now = this.now();
|
|
89
|
+
const id = options.deduplicationKey
|
|
90
|
+
? `${name}:${options.deduplicationKey}`
|
|
91
|
+
: crypto.randomUUID();
|
|
92
|
+
const attempts = options.attempts ?? 1;
|
|
93
|
+
const delayMs = options.delayMs ?? 0;
|
|
94
|
+
if (!Number.isSafeInteger(attempts) || attempts < 1) {
|
|
95
|
+
throw new Error("attempts must be a positive integer");
|
|
96
|
+
}
|
|
97
|
+
if (!Number.isSafeInteger(delayMs) || delayMs < 0) {
|
|
98
|
+
throw new Error("delayMs must be a non-negative integer");
|
|
99
|
+
}
|
|
100
|
+
await this.sql`
|
|
101
|
+
INSERT INTO "podokit_job" (
|
|
102
|
+
"id", "name", "payload", "status", "attempts", "maxAttempts",
|
|
103
|
+
"availableAt", "createdAt", "updatedAt", "error"
|
|
104
|
+
) VALUES (
|
|
105
|
+
${id}, ${name}, ${encoded}, 'waiting', 0, ${attempts},
|
|
106
|
+
${now + delayMs}, ${now}, ${now}, NULL
|
|
107
|
+
)
|
|
108
|
+
ON CONFLICT ("id") DO NOTHING
|
|
109
|
+
`;
|
|
110
|
+
return { id, name };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async get(id: string): Promise<JobSnapshot | null> {
|
|
114
|
+
await this.initialize();
|
|
115
|
+
const rows = await this.sql<JobRow[]>`
|
|
116
|
+
SELECT "id", "name", "payload", "status", "attempts", "maxAttempts",
|
|
117
|
+
"availableAt", "createdAt", "updatedAt", "error"
|
|
118
|
+
FROM "podokit_job" WHERE "id" = ${id}
|
|
119
|
+
`;
|
|
120
|
+
const row = rows[0];
|
|
121
|
+
return row ? this.snapshot(row) : null;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
process(name: string, handler: JobHandler): Promise<Unsubscribe> {
|
|
125
|
+
if (this.handlers.has(name)) throw new Error(`A handler is already registered for job ${name}`);
|
|
126
|
+
this.handlers.set(name, handler);
|
|
127
|
+
return Promise.resolve(() => {
|
|
128
|
+
this.handlers.delete(name);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async runOnce(): Promise<boolean> {
|
|
133
|
+
await this.initialize();
|
|
134
|
+
const now = this.now();
|
|
135
|
+
const rows = await this.sql<JobRow[]>`
|
|
136
|
+
UPDATE "podokit_job"
|
|
137
|
+
SET "status" = 'active', "attempts" = "attempts" + 1, "updatedAt" = ${now}
|
|
138
|
+
WHERE "id" = (
|
|
139
|
+
SELECT "id" FROM "podokit_job"
|
|
140
|
+
WHERE "status" = 'waiting' AND "availableAt" <= ${now}
|
|
141
|
+
ORDER BY "createdAt" ASC LIMIT 1
|
|
142
|
+
) AND "status" = 'waiting'
|
|
143
|
+
RETURNING "id", "name", "payload", "status", "attempts", "maxAttempts",
|
|
144
|
+
"availableAt", "createdAt", "updatedAt", "error"
|
|
145
|
+
`;
|
|
146
|
+
const row = rows[0];
|
|
147
|
+
if (!row) return false;
|
|
148
|
+
const handler = this.handlers.get(row.name);
|
|
149
|
+
if (!handler) {
|
|
150
|
+
await this.fail(row, new Error(`No handler is registered for job ${row.name}`));
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
await handler(JSON.parse(row.payload) as unknown, { id: row.id, name: row.name });
|
|
155
|
+
const completedAt = this.now();
|
|
156
|
+
await this.sql`
|
|
157
|
+
UPDATE "podokit_job"
|
|
158
|
+
SET "status" = 'completed', "updatedAt" = ${completedAt}, "error" = NULL
|
|
159
|
+
WHERE "id" = ${row.id}
|
|
160
|
+
`;
|
|
161
|
+
} catch (error) {
|
|
162
|
+
await this.fail(row, error);
|
|
163
|
+
}
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
close(): void {
|
|
168
|
+
this.started = false;
|
|
169
|
+
if (this.timer) clearTimeout(this.timer);
|
|
170
|
+
this.timer = undefined;
|
|
171
|
+
this.handlers.clear();
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
private schedule(delay: number): void {
|
|
175
|
+
if (!this.started) return;
|
|
176
|
+
this.timer = setTimeout(() => {
|
|
177
|
+
void this.runOnce()
|
|
178
|
+
.catch((error: unknown) => {
|
|
179
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
180
|
+
process.stderr.write(`Process local job failed: ${message}\n`);
|
|
181
|
+
})
|
|
182
|
+
.finally(() => this.schedule(this.pollIntervalMs));
|
|
183
|
+
}, delay);
|
|
184
|
+
this.timer.unref?.();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
private async fail(row: JobRow, error: unknown): Promise<void> {
|
|
188
|
+
const failedAt = this.now();
|
|
189
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
190
|
+
const retry = row.attempts < row.maxAttempts;
|
|
191
|
+
await this.sql`
|
|
192
|
+
UPDATE "podokit_job"
|
|
193
|
+
SET "status" = ${retry ? "waiting" : "failed"},
|
|
194
|
+
"availableAt" = ${retry ? failedAt + this.pollIntervalMs : failedAt},
|
|
195
|
+
"updatedAt" = ${failedAt},
|
|
196
|
+
"error" = ${message.slice(0, 2_000)}
|
|
197
|
+
WHERE "id" = ${row.id}
|
|
198
|
+
`;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
private snapshot(row: JobRow): JobSnapshot {
|
|
202
|
+
return {
|
|
203
|
+
id: row.id,
|
|
204
|
+
name: row.name,
|
|
205
|
+
status: row.status,
|
|
206
|
+
attempts: row.attempts,
|
|
207
|
+
createdAt: Number(row.createdAt),
|
|
208
|
+
updatedAt: Number(row.updatedAt),
|
|
209
|
+
...(row.error ? { error: row.error } : {}),
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"manifestVersion": 3,
|
|
3
|
+
"name": "jobs-local",
|
|
4
|
+
"description": "Database-persistent local job queue with an embedded single-process worker.",
|
|
5
|
+
"provides": { "jobs": "local" },
|
|
6
|
+
"runtimeConstraints": { "maxApiReplicas": 1, "embeddedWorker": true },
|
|
7
|
+
"requiredProjectFiles": [
|
|
8
|
+
"apps/api/src/config/providers.ts",
|
|
9
|
+
"apps/api/src/database/database.ts"
|
|
10
|
+
],
|
|
11
|
+
"managedGlobs": [
|
|
12
|
+
"apps/api/src/jobs-local/**"
|
|
13
|
+
],
|
|
14
|
+
"targetApp": "api",
|
|
15
|
+
"env": [
|
|
16
|
+
"LOCAL_JOBS_POLL_INTERVAL_MS=250"
|
|
17
|
+
],
|
|
18
|
+
"inject": [
|
|
19
|
+
{
|
|
20
|
+
"file": "apps/api/src/app.ts",
|
|
21
|
+
"marker": "// podokit:end:imports",
|
|
22
|
+
"text": "import { jobsLocalModule } from \"./jobs-local/jobs-local.module\";"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"file": "apps/api/src/app.ts",
|
|
26
|
+
"marker": "// podokit:end:modules",
|
|
27
|
+
"text": "jobsLocalModule,"
|
|
28
|
+
}
|
|
29
|
+
],
|
|
30
|
+
"instructions": [
|
|
31
|
+
"The local worker runs inside the API process and persists jobs in the selected database.",
|
|
32
|
+
"Keep the API at one replica or select the BullMQ jobs provider."
|
|
33
|
+
]
|
|
34
|
+
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import nodemailer, { type Transporter } from "nodemailer";
|
|
2
|
-
import { pool } from "./db";
|
|
3
2
|
import { createConfigStore, type SmtpConfig } from "@podosoft/podokit-auth";
|
|
4
3
|
|
|
5
4
|
// Reusable email sending. SMTP is resolved in priority order, applied live (short
|
|
@@ -10,9 +9,14 @@ import { createConfigStore, type SmtpConfig } from "@podosoft/podokit-auth";
|
|
|
10
9
|
// 3. none → a JSON transport that logs the message so dev links are grabbable.
|
|
11
10
|
// The whole transport can also be overridden from app code (setMailTransport),
|
|
12
11
|
// e.g. from apps/api/src/app.extensions.ts, to plug in a provider SDK.
|
|
13
|
-
|
|
12
|
+
type MailConfigStore = Pick<ReturnType<typeof createConfigStore>, "smtpConfig">;
|
|
13
|
+
let store: MailConfigStore | undefined;
|
|
14
14
|
const TTL_MS = 3_000;
|
|
15
15
|
|
|
16
|
+
export function setMailConfigStore(configStore: MailConfigStore): void {
|
|
17
|
+
store = configStore;
|
|
18
|
+
}
|
|
19
|
+
|
|
16
20
|
function envSmtp(): SmtpConfig | null {
|
|
17
21
|
if (!process.env.SMTP_HOST) return null;
|
|
18
22
|
return {
|
|
@@ -28,7 +32,7 @@ function envSmtp(): SmtpConfig | null {
|
|
|
28
32
|
async function resolveSmtp(): Promise<SmtpConfig | null> {
|
|
29
33
|
try {
|
|
30
34
|
// Reads auth_config and falls back to env internally when auth is installed.
|
|
31
|
-
return await store.smtpConfig();
|
|
35
|
+
return store ? await store.smtpConfig() : envSmtp();
|
|
32
36
|
} catch {
|
|
33
37
|
// No auth_config table (auth not installed) — use env directly.
|
|
34
38
|
return envSmtp();
|
|
@@ -5,12 +5,10 @@
|
|
|
5
5
|
"targetApp": "api",
|
|
6
6
|
"dependencies": {
|
|
7
7
|
"nodemailer": "^9.0.5",
|
|
8
|
-
"@podosoft/podokit-auth": "^0.
|
|
9
|
-
"pg": "^8.12.0"
|
|
8
|
+
"@podosoft/podokit-auth": "^0.4.0"
|
|
10
9
|
},
|
|
11
10
|
"devDependencies": {
|
|
12
|
-
"@types/nodemailer": "^8.0.1"
|
|
13
|
-
"@types/pg": "^8.11.10"
|
|
11
|
+
"@types/nodemailer": "^8.0.1"
|
|
14
12
|
},
|
|
15
13
|
"env": [
|
|
16
14
|
"# ── Mailer (SMTP) — optional ──",
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { LocalObjectStore } from "@podosoft/podokit-runtime";
|
|
6
|
+
|
|
7
|
+
let root: string | undefined;
|
|
8
|
+
|
|
9
|
+
afterEach(async () => {
|
|
10
|
+
if (root) await rm(root, { recursive: true, force: true });
|
|
11
|
+
root = undefined;
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
describe("local object storage provider", () => {
|
|
15
|
+
it("stores and reads an object without allowing traversal", async () => {
|
|
16
|
+
root = await mkdtemp(join(tmpdir(), "podokit-storage-"));
|
|
17
|
+
const storage = new LocalObjectStore({ rootDirectory: root });
|
|
18
|
+
await storage.put("uploads/hello.txt", "hello", { contentType: "text/plain" });
|
|
19
|
+
expect(await storage.exists("uploads/hello.txt")).toBe(true);
|
|
20
|
+
expect(await storage.getDownloadUrl("uploads/hello.txt")).toBe(
|
|
21
|
+
"/files/content/uploads/hello.txt",
|
|
22
|
+
);
|
|
23
|
+
await expect(storage.put("../outside.txt", "no")).rejects.toThrow("safe relative path");
|
|
24
|
+
});
|
|
25
|
+
});
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import {
|
|
2
|
+
LocalObjectStore,
|
|
3
|
+
OBJECT_STORAGE,
|
|
4
|
+
type ObjectData,
|
|
5
|
+
} from "@podosoft/podokit-runtime";
|
|
6
|
+
import { Elysia, t } from "elysia";
|
|
7
|
+
import { resolve } from "node:path";
|
|
8
|
+
import type { AppPlugin, PodokitModule } from "../core/services";
|
|
9
|
+
import { PROVIDERS } from "../config/providers";
|
|
10
|
+
|
|
11
|
+
function responseBody(object: ObjectData): ReadableStream<Uint8Array> {
|
|
12
|
+
const iterator = object.body[Symbol.asyncIterator]();
|
|
13
|
+
return new ReadableStream<Uint8Array>({
|
|
14
|
+
async pull(controller): Promise<void> {
|
|
15
|
+
const next = await iterator.next();
|
|
16
|
+
if (next.done) controller.close();
|
|
17
|
+
else controller.enqueue(next.value);
|
|
18
|
+
},
|
|
19
|
+
async cancel(): Promise<void> {
|
|
20
|
+
await iterator.return?.();
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const localStoragePlugin: AppPlugin = ({ services }) => {
|
|
26
|
+
if (PROVIDERS["object-storage"] !== "local") {
|
|
27
|
+
return new Elysia({ name: "podokit.storage-local.inactive" });
|
|
28
|
+
}
|
|
29
|
+
const storage = services.resolve(OBJECT_STORAGE);
|
|
30
|
+
return new Elysia({ name: "podokit.storage-local" })
|
|
31
|
+
.get("/files/content/*", async ({ params, set }) => {
|
|
32
|
+
const object = await storage.get(params["*"]);
|
|
33
|
+
if (object.contentType) set.headers["content-type"] = object.contentType;
|
|
34
|
+
set.headers["content-length"] = String(object.size);
|
|
35
|
+
return new Response(responseBody(object));
|
|
36
|
+
}, {
|
|
37
|
+
detail: { tags: ["files"], summary: "Read a local object" },
|
|
38
|
+
})
|
|
39
|
+
.put("/storage/:key", async ({ params, body }) => {
|
|
40
|
+
await storage.put(params.key, body.content, { contentType: "text/plain" });
|
|
41
|
+
return { key: params.key };
|
|
42
|
+
}, {
|
|
43
|
+
params: t.Object({ key: t.String({ minLength: 1 }) }),
|
|
44
|
+
body: t.Object({ content: t.String() }),
|
|
45
|
+
detail: { tags: ["storage"], summary: "Store an object" },
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export const storageLocalModule: PodokitModule = {
|
|
50
|
+
name: "object-storage-local",
|
|
51
|
+
configure: (_env, services): void => {
|
|
52
|
+
if (PROVIDERS["object-storage"] !== "local") return;
|
|
53
|
+
const storage = new LocalObjectStore({
|
|
54
|
+
rootDirectory: resolve(process.env.LOCAL_STORAGE_PATH ?? "./data/files"),
|
|
55
|
+
publicBaseUrl: "/files/content",
|
|
56
|
+
});
|
|
57
|
+
services.register(OBJECT_STORAGE, storage, () => storage.close());
|
|
58
|
+
},
|
|
59
|
+
plugin: localStoragePlugin,
|
|
60
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"manifestVersion": 3,
|
|
3
|
+
"name": "object-storage-local",
|
|
4
|
+
"description": "Atomic path-confined local filesystem object storage for desktop and single-host applications.",
|
|
5
|
+
"provides": { "object-storage": "local" },
|
|
6
|
+
"runtimeConstraints": { "maxApiReplicas": 1 },
|
|
7
|
+
"requiredProjectFiles": [
|
|
8
|
+
"apps/api/src/config/providers.ts"
|
|
9
|
+
],
|
|
10
|
+
"managedGlobs": [
|
|
11
|
+
"apps/api/src/storage-local/**"
|
|
12
|
+
],
|
|
13
|
+
"targetApp": "api",
|
|
14
|
+
"env": [
|
|
15
|
+
"# Local object storage root. Use an absolute path in production and desktop packaging.",
|
|
16
|
+
"LOCAL_STORAGE_PATH=./data/files"
|
|
17
|
+
],
|
|
18
|
+
"inject": [
|
|
19
|
+
{
|
|
20
|
+
"file": "apps/api/src/app.ts",
|
|
21
|
+
"marker": "// podokit:end:imports",
|
|
22
|
+
"text": "import { storageLocalModule } from \"./storage-local/storage-local.module\";"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"file": "apps/api/src/app.ts",
|
|
26
|
+
"marker": "// podokit:end:modules",
|
|
27
|
+
"text": "storageLocalModule,"
|
|
28
|
+
}
|
|
29
|
+
],
|
|
30
|
+
"instructions": [
|
|
31
|
+
"Local object storage is single-host. Back up LOCAL_STORAGE_PATH together with the database.",
|
|
32
|
+
"Keep the API at one replica or select the S3 provider."
|
|
33
|
+
]
|
|
34
|
+
}
|
package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-object.store.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
DownloadUrlOptions,
|
|
3
|
+
ObjectBody,
|
|
4
|
+
ObjectData,
|
|
5
|
+
ObjectStore,
|
|
6
|
+
PutObjectOptions,
|
|
7
|
+
StoredObject,
|
|
8
|
+
} from "@podosoft/podokit-runtime";
|
|
9
|
+
import type { StorageService } from "./storage.service";
|
|
10
|
+
|
|
11
|
+
async function bodyBytes(body: ObjectBody): Promise<Uint8Array> {
|
|
12
|
+
if (typeof body === "string") return Buffer.from(body);
|
|
13
|
+
if (body instanceof Uint8Array) return body;
|
|
14
|
+
const chunks: Uint8Array[] = [];
|
|
15
|
+
for await (const chunk of body) chunks.push(chunk);
|
|
16
|
+
return Buffer.concat(chunks);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async function* oneChunk(body: Uint8Array): AsyncGenerator<Uint8Array> {
|
|
20
|
+
yield body;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class S3ObjectStore implements ObjectStore {
|
|
24
|
+
constructor(private readonly storage: StorageService) {}
|
|
25
|
+
|
|
26
|
+
async put(key: string, body: ObjectBody, options: PutObjectOptions = {}): Promise<StoredObject> {
|
|
27
|
+
const bytes = await bodyBytes(body);
|
|
28
|
+
await this.storage.put(key, bytes, options.contentType);
|
|
29
|
+
return { key, size: bytes.byteLength, ...options };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async get(key: string): Promise<ObjectData> {
|
|
33
|
+
const bytes = await this.storage.get(key);
|
|
34
|
+
return { key, size: bytes.byteLength, body: oneChunk(bytes) };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
exists(key: string): Promise<boolean> {
|
|
38
|
+
return this.storage.exists(key);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
delete(key: string): Promise<void> {
|
|
42
|
+
return this.storage.delete(key);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
getDownloadUrl(key: string, options: DownloadUrlOptions = {}): Promise<string> {
|
|
46
|
+
return this.storage.presignedGetUrl(key, options.expiresInSeconds);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
close(): void {}
|
|
50
|
+
}
|
package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage.module.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { Elysia, t } from "elysia";
|
|
2
|
+
import { OBJECT_STORAGE } from "@podosoft/podokit-runtime";
|
|
3
|
+
import { PROVIDERS } from "../config/providers";
|
|
2
4
|
import {
|
|
3
5
|
type AppPlugin,
|
|
4
6
|
type PodokitModule,
|
|
@@ -6,10 +8,14 @@ import {
|
|
|
6
8
|
type ServiceKey,
|
|
7
9
|
} from "../core/services";
|
|
8
10
|
import { StorageService } from "./storage.service";
|
|
11
|
+
import { S3ObjectStore } from "./s3-object.store";
|
|
9
12
|
|
|
10
13
|
export const STORAGE = Symbol("storage") as ServiceKey<StorageService>;
|
|
11
14
|
|
|
12
15
|
const storagePlugin: AppPlugin = ({ services }) => {
|
|
16
|
+
if (PROVIDERS["object-storage"] !== "s3") {
|
|
17
|
+
return new Elysia({ name: "podokit.storage-s3.inactive" });
|
|
18
|
+
}
|
|
13
19
|
const storage = services.resolve(STORAGE);
|
|
14
20
|
return new Elysia({ name: "podokit.storage" })
|
|
15
21
|
.put("/storage/:key", async ({ params, body }) => {
|
|
@@ -38,8 +44,10 @@ const storagePlugin: AppPlugin = ({ services }) => {
|
|
|
38
44
|
export const storageModule: PodokitModule = {
|
|
39
45
|
name: "object-storage-s3",
|
|
40
46
|
configure: (_env, services): void => {
|
|
47
|
+
if (PROVIDERS["object-storage"] !== "s3") return;
|
|
41
48
|
const storage = new StorageService(services.resolve(READINESS));
|
|
42
49
|
services.register(STORAGE, storage, () => storage.close());
|
|
50
|
+
services.register(OBJECT_STORAGE, new S3ObjectStore(storage));
|
|
43
51
|
},
|
|
44
52
|
plugin: storagePlugin,
|
|
45
53
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Silo for local S3-compatible object storage — development only (in production
|
|
2
2
|
# point S3_* at a real S3-compatible service). Part of the `dev` profile; add it with:
|
|
3
3
|
#
|
|
4
4
|
# docker compose -f infra/docker/docker-compose.yml -f infra/docker/minio.compose.yml --profile dev up -d
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# Console: http://localhost:9001 (S3 API: http://localhost:9000)
|
|
7
7
|
services:
|
|
8
8
|
minio:
|
|
9
|
-
image:
|
|
9
|
+
image: pgsty/silo:RELEASE.2026-09-16T00-00-00Z@sha256:635197cb9f36d01bee221d34d1c7d7960f6a95c48b0b6c01d99cd13bdae51a46
|
|
10
10
|
profiles: ["dev"]
|
|
11
11
|
command: server /data --console-address ":9001"
|
|
12
12
|
environment:
|
|
@@ -18,9 +18,9 @@ services:
|
|
|
18
18
|
volumes:
|
|
19
19
|
- minio-data:/data
|
|
20
20
|
|
|
21
|
-
# Waits for
|
|
21
|
+
# Waits for object storage, then creates the bucket and exits.
|
|
22
22
|
minio-init:
|
|
23
|
-
image:
|
|
23
|
+
image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z@sha256:cfc83108c3abb371f8fb84d99c1fdc88f8c237e022409b0081fb7c0a3be634dd
|
|
24
24
|
profiles: ["dev"]
|
|
25
25
|
depends_on:
|
|
26
26
|
- minio
|