@tatara/cli 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.mjs +1161 -312
- package/package.json +1 -1
- package/templates/features/base/files/AGENTS.md +17 -0
- package/templates/features/base/files/biome.template.json +1 -1
- package/templates/features/base/files/src/apps/api/server.ts +7 -1
- package/templates/features/base/files/src/apps/cli/index.ts +4 -1
- package/templates/features/base/files/src/apps/http.ts +6 -0
- package/templates/features/base/files/src/core/logging/logger.ts +5 -3
- package/templates/features/cache/files/src/core/cache/index.ts +23 -0
- package/templates/features/cache/files/src/core/http/rate-limit.ts +15 -0
- package/templates/features/cache/files/tests/tatara/cache.test.ts +22 -0
- package/templates/features/cache/files/tests/tatara/rate-limit.test.ts +27 -0
- package/templates/features/cache/files+postgres/src/core/cache/index.ts +36 -0
- package/templates/features/db/files+mysql+drizzle/src/core/commands/db/command-runs.schema.ts +10 -0
- package/templates/features/db/files+mysql+drizzle/src/core/db/admin.ts +6 -1
- package/templates/features/db/files+mysql+prisma/prisma/models/command-runs.prisma +9 -0
- package/templates/features/db/files+mysql+prisma/src/core/db/admin.ts +1 -0
- package/templates/features/db/files+postgres/src/core/db/admin.ts +6 -1
- package/templates/features/email/files/src/core/email/send.ts +22 -2
- package/templates/features/email/files/src/modules/example/emails/welcome.ts +11 -0
- package/templates/features/jobs/files/src/apps/cron/index.ts +2 -3
- package/templates/features/jobs/files/src/apps/cron/runner.ts +11 -0
- package/templates/features/jobs/files/src/apps/worker/index.ts +2 -9
- package/templates/features/jobs/files/src/core/jobs/index.ts +37 -6
- package/templates/features/jobs/files/src/modules/example/tests/jobs.test.ts +7 -10
- package/templates/features/jobs/files+drizzle+postgres/src/core/jobs/outbox.ts +23 -0
- package/templates/features/jobs/files+drizzle+postgres/tests/tatara/jobs-outbox.test.ts +34 -0
- package/templates/features/jobs/files+postgres/src/core/jobs/index.ts +47 -0
- package/templates/features/jobs/files+prisma+postgres/src/core/jobs/outbox.ts +17 -0
- package/templates/features/jobs/files+prisma+postgres/tests/tatara/jobs-outbox.test.ts +34 -0
- package/templates/features/notifications/files/src/modules/example/notifications/welcome.ts +14 -0
- package/templates/features/notifications/files/src/modules/notifications/consumers/deliver.ts +6 -0
- package/templates/features/notifications/files/src/modules/notifications/consumers/fanout.ts +6 -0
- package/templates/features/notifications/files/src/modules/notifications/routes/notifications.routes.ts +168 -0
- package/templates/features/notifications/files/src/modules/notifications/services/notifier.ts +59 -0
- package/templates/features/notifications/files/src/modules/notifications/tests/notifications.test.ts +72 -0
- package/templates/features/notifications/files+drizzle/src/modules/notifications/services/store.ts +5 -0
- package/templates/features/notifications/files+drizzle+mysql/src/modules/notifications/db/notifications.schema.ts +40 -0
- package/templates/features/notifications/files+drizzle+postgres/src/modules/notifications/db/notifications.schema.ts +41 -0
- package/templates/features/notifications/files+prisma/prisma/models/notifications.prisma +34 -0
- package/templates/features/notifications/files+prisma/src/modules/notifications/services/store.ts +4 -0
- package/templates/features/observability/files/src/apps/instrument.ts +15 -0
- package/templates/features/realtime/files/src/core/realtime/index.ts +29 -0
- package/templates/features/realtime/files/src/modules/realtime/routes/realtime.routes.ts +38 -0
- package/templates/features/realtime/files/tests/tatara/realtime.test.ts +37 -0
- package/templates/versions.json +26 -6
package/package.json
CHANGED
|
@@ -55,6 +55,12 @@ __PM_EXEC__ tatara upgrade # after bumping @tatara/cli: run its co
|
|
|
55
55
|
`{ success: false, error, code, details }`.
|
|
56
56
|
- Route files are `routes/<name>.routes.ts` and default-export `defineRoutes({ app, basePath, router })`.
|
|
57
57
|
They are found automatically; there is no central list to edit.
|
|
58
|
+
- Long or live lists page by cursor: `cursorQuerySchema` for the query, fetch `limit + 1` rows after
|
|
59
|
+
`decodeCursor(positionSchema, cursor)`, answer `cursorPage(rows, limit, (row) => ({ createdAt, id }))`
|
|
60
|
+
(schema: `cursorPageSchema(item)`).
|
|
61
|
+
- Requests time out after `REQUEST_TIMEOUT_MS` (504); give slow paths their own limit in
|
|
62
|
+
`src/apps/http.ts` (`timeouts.paths`). Behind proxies, set `TRUSTED_PROXY_HOPS` and read the caller
|
|
63
|
+
with `clientIp(c, { trustedHops })`.
|
|
58
64
|
|
|
59
65
|
## Configuration and logging
|
|
60
66
|
|
|
@@ -62,6 +68,17 @@ __PM_EXEC__ tatara upgrade # after bumping @tatara/cli: run its co
|
|
|
62
68
|
`.env.example`. Never read `process.env` elsewhere.
|
|
63
69
|
- Log with `c.get("logger")` in requests and `logger` from `#core/logging/logger` elsewhere.
|
|
64
70
|
No `console.*` outside commands and tests.
|
|
71
|
+
- Secrets and codes come from `@tatara/core`: `createEncryption(keys)` for secrets at rest (AES-GCM,
|
|
72
|
+
rotatable keys), `randomToken()`, `numericCode()`, `randomCode()`, `hashToken()`, `safeEqual()`.
|
|
73
|
+
Never `Math.random()` for anything a user could guess.
|
|
74
|
+
- User text: `text({ max })` and `url()` from `@tatara/core` in validators clean and check it (control,
|
|
75
|
+
invisible and bidi characters, whitespace; only http(s) links). Escape when you render HTML.
|
|
76
|
+
|
|
77
|
+
## Deploying
|
|
78
|
+
|
|
79
|
+
- Every deploy runs `node dist/apps/cli/index.mjs deploy`: commands with `phase: "deploy"` (migrations
|
|
80
|
+
first, then the rest by `order`), and each `phase: "once"` command the first time a database sees
|
|
81
|
+
it (recorded in a ledger). Commands without a phase only run when asked for.
|
|
65
82
|
|
|
66
83
|
## Tests
|
|
67
84
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://biomejs.dev/schemas/__BIOME_VERSION__/schema.json",
|
|
3
3
|
"files": {
|
|
4
|
-
"includes": ["**", "!**/node_modules", "!**/dist", "!**/coverage", "!src/generated", "!drizzle", "!prisma/migrations"]
|
|
4
|
+
"includes": ["**", "!**/node_modules", "!**/dist", "!**/coverage", "!src/generated", "!drizzle", "!prisma/migrations", "!contracts"]
|
|
5
5
|
},
|
|
6
6
|
"formatter": { "indentStyle": "tab", "lineWidth": 100 },
|
|
7
7
|
"javascript": { "formatter": { "quoteStyle": "double" } },
|
|
@@ -2,5 +2,11 @@ import { startServer } from "@tatara/http";
|
|
|
2
2
|
import { app } from "#apps/api/app";
|
|
3
3
|
import { env } from "#core/config/env";
|
|
4
4
|
import { logger } from "#core/logging/logger";
|
|
5
|
+
// tatara:slot imports
|
|
5
6
|
|
|
6
|
-
startServer(app, {
|
|
7
|
+
startServer(app, {
|
|
8
|
+
port: env.PORT,
|
|
9
|
+
name: "api",
|
|
10
|
+
logger,
|
|
11
|
+
// tatara:slot server-options
|
|
12
|
+
});
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { runCommand, shutdown } from "@tatara/core";
|
|
2
2
|
import { logger } from "#core/logging/logger";
|
|
3
3
|
import { commands } from "#generated/commands";
|
|
4
|
+
// tatara:slot imports
|
|
4
5
|
|
|
5
|
-
const code = await runCommand(commands, process.argv.slice(2), logger
|
|
6
|
+
const code = await runCommand(commands, process.argv.slice(2), logger, {
|
|
7
|
+
// tatara:slot options
|
|
8
|
+
});
|
|
6
9
|
await shutdown(code, logger);
|
|
@@ -20,6 +20,12 @@ export function createHttpApp(name: AppName, options: { title: string }) {
|
|
|
20
20
|
],
|
|
21
21
|
},
|
|
22
22
|
bodyLimitBytes: env.BODY_LIMIT_BYTES,
|
|
23
|
+
timeouts: {
|
|
24
|
+
defaultMs: env.REQUEST_TIMEOUT_MS,
|
|
25
|
+
paths: {
|
|
26
|
+
// tatara:slot timeout-paths
|
|
27
|
+
},
|
|
28
|
+
},
|
|
23
29
|
defaults: { principal: null, language: "en" },
|
|
24
30
|
});
|
|
25
31
|
// tatara:slot middleware
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { loggerOptions } from "@tatara/core";
|
|
2
2
|
import { pino } from "pino";
|
|
3
3
|
import { env } from "#core/config/env";
|
|
4
|
+
// tatara:slot imports
|
|
4
5
|
|
|
5
|
-
export const logger = pino(
|
|
6
|
-
loggerOptions({ env: env.NODE_ENV, level: env.LOG_LEVEL, service: "__APP_NAME__" }),
|
|
7
|
-
|
|
6
|
+
export const logger = pino({
|
|
7
|
+
...loggerOptions({ env: env.NODE_ENV, level: env.LOG_LEVEL, service: "__APP_NAME__" }),
|
|
8
|
+
// tatara:slot logger-options
|
|
9
|
+
});
|
|
8
10
|
|
|
9
11
|
export type Logger = typeof logger;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { onShutdown } from "@tatara/core";
|
|
2
|
+
import { type Cache, createCache, type CacheStore, memoryStore } from "@tatara/cache";
|
|
3
|
+
import { redisStore } from "@tatara/cache/redis";
|
|
4
|
+
import { Redis } from "ioredis";
|
|
5
|
+
import { env } from "#core/config/env";
|
|
6
|
+
import { logger } from "#core/logging/logger";
|
|
7
|
+
|
|
8
|
+
export type { Cache } from "@tatara/cache";
|
|
9
|
+
|
|
10
|
+
function redis(): CacheStore {
|
|
11
|
+
// connects on first command, so importing this file opens nothing
|
|
12
|
+
const client = new Redis(env.REDIS_URL, { lazyConnect: true, maxRetriesPerRequest: 3 });
|
|
13
|
+
onShutdown("cache", async () => {
|
|
14
|
+
await client.quit();
|
|
15
|
+
});
|
|
16
|
+
return redisStore(client);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** The app's cache; pick the store with CACHE_DRIVER. */
|
|
20
|
+
export const cache: Cache = createCache(env.CACHE_DRIVER === "redis" ? redis() : memoryStore(), {
|
|
21
|
+
prefix: "__APP_NAME__",
|
|
22
|
+
onError: (error, key) => logger.warn({ err: error, key }, "cache unavailable, computing instead"),
|
|
23
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { clientIp, type RateLimitOptions, rateLimit } from "@tatara/http";
|
|
2
|
+
import { cache } from "#core/cache/index";
|
|
3
|
+
import { env } from "#core/config/env";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* `middleware: [limitRequests({ name: "contact", windowMs: 60_000, max: 5 })]` on a route: per
|
|
7
|
+
* caller IP, or per whatever `key` returns (e.g. the signed-in user).
|
|
8
|
+
*/
|
|
9
|
+
export function limitRequests(options: Omit<RateLimitOptions, "counter" | "key"> & Partial<Pick<RateLimitOptions, "key">>) {
|
|
10
|
+
return rateLimit({
|
|
11
|
+
counter: cache,
|
|
12
|
+
key: (c) => clientIp(c, { trustedHops: env.TRUSTED_PROXY_HOPS }),
|
|
13
|
+
...options,
|
|
14
|
+
});
|
|
15
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { orgKey } from "@tatara/cache";
|
|
2
|
+
import { describe, expect, it, vi } from "vitest";
|
|
3
|
+
import { cache } from "#core/cache/index";
|
|
4
|
+
|
|
5
|
+
describe("cache", () => {
|
|
6
|
+
it("wraps a computation and drops an org's keys together", async () => {
|
|
7
|
+
const orgId = crypto.randomUUID();
|
|
8
|
+
const load = vi.fn(async () => ({ total: 3 }));
|
|
9
|
+
expect(await cache.wrap(orgKey(orgId, "dashboard"), 60_000, load)).toEqual({ total: 3 });
|
|
10
|
+
expect(await cache.wrap(orgKey(orgId, "dashboard"), 60_000, load)).toEqual({ total: 3 });
|
|
11
|
+
expect(load).toHaveBeenCalledTimes(1);
|
|
12
|
+
expect(await cache.delPrefix(orgKey(orgId))).toBe(1);
|
|
13
|
+
expect(await cache.get(orgKey(orgId, "dashboard"))).toBeNull();
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it("lets one holder run under a lock", async () => {
|
|
17
|
+
const key = crypto.randomUUID();
|
|
18
|
+
const first = cache.lock(key, 10_000, () => new Promise((r) => setTimeout(() => r("ran"), 50)));
|
|
19
|
+
expect(await cache.lock(key, 10_000, async () => "second")).toEqual({ acquired: false });
|
|
20
|
+
expect(await first).toEqual({ acquired: true, value: "ran" });
|
|
21
|
+
});
|
|
22
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { createRoute } from "@hono/zod-openapi";
|
|
2
|
+
import { createRouter, handleErrors } from "@tatara/http";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
import { limitRequests } from "#core/http/rate-limit";
|
|
5
|
+
|
|
6
|
+
describe("limitRequests", () => {
|
|
7
|
+
it("answers 429 once a caller passes the limit", async () => {
|
|
8
|
+
const name = crypto.randomUUID();
|
|
9
|
+
const app = createRouter();
|
|
10
|
+
app.openapi(
|
|
11
|
+
createRoute({
|
|
12
|
+
method: "post",
|
|
13
|
+
path: "/contact",
|
|
14
|
+
middleware: [limitRequests({ name, windowMs: 60_000, max: 2, key: (c) => c.req.header("x-who") })],
|
|
15
|
+
responses: { 204: { description: "Sent" } },
|
|
16
|
+
}),
|
|
17
|
+
(c) => c.body(null, 204),
|
|
18
|
+
);
|
|
19
|
+
handleErrors(app);
|
|
20
|
+
const send = () => app.request("/contact", { method: "POST", headers: { "x-who": "a" } });
|
|
21
|
+
expect((await send()).status).toBe(204);
|
|
22
|
+
expect((await send()).status).toBe(204);
|
|
23
|
+
const limited = await send();
|
|
24
|
+
expect(limited.status).toBe(429);
|
|
25
|
+
expect(limited.headers.get("retry-after")).toBe("60");
|
|
26
|
+
});
|
|
27
|
+
});
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { onShutdown } from "@tatara/core";
|
|
2
|
+
import { type Cache, createCache, type CacheStore, memoryStore } from "@tatara/cache";
|
|
3
|
+
import { postgresStore } from "@tatara/cache/postgres";
|
|
4
|
+
import { redisStore } from "@tatara/cache/redis";
|
|
5
|
+
import { Redis } from "ioredis";
|
|
6
|
+
import pg from "pg";
|
|
7
|
+
import { env } from "#core/config/env";
|
|
8
|
+
import { logger } from "#core/logging/logger";
|
|
9
|
+
|
|
10
|
+
export type { Cache } from "@tatara/cache";
|
|
11
|
+
|
|
12
|
+
function redis(): CacheStore {
|
|
13
|
+
// connects on first command, so importing this file opens nothing
|
|
14
|
+
const client = new Redis(env.REDIS_URL, { lazyConnect: true, maxRetriesPerRequest: 3 });
|
|
15
|
+
onShutdown("cache", async () => {
|
|
16
|
+
await client.quit();
|
|
17
|
+
});
|
|
18
|
+
return redisStore(client);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function postgres(): CacheStore {
|
|
22
|
+
// an unlogged table in the "tatara" schema, created on first use
|
|
23
|
+
const pool = new pg.Pool({ connectionString: env.DATABASE_URL, max: 5 });
|
|
24
|
+
onShutdown("cache", async () => {
|
|
25
|
+
await pool.end();
|
|
26
|
+
});
|
|
27
|
+
return postgresStore(pool);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const stores = { redis, postgres, memory: memoryStore };
|
|
31
|
+
|
|
32
|
+
/** The app's cache; pick the store with CACHE_DRIVER. */
|
|
33
|
+
export const cache: Cache = createCache(stores[env.CACHE_DRIVER](), {
|
|
34
|
+
prefix: "__APP_NAME__",
|
|
35
|
+
onError: (error, key) => logger.warn({ err: error, key }, "cache unavailable, computing instead"),
|
|
36
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
import { datetime, int, mysqlTable, varchar } from "drizzle-orm/mysql-core";
|
|
3
|
+
|
|
4
|
+
/** Which "once" commands this database has run (written by `deploy`). */
|
|
5
|
+
export const tataraCommandRuns = mysqlTable("tatara_command_runs", {
|
|
6
|
+
name: varchar({ length: 255 }).primaryKey(),
|
|
7
|
+
phase: varchar({ length: 16 }).notNull(),
|
|
8
|
+
ranAt: datetime({ fsp: 3 }).notNull().default(sql`current_timestamp(3)`),
|
|
9
|
+
durationMs: int().notNull(),
|
|
10
|
+
});
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Which "once" commands this database has run (written by `deploy`).
|
|
2
|
+
model TataraCommandRun {
|
|
3
|
+
name String @id @db.VarChar(255)
|
|
4
|
+
phase String @db.VarChar(16)
|
|
5
|
+
ranAt DateTime @default(now()) @map("ran_at") @db.DateTime(3)
|
|
6
|
+
durationMs Int @map("duration_ms")
|
|
7
|
+
|
|
8
|
+
@@map("tatara_command_runs")
|
|
9
|
+
}
|
|
@@ -1,13 +1,33 @@
|
|
|
1
|
-
import { createMailer } from "@tatara/email";
|
|
1
|
+
import { createMailer, renderEmail } from "@tatara/email";
|
|
2
2
|
import { env } from "#core/config/env";
|
|
3
3
|
import { logger } from "#core/logging/logger";
|
|
4
|
+
import type { emails } from "#generated/emails";
|
|
4
5
|
|
|
5
6
|
export type { EmailMessage } from "@tatara/email";
|
|
6
7
|
|
|
7
|
-
/** Sends through SMTP (Mailpit locally) or logs it when EMAIL_DRIVER=console. */
|
|
8
|
+
/** Sends through SMTP (Mailpit locally) or Mailgun, or logs it when EMAIL_DRIVER=console. */
|
|
8
9
|
export const { send: sendEmail } = createMailer({
|
|
9
10
|
driver: env.EMAIL_DRIVER,
|
|
10
11
|
smtpUrl: env.SMTP_URL,
|
|
12
|
+
...(env.MAILGUN_API_KEY && env.MAILGUN_DOMAIN
|
|
13
|
+
? { mailgun: { apiKey: env.MAILGUN_API_KEY, domain: env.MAILGUN_DOMAIN, apiUrl: env.MAILGUN_API_URL } }
|
|
14
|
+
: {}),
|
|
11
15
|
from: env.EMAIL_FROM,
|
|
12
16
|
logger,
|
|
13
17
|
});
|
|
18
|
+
|
|
19
|
+
type Templates = { [E in (typeof emails)[number] as E["name"]]: Parameters<E["render"]>[0] };
|
|
20
|
+
|
|
21
|
+
/** Renders an `emails/*.ts` template in `language` and sends it. */
|
|
22
|
+
export async function sendTemplate<N extends keyof Templates & string>(
|
|
23
|
+
name: N,
|
|
24
|
+
to: string,
|
|
25
|
+
data: Templates[N],
|
|
26
|
+
language = "en",
|
|
27
|
+
): Promise<void> {
|
|
28
|
+
// loaded on first use: core doesn't import module code until it's needed
|
|
29
|
+
const { emails: all } = await import("#generated/emails");
|
|
30
|
+
const template = all.find((e) => e.name === name);
|
|
31
|
+
if (!template) throw new Error(`No email template named "${name}"`);
|
|
32
|
+
await sendEmail({ to, ...renderEmail(template, data, language) });
|
|
33
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { defineEmail } from "@tatara/email";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
export default defineEmail({
|
|
5
|
+
name: "example.welcome",
|
|
6
|
+
schema: z.object({ name: z.string().min(1) }),
|
|
7
|
+
render: ({ name }, language) =>
|
|
8
|
+
language === "ar"
|
|
9
|
+
? { subject: "أهلاً بك", text: `أهلاً ${name}` }
|
|
10
|
+
: { subject: "Welcome", text: `Hello ${name}, welcome aboard.` },
|
|
11
|
+
});
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { handleSignals } from "@tatara/core";
|
|
2
2
|
import { startCron } from "@tatara/jobs";
|
|
3
|
-
import { redis } from "#core/jobs/index";
|
|
4
3
|
import { logger } from "#core/logging/logger";
|
|
5
|
-
import {
|
|
4
|
+
import { scheduleRunner } from "#apps/cron/runner";
|
|
6
5
|
|
|
7
|
-
startCron({
|
|
6
|
+
startCron({ ...scheduleRunner, logger });
|
|
8
7
|
handleSignals(logger);
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { cache } from "#core/cache/index";
|
|
2
|
+
import { schedules } from "#generated/schedules";
|
|
3
|
+
// tatara:slot imports
|
|
4
|
+
|
|
5
|
+
/** What runs a schedule, wherever it's triggered from: the cron app, `schedule:run`, or HTTP. */
|
|
6
|
+
export const scheduleRunner = {
|
|
7
|
+
schedules,
|
|
8
|
+
// one run per tick across every process and trigger
|
|
9
|
+
lock: cache,
|
|
10
|
+
// tatara:slot options
|
|
11
|
+
};
|
|
@@ -1,16 +1,9 @@
|
|
|
1
1
|
import { handleSignals } from "@tatara/core";
|
|
2
2
|
import { startWorker } from "@tatara/jobs";
|
|
3
3
|
import { env } from "#core/config/env";
|
|
4
|
-
import {
|
|
4
|
+
import { jobs } from "#core/jobs/index";
|
|
5
5
|
import { logger } from "#core/logging/logger";
|
|
6
6
|
import { consumers } from "#generated/consumers";
|
|
7
7
|
|
|
8
|
-
startWorker({
|
|
9
|
-
queue: QUEUE_NAME,
|
|
10
|
-
prefix: env.JOBS_PREFIX,
|
|
11
|
-
redis: redis(),
|
|
12
|
-
concurrency: env.WORKER_CONCURRENCY,
|
|
13
|
-
consumers,
|
|
14
|
-
logger,
|
|
15
|
-
});
|
|
8
|
+
await startWorker({ driver: jobs.driver, consumers, concurrency: env.WORKER_CONCURRENCY, logger });
|
|
16
9
|
handleSignals(logger);
|
|
@@ -1,10 +1,41 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createJobs, inlineDriver, type JobDriver, lazyRedis, type TopicsOf } from "@tatara/jobs";
|
|
2
|
+
import { bullmqDriver } from "@tatara/jobs/bullmq";
|
|
3
|
+
import { kafkaDriver } from "@tatara/jobs/kafka";
|
|
4
|
+
import { Kafka } from "kafkajs";
|
|
2
5
|
import { env } from "#core/config/env";
|
|
6
|
+
import { logger } from "#core/logging/logger";
|
|
7
|
+
import type { consumers } from "#generated/consumers";
|
|
3
8
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
/** The shared Redis connection, opened on first use. */
|
|
9
|
+
/** The shared Redis connection for BullMQ, opened on first use. */
|
|
7
10
|
export const redis = lazyRedis(env.REDIS_URL);
|
|
8
11
|
|
|
9
|
-
|
|
10
|
-
|
|
12
|
+
function kafka(): JobDriver {
|
|
13
|
+
if (!env.KAFKA_BROKERS) throw new Error("JOBS_DRIVER=kafka needs KAFKA_BROKERS");
|
|
14
|
+
const client = new Kafka({
|
|
15
|
+
clientId: env.KAFKA_CLIENT_ID,
|
|
16
|
+
brokers: env.KAFKA_BROKERS.split(","),
|
|
17
|
+
ssl: env.KAFKA_SSL,
|
|
18
|
+
...(env.KAFKA_SASL_USERNAME && env.KAFKA_SASL_PASSWORD
|
|
19
|
+
? {
|
|
20
|
+
sasl: {
|
|
21
|
+
mechanism: "scram-sha-512",
|
|
22
|
+
username: env.KAFKA_SASL_USERNAME,
|
|
23
|
+
password: env.KAFKA_SASL_PASSWORD,
|
|
24
|
+
},
|
|
25
|
+
}
|
|
26
|
+
: {}),
|
|
27
|
+
});
|
|
28
|
+
return kafkaDriver({ kafka: client, topic: "__APP_NAME__.jobs", groupId: env.KAFKA_CLIENT_ID });
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const drivers: Record<typeof env.JOBS_DRIVER, () => JobDriver> = {
|
|
32
|
+
bullmq: () => bullmqDriver({ name: "__APP_NAME__-jobs", prefix: env.JOBS_PREFIX, redis }),
|
|
33
|
+
kafka,
|
|
34
|
+
// jobs run where they're sent, with no worker: tests and quick local work
|
|
35
|
+
inline: () =>
|
|
36
|
+
inlineDriver({ consumers: async () => (await import("#generated/consumers")).consumers, logger }),
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/** `jobs.enqueue(topic, data, { orgId })`, typed by the app's consumers; pick the driver with JOBS_DRIVER. */
|
|
40
|
+
export const jobs = createJobs<TopicsOf<typeof consumers>>({ driver: drivers[env.JOBS_DRIVER]() });
|
|
41
|
+
export const { enqueue } = jobs;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { type InlineDriver, processJob } from "@tatara/jobs";
|
|
3
|
+
import { enqueue, jobs } from "#core/jobs/index";
|
|
4
4
|
import { logger } from "#core/logging/logger";
|
|
5
5
|
import { consumers } from "#generated/consumers";
|
|
6
6
|
|
|
@@ -17,13 +17,10 @@ describe("jobs", () => {
|
|
|
17
17
|
);
|
|
18
18
|
});
|
|
19
19
|
|
|
20
|
-
it("
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
});
|
|
26
|
-
expect(outer).toBe(true);
|
|
27
|
-
expect(inner).toBe(false);
|
|
20
|
+
it("runs an enqueued job in tests (JOBS_DRIVER=inline)", async () => {
|
|
21
|
+
await enqueue("example.send-welcome", { name: "Ada" });
|
|
22
|
+
const driver = jobs.driver as InlineDriver;
|
|
23
|
+
await driver.drain();
|
|
24
|
+
expect(driver.dead).toEqual([]);
|
|
28
25
|
});
|
|
29
26
|
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type SQL, sql } from "drizzle-orm";
|
|
2
|
+
import type { Queryable } from "@tatara/jobs/postgres";
|
|
3
|
+
import { type Executor, pool } from "#core/db/client";
|
|
4
|
+
|
|
5
|
+
/** The app's own pool: the Postgres driver needs no connection of its own. */
|
|
6
|
+
export const outboxPool: Queryable = pool;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* `enqueue(topic, data, { executor: inTransaction(tx) })`: with JOBS_DRIVER=postgres the job is
|
|
10
|
+
* written in `tx`, so it exists exactly when the transaction commits.
|
|
11
|
+
*/
|
|
12
|
+
export function inTransaction(executor: Executor): Queryable {
|
|
13
|
+
return {
|
|
14
|
+
query(text, values = []) {
|
|
15
|
+
// $1, $2… become drizzle parameters
|
|
16
|
+
const parts = text.split(/\$(\d+)/);
|
|
17
|
+
let query: SQL = sql.raw(parts[0] ?? "");
|
|
18
|
+
for (let i = 1; i < parts.length; i += 2)
|
|
19
|
+
query = sql`${query}${values[Number(parts[i]) - 1]}${sql.raw(parts[i + 1] ?? "")}`;
|
|
20
|
+
return executor.execute(query);
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { createJobs, type TopicsOf } from "@tatara/jobs";
|
|
2
|
+
import { postgresDriver } from "@tatara/jobs/postgres";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
import { db } from "#core/db/client";
|
|
5
|
+
import { inTransaction, outboxPool } from "#core/jobs/outbox";
|
|
6
|
+
import type { consumers } from "#generated/consumers";
|
|
7
|
+
|
|
8
|
+
const jobs = createJobs<TopicsOf<typeof consumers>>({ driver: postgresDriver({ db: outboxPool }) });
|
|
9
|
+
|
|
10
|
+
async function stored(id: string): Promise<number> {
|
|
11
|
+
const result = (await outboxPool.query("select count(*)::int as n from tatara.jobs where id = $1", [id])) as {
|
|
12
|
+
rows: { n: number }[];
|
|
13
|
+
};
|
|
14
|
+
return result.rows[0]?.n ?? 0;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
describe("the Postgres job outbox", () => {
|
|
18
|
+
it("stores a job with the caller's transaction, and drops it on rollback", async () => {
|
|
19
|
+
let rolledBack = "";
|
|
20
|
+
await expect(
|
|
21
|
+
db.transaction(async (tx) => {
|
|
22
|
+
rolledBack = await jobs.enqueue("example.send-welcome", { name: "Ada" }, { executor: inTransaction(tx) });
|
|
23
|
+
throw new Error("rollback");
|
|
24
|
+
}),
|
|
25
|
+
).rejects.toThrow("rollback");
|
|
26
|
+
expect(await stored(rolledBack)).toBe(0);
|
|
27
|
+
|
|
28
|
+
const committed = await db.transaction(async (tx) =>
|
|
29
|
+
jobs.enqueue("example.send-welcome", { name: "Ada" }, { executor: inTransaction(tx) }),
|
|
30
|
+
);
|
|
31
|
+
expect(await stored(committed)).toBe(1);
|
|
32
|
+
await outboxPool.query("delete from tatara.jobs where id = $1", [committed]);
|
|
33
|
+
});
|
|
34
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { createJobs, inlineDriver, type JobDriver, lazyRedis, type TopicsOf } from "@tatara/jobs";
|
|
2
|
+
import { bullmqDriver } from "@tatara/jobs/bullmq";
|
|
3
|
+
import { kafkaDriver } from "@tatara/jobs/kafka";
|
|
4
|
+
import { postgresDriver } from "@tatara/jobs/postgres";
|
|
5
|
+
import { Kafka } from "kafkajs";
|
|
6
|
+
import { env } from "#core/config/env";
|
|
7
|
+
import { outboxPool } from "#core/jobs/outbox";
|
|
8
|
+
import { logger } from "#core/logging/logger";
|
|
9
|
+
import type { consumers } from "#generated/consumers";
|
|
10
|
+
|
|
11
|
+
export { inTransaction } from "#core/jobs/outbox";
|
|
12
|
+
|
|
13
|
+
/** The shared Redis connection for BullMQ, opened on first use. */
|
|
14
|
+
export const redis = lazyRedis(env.REDIS_URL);
|
|
15
|
+
|
|
16
|
+
function kafka(): JobDriver {
|
|
17
|
+
if (!env.KAFKA_BROKERS) throw new Error("JOBS_DRIVER=kafka needs KAFKA_BROKERS");
|
|
18
|
+
const client = new Kafka({
|
|
19
|
+
clientId: env.KAFKA_CLIENT_ID,
|
|
20
|
+
brokers: env.KAFKA_BROKERS.split(","),
|
|
21
|
+
ssl: env.KAFKA_SSL,
|
|
22
|
+
...(env.KAFKA_SASL_USERNAME && env.KAFKA_SASL_PASSWORD
|
|
23
|
+
? {
|
|
24
|
+
sasl: {
|
|
25
|
+
mechanism: "scram-sha-512",
|
|
26
|
+
username: env.KAFKA_SASL_USERNAME,
|
|
27
|
+
password: env.KAFKA_SASL_PASSWORD,
|
|
28
|
+
},
|
|
29
|
+
}
|
|
30
|
+
: {}),
|
|
31
|
+
});
|
|
32
|
+
return kafkaDriver({ kafka: client, topic: "__APP_NAME__.jobs", groupId: env.KAFKA_CLIENT_ID });
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const drivers: Record<typeof env.JOBS_DRIVER, () => JobDriver> = {
|
|
36
|
+
bullmq: () => bullmqDriver({ name: "__APP_NAME__-jobs", prefix: env.JOBS_PREFIX, redis }),
|
|
37
|
+
// a "tatara.jobs" table, claimed with SKIP LOCKED; enqueue with { executor: inTransaction(tx) }
|
|
38
|
+
postgres: () => postgresDriver({ db: outboxPool }),
|
|
39
|
+
kafka,
|
|
40
|
+
// jobs run where they're sent, with no worker: tests and quick local work
|
|
41
|
+
inline: () =>
|
|
42
|
+
inlineDriver({ consumers: async () => (await import("#generated/consumers")).consumers, logger }),
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** `jobs.enqueue(topic, data, { orgId })`, typed by the app's consumers; pick the driver with JOBS_DRIVER. */
|
|
46
|
+
export const jobs = createJobs<TopicsOf<typeof consumers>>({ driver: drivers[env.JOBS_DRIVER]() });
|
|
47
|
+
export const { enqueue } = jobs;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { onShutdown } from "@tatara/core";
|
|
2
|
+
import type { Queryable } from "@tatara/jobs/postgres";
|
|
3
|
+
import pg from "pg";
|
|
4
|
+
import { env } from "#core/config/env";
|
|
5
|
+
import type { Executor } from "#core/db/client";
|
|
6
|
+
|
|
7
|
+
/** A small pool for the Postgres job driver (Prisma keeps its own connections to itself). */
|
|
8
|
+
export const outboxPool: Queryable = new pg.Pool({ connectionString: env.DATABASE_URL, max: 5 });
|
|
9
|
+
onShutdown("jobs-pool", () => (outboxPool as pg.Pool).end());
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* `enqueue(topic, data, { executor: inTransaction(tx) })`: with JOBS_DRIVER=postgres the job is
|
|
13
|
+
* written in `tx`, so it exists exactly when the transaction commits.
|
|
14
|
+
*/
|
|
15
|
+
export function inTransaction(executor: Executor): Queryable {
|
|
16
|
+
return { query: (text, values = []) => executor.$executeRawUnsafe(text, ...values) };
|
|
17
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { createJobs, type TopicsOf } from "@tatara/jobs";
|
|
2
|
+
import { postgresDriver } from "@tatara/jobs/postgres";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
import { db } from "#core/db/client";
|
|
5
|
+
import { inTransaction, outboxPool } from "#core/jobs/outbox";
|
|
6
|
+
import type { consumers } from "#generated/consumers";
|
|
7
|
+
|
|
8
|
+
const jobs = createJobs<TopicsOf<typeof consumers>>({ driver: postgresDriver({ db: outboxPool }) });
|
|
9
|
+
|
|
10
|
+
async function stored(id: string): Promise<number> {
|
|
11
|
+
const result = (await outboxPool.query("select count(*)::int as n from tatara.jobs where id = $1", [id])) as {
|
|
12
|
+
rows: { n: number }[];
|
|
13
|
+
};
|
|
14
|
+
return result.rows[0]?.n ?? 0;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
describe("the Postgres job outbox", () => {
|
|
18
|
+
it("stores a job with the caller's transaction, and drops it on rollback", async () => {
|
|
19
|
+
let rolledBack = "";
|
|
20
|
+
await expect(
|
|
21
|
+
db.$transaction(async (tx) => {
|
|
22
|
+
rolledBack = await jobs.enqueue("example.send-welcome", { name: "Ada" }, { executor: inTransaction(tx) });
|
|
23
|
+
throw new Error("rollback");
|
|
24
|
+
}),
|
|
25
|
+
).rejects.toThrow("rollback");
|
|
26
|
+
expect(await stored(rolledBack)).toBe(0);
|
|
27
|
+
|
|
28
|
+
const committed = await db.$transaction(async (tx) =>
|
|
29
|
+
jobs.enqueue("example.send-welcome", { name: "Ada" }, { executor: inTransaction(tx) }),
|
|
30
|
+
);
|
|
31
|
+
expect(await stored(committed)).toBe(1);
|
|
32
|
+
await outboxPool.query("delete from tatara.jobs where id = $1", [committed]);
|
|
33
|
+
});
|
|
34
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { defineNotification } from "@tatara/notifications";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
export default defineNotification({
|
|
5
|
+
name: "example.welcome",
|
|
6
|
+
description: "Example notification: greets a new user",
|
|
7
|
+
schema: z.object({ name: z.string().min(1) }),
|
|
8
|
+
channels: ["in-app", "push"],
|
|
9
|
+
inApp: ({ name }, language) => ({
|
|
10
|
+
title: language === "ar" ? `أهلاً ${name}` : `Welcome, ${name}`,
|
|
11
|
+
url: "/",
|
|
12
|
+
}),
|
|
13
|
+
push: ({ name }) => ({ title: `Welcome, ${name}` }),
|
|
14
|
+
});
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { DeliverConsumer } from "@tatara/notifications";
|
|
2
|
+
import { consumers } from "#modules/notifications/services/notifier";
|
|
3
|
+
|
|
4
|
+
/** Stores one user's notification and sends it on each channel, once each. */
|
|
5
|
+
const deliver: DeliverConsumer = consumers[1];
|
|
6
|
+
export default deliver;
|