notifkit 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-omGPIBny.mjs +89 -0
- package/dist/bulk-C4hTcnLu.mjs +77 -0
- package/dist/index.d.mts +1160 -8886
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
- package/dist/main-B5BwjIMu.mjs +811 -0
- package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
- package/dist/main-Bes11Ad3.mjs +312 -0
- package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
- package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
- package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
- package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +21 -18
- package/scripts/create-admin.mjs +8 -1
- package/scripts/migrate.mjs +43 -0
- package/src/client.ts +15 -5
- package/src/config/index.ts +37 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +174 -7
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +64 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
package/src/db/bulk.ts
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { sql as drizzleSql } from "drizzle-orm";
|
|
2
|
+
import { deliveryOutbox, messageLogs } from "./schema.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Bulk writes for the two tables every delivered notification touches.
|
|
6
|
+
*
|
|
7
|
+
* Drizzle rebuilds the statement for each call and walks every parameter as it
|
|
8
|
+
* does, which for a few hundred rows per batch was the largest single CPU cost
|
|
9
|
+
* in a worker. These send one fixed statement over `unnest()`d arrays instead,
|
|
10
|
+
* so Postgres prepares it once and the client serialises a handful of arrays
|
|
11
|
+
* rather than thousands of parameters.
|
|
12
|
+
*
|
|
13
|
+
* Handles that are not postgres.js-backed Drizzle instances (tests, other
|
|
14
|
+
* drivers) fall back to the equivalent Drizzle query.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
interface RawClient {
|
|
18
|
+
unsafe: (query: string, params: unknown[], options?: { prepare?: boolean }) => Promise<unknown>;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function rawClient(db: any): RawClient | null {
|
|
22
|
+
const client = db?.$client;
|
|
23
|
+
return client && typeof client.unsafe === "function" ? client : null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface MessageLogRow {
|
|
27
|
+
projectId: string;
|
|
28
|
+
taskId: string;
|
|
29
|
+
providerMessageId: string | null;
|
|
30
|
+
channel: string;
|
|
31
|
+
attempt: number;
|
|
32
|
+
kind: string;
|
|
33
|
+
status: string;
|
|
34
|
+
templateId: string | null;
|
|
35
|
+
workflowInstanceId: string | null;
|
|
36
|
+
campaignId: string | null;
|
|
37
|
+
/** ISO time the event happened; the insert time when omitted. */
|
|
38
|
+
timestamp?: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const INSERT_MESSAGE_LOGS = `
|
|
42
|
+
INSERT INTO message_logs
|
|
43
|
+
(project_id, task_id, provider_message_id, channel, attempt, kind, status,
|
|
44
|
+
template_id, workflow_instance_id, campaign_id, "timestamp")
|
|
45
|
+
SELECT project_id::uuid, task_id, provider_message_id, channel::channel, attempt, kind, status,
|
|
46
|
+
template_id, workflow_instance_id::uuid, campaign_id, COALESCE(ts::timestamptz, now())
|
|
47
|
+
FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::int[], $6::text[], $7::text[],
|
|
48
|
+
$8::text[], $9::text[], $10::text[], $11::text[])
|
|
49
|
+
AS t(project_id, task_id, provider_message_id, channel, attempt, kind, status,
|
|
50
|
+
template_id, workflow_instance_id, campaign_id, ts)
|
|
51
|
+
ON CONFLICT DO NOTHING`;
|
|
52
|
+
|
|
53
|
+
/** Inserts delivery log rows, skipping any already recorded. */
|
|
54
|
+
export async function insertMessageLogs(db: any, rows: MessageLogRow[]): Promise<void> {
|
|
55
|
+
if (rows.length === 0) return;
|
|
56
|
+
const client = rawClient(db);
|
|
57
|
+
if (!client) {
|
|
58
|
+
await db
|
|
59
|
+
.insert(messageLogs)
|
|
60
|
+
.values(
|
|
61
|
+
rows.map(({ timestamp, ...r }) =>
|
|
62
|
+
timestamp ? { ...r, timestamp: new Date(timestamp) } : r,
|
|
63
|
+
),
|
|
64
|
+
)
|
|
65
|
+
.onConflictDoNothing();
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
await client.unsafe(
|
|
69
|
+
INSERT_MESSAGE_LOGS,
|
|
70
|
+
[
|
|
71
|
+
rows.map((r) => r.projectId),
|
|
72
|
+
rows.map((r) => r.taskId),
|
|
73
|
+
rows.map((r) => r.providerMessageId),
|
|
74
|
+
rows.map((r) => r.channel),
|
|
75
|
+
rows.map((r) => r.attempt),
|
|
76
|
+
rows.map((r) => r.kind),
|
|
77
|
+
rows.map((r) => r.status),
|
|
78
|
+
rows.map((r) => r.templateId),
|
|
79
|
+
rows.map((r) => r.workflowInstanceId),
|
|
80
|
+
rows.map((r) => r.campaignId),
|
|
81
|
+
rows.map((r) => r.timestamp ?? null),
|
|
82
|
+
],
|
|
83
|
+
{ prepare: true },
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export interface OutboxRow {
|
|
88
|
+
taskId: string;
|
|
89
|
+
channel: string;
|
|
90
|
+
destination: string;
|
|
91
|
+
providerMessageId: string | null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const UPSERT_OUTBOX = `
|
|
95
|
+
INSERT INTO delivery_outbox (task_id, channel, destination, provider_message_id)
|
|
96
|
+
SELECT task_id, channel::channel, destination, provider_message_id
|
|
97
|
+
FROM unnest($1::text[], $2::text[], $3::text[], $4::text[])
|
|
98
|
+
AS t(task_id, channel, destination, provider_message_id)
|
|
99
|
+
ON CONFLICT (task_id, channel, destination)
|
|
100
|
+
DO UPDATE SET provider_message_id = EXCLUDED.provider_message_id`;
|
|
101
|
+
|
|
102
|
+
/** Records the provider's message id for each sent task, inserting rows as needed. */
|
|
103
|
+
export async function upsertOutboxProviderIds(db: any, rows: OutboxRow[]): Promise<void> {
|
|
104
|
+
if (rows.length === 0) return;
|
|
105
|
+
// ON CONFLICT DO UPDATE refuses to touch one row twice in a statement, which
|
|
106
|
+
// a retried task landing in the same batch as its first attempt would do.
|
|
107
|
+
const byKey = new Map<string, OutboxRow>();
|
|
108
|
+
for (const r of rows) byKey.set(`${r.taskId}\u0000${r.channel}\u0000${r.destination}`, r);
|
|
109
|
+
const unique = [...byKey.values()];
|
|
110
|
+
|
|
111
|
+
const client = rawClient(db);
|
|
112
|
+
if (!client) {
|
|
113
|
+
await db
|
|
114
|
+
.insert(deliveryOutbox)
|
|
115
|
+
.values(unique)
|
|
116
|
+
.onConflictDoUpdate({
|
|
117
|
+
target: [deliveryOutbox.taskId, deliveryOutbox.channel, deliveryOutbox.destination],
|
|
118
|
+
set: { providerMessageId: drizzleSql`EXCLUDED.provider_message_id` },
|
|
119
|
+
});
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
await client.unsafe(
|
|
123
|
+
UPSERT_OUTBOX,
|
|
124
|
+
[
|
|
125
|
+
unique.map((r) => r.taskId),
|
|
126
|
+
unique.map((r) => r.channel),
|
|
127
|
+
unique.map((r) => r.destination),
|
|
128
|
+
unique.map((r) => r.providerMessageId),
|
|
129
|
+
],
|
|
130
|
+
{ prepare: true },
|
|
131
|
+
);
|
|
132
|
+
}
|
package/src/db/index.ts
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import postgres from "postgres";
|
|
2
2
|
import { drizzle, type PostgresJsDatabase } from "drizzle-orm/postgres-js";
|
|
3
|
-
import type { Logger } from "@/index.js";
|
|
3
|
+
import type { Logger } from "@/logger/index.js";
|
|
4
4
|
import * as schema from "./schema.js";
|
|
5
|
-
import { readBaseConfig } from "@/index.js";
|
|
5
|
+
import { readBaseConfig } from "@/config/index.js";
|
|
6
6
|
|
|
7
7
|
export type Sql = postgres.Sql;
|
|
8
8
|
export type Db = PostgresJsDatabase<typeof schema>;
|
|
9
9
|
|
|
10
10
|
export interface DatabaseOptions {
|
|
11
11
|
url: string;
|
|
12
|
+
/**
|
|
13
|
+
* Schema holding notifkit's tables. Defaults to `DB_SCHEMA` (`public`).
|
|
14
|
+
* Anything else is set as the connection's `search_path`.
|
|
15
|
+
*/
|
|
16
|
+
schema?: string;
|
|
12
17
|
applicationName?: string;
|
|
13
18
|
maxConnections?: number;
|
|
14
19
|
idleTimeoutSeconds?: number;
|
|
@@ -27,12 +32,14 @@ export interface DatabaseClients {
|
|
|
27
32
|
*/
|
|
28
33
|
export function createDatabase({
|
|
29
34
|
url,
|
|
35
|
+
schema: dbSchema,
|
|
30
36
|
applicationName = "notifkit",
|
|
31
37
|
maxConnections,
|
|
32
38
|
idleTimeoutSeconds = 30,
|
|
33
39
|
logger,
|
|
34
40
|
}: DatabaseOptions): DatabaseClients {
|
|
35
41
|
const finalMaxConnections = maxConnections ?? readBaseConfig().DB_MAX_CONNECTIONS;
|
|
42
|
+
const finalSchema = resolveSchema(dbSchema);
|
|
36
43
|
|
|
37
44
|
const sql = postgres(url, {
|
|
38
45
|
max: finalMaxConnections,
|
|
@@ -40,6 +47,9 @@ export function createDatabase({
|
|
|
40
47
|
connection: {
|
|
41
48
|
application_name: applicationName,
|
|
42
49
|
statement_timeout: 10000 as any, // prevent TS issues with postgres.js types
|
|
50
|
+
// Unqualified queries resolve against this schema only, so a same-named
|
|
51
|
+
// table in `public` can never be picked up by mistake.
|
|
52
|
+
...(finalSchema === DEFAULT_SCHEMA ? {} : { search_path: quoteIdentifier(finalSchema) }),
|
|
43
53
|
},
|
|
44
54
|
onnotice: (notice) => {
|
|
45
55
|
logger?.debug({ notice }, "postgres notice");
|
|
@@ -52,18 +62,175 @@ export function createDatabase({
|
|
|
52
62
|
}
|
|
53
63
|
|
|
54
64
|
import { migrate } from "drizzle-orm/postgres-js/migrator";
|
|
65
|
+
import { getTableName, is, sql as drizzleSql } from "drizzle-orm";
|
|
66
|
+
import { PgTable } from "drizzle-orm/pg-core";
|
|
55
67
|
import { fileURLToPath } from "url";
|
|
68
|
+
import os from "os";
|
|
56
69
|
import path from "path";
|
|
57
70
|
import fs from "fs";
|
|
58
71
|
|
|
59
|
-
|
|
72
|
+
const DEFAULT_SCHEMA = "public";
|
|
73
|
+
const SCHEMA_NAME = /^[a-z_][a-z0-9_]{0,62}$/;
|
|
74
|
+
const STATEMENT_BREAKPOINT = "--> statement-breakpoint";
|
|
75
|
+
|
|
76
|
+
export interface MigrationOptions {
|
|
77
|
+
/** Schema to create notifkit's objects in. Defaults to `DB_SCHEMA` (`public`). */
|
|
78
|
+
schema?: string;
|
|
79
|
+
/**
|
|
80
|
+
* Schema of drizzle's migration journal table. Defaults to drizzle's own
|
|
81
|
+
* `drizzle` for `public`, and to `schema` otherwise, so two installs in
|
|
82
|
+
* different schemas of one database never share a journal.
|
|
83
|
+
*/
|
|
84
|
+
migrationsSchema?: string;
|
|
85
|
+
/** Name of drizzle's migration journal table. Defaults to `__drizzle_migrations`. */
|
|
86
|
+
migrationsTable?: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function resolveSchema(schema?: string): string {
|
|
90
|
+
const name = schema ?? readBaseConfig().DB_SCHEMA;
|
|
91
|
+
if (!SCHEMA_NAME.test(name)) {
|
|
92
|
+
throw new Error(`Invalid database schema "${name}": must be a lowercase PostgreSQL identifier`);
|
|
93
|
+
}
|
|
94
|
+
return name;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function quoteIdentifier(name: string): string {
|
|
98
|
+
return `"${name}"`;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function packagedMigrationsFolder(): string {
|
|
60
102
|
const __filename = fileURLToPath(import.meta.url);
|
|
61
103
|
const __dirname = path.dirname(__filename);
|
|
62
104
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
105
|
+
const folder = path.resolve(__dirname, "../../drizzle");
|
|
106
|
+
return fs.existsSync(folder) ? folder : path.resolve(__dirname, "../drizzle");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Retarget one generated migration at `schema`. drizzle-kit qualifies enum
|
|
111
|
+
* types and foreign-key targets with `"public".` but leaves tables and indexes
|
|
112
|
+
* unqualified, so both halves have to move: the qualifier is rewritten, and
|
|
113
|
+
* `SET LOCAL search_path` places the unqualified objects. `SET LOCAL` ends with
|
|
114
|
+
* the migration transaction and never leaks into a pooled connection.
|
|
115
|
+
*
|
|
116
|
+
* Throws if `public` is still mentioned after the rewrite: a migration that
|
|
117
|
+
* spells the schema some other way would otherwise put objects into `public`
|
|
118
|
+
* without a word.
|
|
119
|
+
*/
|
|
120
|
+
export function qualifyMigrationSql(source: string, schema: string): string {
|
|
121
|
+
const rewritten = source.replaceAll(`"${DEFAULT_SCHEMA}".`, `${quoteIdentifier(schema)}.`);
|
|
122
|
+
if (/\bpublic\b/i.test(rewritten)) {
|
|
123
|
+
throw new Error(
|
|
124
|
+
`Migration still references the public schema after retargeting it to "${schema}"`,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
return `SET LOCAL search_path TO ${quoteIdentifier(schema)};\n${STATEMENT_BREAKPOINT}\n${rewritten}`;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Copy the journal and every migration it lists into `target`, retargeted at `schema`. */
|
|
131
|
+
function writeSchemaMigrations(source: string, target: string, schema: string): void {
|
|
132
|
+
const journalPath = path.join(source, "meta", "_journal.json");
|
|
133
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf8")) as {
|
|
134
|
+
entries: { tag: string }[];
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
fs.mkdirSync(path.join(target, "meta"));
|
|
138
|
+
fs.copyFileSync(journalPath, path.join(target, "meta", "_journal.json"));
|
|
139
|
+
for (const { tag } of journal.entries) {
|
|
140
|
+
const file = `${tag}.sql`;
|
|
141
|
+
const migration = fs.readFileSync(path.join(source, file), "utf8");
|
|
142
|
+
fs.writeFileSync(path.join(target, file), qualifyMigrationSql(migration, schema));
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export async function runMigrations(db: Db, options: MigrationOptions = {}) {
|
|
147
|
+
const migrationsFolder = packagedMigrationsFolder();
|
|
148
|
+
const schema = resolveSchema(options.schema);
|
|
149
|
+
const { migrationsTable } = options;
|
|
150
|
+
|
|
151
|
+
if (schema === DEFAULT_SCHEMA) {
|
|
152
|
+
await migrate(db, {
|
|
153
|
+
migrationsFolder,
|
|
154
|
+
migrationsTable,
|
|
155
|
+
migrationsSchema: options.migrationsSchema,
|
|
156
|
+
});
|
|
157
|
+
return;
|
|
66
158
|
}
|
|
67
159
|
|
|
68
|
-
await
|
|
160
|
+
await db.execute(drizzleSql`CREATE SCHEMA IF NOT EXISTS ${drizzleSql.identifier(schema)}`);
|
|
161
|
+
|
|
162
|
+
const retargeted = fs.mkdtempSync(path.join(os.tmpdir(), "notifkit-migrations-"));
|
|
163
|
+
try {
|
|
164
|
+
writeSchemaMigrations(migrationsFolder, retargeted, schema);
|
|
165
|
+
await migrate(db, {
|
|
166
|
+
migrationsFolder: retargeted,
|
|
167
|
+
migrationsTable,
|
|
168
|
+
migrationsSchema: options.migrationsSchema ?? schema,
|
|
169
|
+
});
|
|
170
|
+
} finally {
|
|
171
|
+
fs.rmSync(retargeted, { recursive: true, force: true });
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
await assertTablesInSchema(db, schema);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Names of every table notifkit's Drizzle schema defines, sorted. This is the
|
|
179
|
+
* same object the ORM queries at run time, so the list cannot drift from the
|
|
180
|
+
* tables notifkit actually uses.
|
|
181
|
+
*/
|
|
182
|
+
export function notifkitTableNames(): string[] {
|
|
183
|
+
return Object.values(schema as Record<string, unknown>)
|
|
184
|
+
.filter((value): value is PgTable => is(value, PgTable))
|
|
185
|
+
.map((table) => getTableName(table))
|
|
186
|
+
.sort();
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Fail if any notifkit table is missing from `targetSchema`. Run after
|
|
191
|
+
* migrating into a custom schema: if the retargeting ever stops working (the
|
|
192
|
+
* `search_path` is overridden, a migration is skipped because of a shared
|
|
193
|
+
* journal, ...), the tables land elsewhere or nowhere, and every runtime query
|
|
194
|
+
* would fail with "relation does not exist". Only the target schema is checked:
|
|
195
|
+
* the application sharing the database may well own `users` or `projects` in
|
|
196
|
+
* `public`, and that is not an error.
|
|
197
|
+
*/
|
|
198
|
+
export async function assertTablesInSchema(db: Db, targetSchema: string): Promise<void> {
|
|
199
|
+
const rows = await db.execute<{ table_name: string }>(
|
|
200
|
+
drizzleSql`SELECT table_name FROM information_schema.tables WHERE table_schema = ${targetSchema}`,
|
|
201
|
+
);
|
|
202
|
+
const present = new Set(rows.map((row) => row.table_name));
|
|
203
|
+
const missing = notifkitTableNames().filter((name) => !present.has(name));
|
|
204
|
+
if (missing.length > 0) {
|
|
205
|
+
throw new Error(
|
|
206
|
+
`Migrations finished, but schema "${targetSchema}" is missing notifkit tables: ${missing.join(", ")}`,
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Apply the bundled migrations to the database at `url` and close the
|
|
213
|
+
* connection, whether or not they succeed. `schema` defaults to `DB_SCHEMA`.
|
|
214
|
+
* This is what `notifkit-migrate` runs.
|
|
215
|
+
*/
|
|
216
|
+
export async function migrateDatabase({
|
|
217
|
+
url,
|
|
218
|
+
schema: dbSchema,
|
|
219
|
+
}: {
|
|
220
|
+
url: string;
|
|
221
|
+
schema?: string;
|
|
222
|
+
}): Promise<string> {
|
|
223
|
+
const targetSchema = resolveSchema(dbSchema);
|
|
224
|
+
const { db, sql } = createDatabase({
|
|
225
|
+
url,
|
|
226
|
+
schema: targetSchema,
|
|
227
|
+
applicationName: "notifkit-migrate",
|
|
228
|
+
maxConnections: 1,
|
|
229
|
+
});
|
|
230
|
+
try {
|
|
231
|
+
await runMigrations(db, { schema: targetSchema });
|
|
232
|
+
} finally {
|
|
233
|
+
await sql.end();
|
|
234
|
+
}
|
|
235
|
+
return targetSchema;
|
|
69
236
|
}
|
package/src/db/schema.ts
CHANGED
|
@@ -15,7 +15,6 @@ import {
|
|
|
15
15
|
integer,
|
|
16
16
|
} from "drizzle-orm/pg-core";
|
|
17
17
|
import { sql } from "drizzle-orm";
|
|
18
|
-
import { createInsertSchema, createSelectSchema } from "drizzle-zod";
|
|
19
18
|
|
|
20
19
|
export const channelEnum = pgEnum("channel", [
|
|
21
20
|
"email",
|
|
@@ -211,8 +210,8 @@ export const messageLogs = pgTable(
|
|
|
211
210
|
timestamp: timestamp("timestamp", { withTimezone: true }).defaultNow().notNull(),
|
|
212
211
|
},
|
|
213
212
|
(table) => ({
|
|
214
|
-
|
|
215
|
-
|
|
213
|
+
// No standalone project_id or task_id index: the composites below lead
|
|
214
|
+
// with project_id, and the unique constraint leads with task_id.
|
|
216
215
|
projectIdTaskIdIdx: index("message_logs_project_task_idx").on(table.projectId, table.taskId),
|
|
217
216
|
providerMsgIdx: index("provider_msg_idx").on(table.providerMessageId),
|
|
218
217
|
projectTimeIdx: index("msg_log_proj_time_idx").on(table.projectId, table.timestamp),
|
|
@@ -271,6 +270,7 @@ export const workflowStatusEnum = pgEnum("workflow_status", [
|
|
|
271
270
|
"running",
|
|
272
271
|
"completed",
|
|
273
272
|
"failed",
|
|
273
|
+
"canceled",
|
|
274
274
|
]);
|
|
275
275
|
|
|
276
276
|
export const workflowDefinitions = pgTable(
|
|
@@ -376,50 +376,3 @@ export const adminUsers = pgTable("admin_users", {
|
|
|
376
376
|
createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(),
|
|
377
377
|
updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(),
|
|
378
378
|
});
|
|
379
|
-
|
|
380
|
-
// ─── Generated Zod Schemas ──────────────────────────────────────────────────
|
|
381
|
-
|
|
382
|
-
export const insertAdminUserSchema = createInsertSchema(adminUsers);
|
|
383
|
-
export const selectAdminUserSchema = createSelectSchema(adminUsers);
|
|
384
|
-
|
|
385
|
-
export const insertProjectSchema = createInsertSchema(projects);
|
|
386
|
-
export const selectProjectSchema = createSelectSchema(projects);
|
|
387
|
-
|
|
388
|
-
export const insertProjectApiKeySchema = createInsertSchema(projectApiKeys);
|
|
389
|
-
export const selectProjectApiKeySchema = createSelectSchema(projectApiKeys);
|
|
390
|
-
|
|
391
|
-
export const insertUserSchema = createInsertSchema(users);
|
|
392
|
-
export const selectUserSchema = createSelectSchema(users);
|
|
393
|
-
|
|
394
|
-
export const insertUserSegmentSchema = createInsertSchema(userSegments);
|
|
395
|
-
export const selectUserSegmentSchema = createSelectSchema(userSegments);
|
|
396
|
-
|
|
397
|
-
export const insertUserContactSchema = createInsertSchema(userContacts);
|
|
398
|
-
export const selectUserContactSchema = createSelectSchema(userContacts);
|
|
399
|
-
|
|
400
|
-
export const insertUserChannelPreferenceSchema = createInsertSchema(userChannelPreferences);
|
|
401
|
-
export const selectUserChannelPreferenceSchema = createSelectSchema(userChannelPreferences);
|
|
402
|
-
|
|
403
|
-
export const insertUserTopicPreferenceSchema = createInsertSchema(userTopicPreferences);
|
|
404
|
-
export const selectUserTopicPreferenceSchema = createSelectSchema(userTopicPreferences);
|
|
405
|
-
|
|
406
|
-
export const insertMessageLogSchema = createInsertSchema(messageLogs);
|
|
407
|
-
export const selectMessageLogSchema = createSelectSchema(messageLogs);
|
|
408
|
-
|
|
409
|
-
export const insertSuppressionSchema = createInsertSchema(suppressions);
|
|
410
|
-
export const selectSuppressionSchema = createSelectSchema(suppressions);
|
|
411
|
-
|
|
412
|
-
export const insertWorkflowInstanceSchema = createInsertSchema(workflowInstances);
|
|
413
|
-
export const selectWorkflowInstanceSchema = createSelectSchema(workflowInstances);
|
|
414
|
-
|
|
415
|
-
export const insertWorkflowStepSchema = createInsertSchema(workflowSteps);
|
|
416
|
-
export const selectWorkflowStepSchema = createSelectSchema(workflowSteps);
|
|
417
|
-
|
|
418
|
-
export const insertWorkflowWaiterSchema = createInsertSchema(workflowWaiters);
|
|
419
|
-
export const selectWorkflowWaiterSchema = createSelectSchema(workflowWaiters);
|
|
420
|
-
|
|
421
|
-
export const insertDeliveryOutboxSchema = createInsertSchema(deliveryOutbox);
|
|
422
|
-
export const selectDeliveryOutboxSchema = createSelectSchema(deliveryOutbox);
|
|
423
|
-
|
|
424
|
-
export const insertScheduledPayloadSchema = createInsertSchema(scheduledPayloads);
|
|
425
|
-
export const selectScheduledPayloadSchema = createSelectSchema(scheduledPayloads);
|
package/src/idempotency/index.ts
CHANGED
|
@@ -6,10 +6,13 @@ export interface IdempotencyOptions {
|
|
|
6
6
|
ttlSeconds?: number;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
export type LeaseResult = "acquired" | "locked" | "completed";
|
|
10
|
+
|
|
9
11
|
/**
|
|
10
|
-
* SETNX-based idempotency guard.
|
|
11
|
-
*
|
|
12
|
-
*
|
|
12
|
+
* Two-phase SETNX-based idempotency guard.
|
|
13
|
+
* Supports:
|
|
14
|
+
* - Phase 1 (:lock): Short lease acquired at the start of message processing.
|
|
15
|
+
* - Phase 2 (:sent): Long-lived completion marker recorded when processing succeeds.
|
|
13
16
|
*/
|
|
14
17
|
export class IdempotencyGuard {
|
|
15
18
|
private readonly redis: Redis;
|
|
@@ -26,6 +29,52 @@ export class IdempotencyGuard {
|
|
|
26
29
|
return `${this.keyPrefix}:${id}`;
|
|
27
30
|
}
|
|
28
31
|
|
|
32
|
+
private lockKey(id: string): string {
|
|
33
|
+
return `${this.keyPrefix}:${id}:lock`;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
private completedKey(id: string): string {
|
|
37
|
+
return `${this.keyPrefix}:${id}:sent`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Two-phase lease acquisition.
|
|
42
|
+
* Atomically checks existing state and acquires lock if available.
|
|
43
|
+
* - Happy path (new message): single native SET NX command storing "L" (lock lease).
|
|
44
|
+
* - If already present: checks if "S" (sent/completed) or "L" (locked by another worker).
|
|
45
|
+
*/
|
|
46
|
+
async acquireLease(id: string, lockTtlSeconds: number = 30): Promise<LeaseResult> {
|
|
47
|
+
const mainKey = this.key(id);
|
|
48
|
+
const lockKey = this.lockKey(id);
|
|
49
|
+
|
|
50
|
+
// Fast path: attempt native atomic SET NX
|
|
51
|
+
const acquired = await this.redis.set(mainKey, "L", "EX", lockTtlSeconds, "NX");
|
|
52
|
+
if (acquired === "OK") {
|
|
53
|
+
return "acquired";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Key exists — check whether it's completed ("S" / "1" / :sent) or in-flight ("L" / :lock)
|
|
57
|
+
const [val, sent, lockVal] = await Promise.all([
|
|
58
|
+
this.redis.get(mainKey),
|
|
59
|
+
this.redis.get(this.completedKey(id)),
|
|
60
|
+
this.redis.get(lockKey),
|
|
61
|
+
]);
|
|
62
|
+
|
|
63
|
+
if (val === "S" || val === "1" || sent !== null) {
|
|
64
|
+
return "completed";
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (val === "L" || lockVal !== null) {
|
|
68
|
+
return "locked";
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Key may have expired between initial SET NX and GETs; retry once
|
|
72
|
+
const retryAcquired = await this.redis.set(mainKey, "L", "EX", lockTtlSeconds, "NX");
|
|
73
|
+
if (retryAcquired === "OK") return "acquired";
|
|
74
|
+
|
|
75
|
+
return "locked";
|
|
76
|
+
}
|
|
77
|
+
|
|
29
78
|
/** Atomically mark id as processed. Returns true on first call; false if already seen. */
|
|
30
79
|
async checkAndMark(id: string, customTtlSeconds?: number): Promise<boolean> {
|
|
31
80
|
const ttl = customTtlSeconds ?? this.ttlSeconds;
|
|
@@ -33,18 +82,23 @@ export class IdempotencyGuard {
|
|
|
33
82
|
return result === "OK";
|
|
34
83
|
}
|
|
35
84
|
|
|
36
|
-
/** Unconditionally mark id as processed (
|
|
85
|
+
/** Unconditionally mark id as processed (upgrades lock to completed "1" with full TTL). */
|
|
37
86
|
async markProcessed(id: string, customTtlSeconds?: number): Promise<void> {
|
|
38
87
|
const ttl = customTtlSeconds ?? this.ttlSeconds;
|
|
39
88
|
await this.redis.set(this.key(id), "1", "EX", ttl);
|
|
40
89
|
}
|
|
41
90
|
|
|
42
91
|
async isProcessed(id: string): Promise<boolean> {
|
|
43
|
-
|
|
92
|
+
const val = await this.redis.get(this.key(id));
|
|
93
|
+
return val !== null && val !== undefined;
|
|
44
94
|
}
|
|
45
95
|
|
|
46
96
|
/** Remove the idempotency marker (useful in tests or manual rollbacks). */
|
|
47
97
|
async unmark(id: string): Promise<void> {
|
|
48
|
-
await
|
|
98
|
+
await Promise.all([
|
|
99
|
+
this.redis.del(this.key(id)),
|
|
100
|
+
this.redis.del(this.lockKey(id)),
|
|
101
|
+
this.redis.del(this.completedKey(id)),
|
|
102
|
+
]);
|
|
49
103
|
}
|
|
50
104
|
}
|
package/src/logger/index.ts
CHANGED
|
@@ -46,15 +46,3 @@ export function createLogger({ name, level = "info", pretty, context }: LoggerOp
|
|
|
46
46
|
|
|
47
47
|
return pino(options);
|
|
48
48
|
}
|
|
49
|
-
|
|
50
|
-
export function withRequestId(logger: Logger, requestId: string): Logger {
|
|
51
|
-
return logger.child({ requestId });
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export function withContext(logger: Logger, context: Record<string, unknown>): Logger {
|
|
55
|
-
return logger.child(context);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
export function childLogger(logger: Logger, bindings: Record<string, unknown>): Logger {
|
|
59
|
-
return logger.child(bindings);
|
|
60
|
-
}
|