@restura/core 2.0.2 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -83,16 +83,20 @@ var EventManager = class {
83
83
  filter
84
84
  });
85
85
  }
86
- async fireActionFromDbTrigger(sqlMutationData, result) {
86
+ async fireActionFromDbTrigger(sqlMutationData, result, options) {
87
+ const handlerErrors = [];
87
88
  if (sqlMutationData.mutationType === "INSERT") {
88
- await this.fireInsertActions(sqlMutationData, result);
89
+ await this.fireInsertActions(sqlMutationData, result, handlerErrors);
89
90
  } else if (sqlMutationData.mutationType === "UPDATE") {
90
- await this.fireUpdateActions(sqlMutationData, result);
91
+ await this.fireUpdateActions(sqlMutationData, result, handlerErrors);
91
92
  } else if (sqlMutationData.mutationType === "DELETE") {
92
- await this.fireDeleteActions(sqlMutationData, result);
93
+ await this.fireDeleteActions(sqlMutationData, result, handlerErrors);
94
+ }
95
+ if (options?.rethrowHandlerErrors && handlerErrors.length) {
96
+ throw handlerErrors[0];
93
97
  }
94
98
  }
95
- async fireInsertActions(data, triggerResult) {
99
+ async fireInsertActions(data, triggerResult, handlerErrors) {
96
100
  await Bluebird.map(
97
101
  this.actionHandlers.DATABASE_ROW_INSERT,
98
102
  async ({ callback, filter }) => {
@@ -107,12 +111,13 @@ var EventManager = class {
107
111
  await callback(insertData, data.queryMetadata);
108
112
  } catch (error) {
109
113
  logger.error(`Error firing insert action for table ${triggerResult.table}`, error);
114
+ handlerErrors.push(error);
110
115
  }
111
116
  },
112
117
  { concurrency: 10 }
113
118
  );
114
119
  }
115
- async fireDeleteActions(data, triggerResult) {
120
+ async fireDeleteActions(data, triggerResult, handlerErrors) {
116
121
  await Bluebird.map(
117
122
  this.actionHandlers.DATABASE_ROW_DELETE,
118
123
  async ({ callback, filter }) => {
@@ -127,12 +132,13 @@ var EventManager = class {
127
132
  await callback(deleteData, data.queryMetadata);
128
133
  } catch (error) {
129
134
  logger.error(`Error firing delete action for table ${triggerResult.table}`, error);
135
+ handlerErrors.push(error);
130
136
  }
131
137
  },
132
138
  { concurrency: 10 }
133
139
  );
134
140
  }
135
- async fireUpdateActions(data, triggerResult) {
141
+ async fireUpdateActions(data, triggerResult, handlerErrors) {
136
142
  await Bluebird.map(
137
143
  this.actionHandlers.DATABASE_COLUMN_UPDATE,
138
144
  async ({ callback, filter }) => {
@@ -148,11 +154,53 @@ var EventManager = class {
148
154
  await callback(columnChangeData, data.queryMetadata);
149
155
  } catch (error) {
150
156
  logger.error(`Error firing update action for table ${triggerResult.table}`, error);
157
+ handlerErrors.push(error);
151
158
  }
152
159
  },
153
160
  { concurrency: 10 }
154
161
  );
155
162
  }
163
+ // A handler whose filter doesn't match the schema's notify config silently never fires — surface it at startup
164
+ validateHandlersAgainstSchema(schema, mode) {
165
+ if (mode === "off") return;
166
+ const problems = [];
167
+ const checkTable = (tableName, handlerKind) => {
168
+ if (!tableName) return void 0;
169
+ const table = schema.database.find((item) => item.name === tableName);
170
+ if (!table) {
171
+ problems.push(`${handlerKind} handler registered for unknown table "${tableName}"`);
172
+ return void 0;
173
+ }
174
+ if (!table.notify) {
175
+ problems.push(
176
+ `${handlerKind} handler registered for table "${tableName}" which has no notify config \u2014 it will never fire`
177
+ );
178
+ return void 0;
179
+ }
180
+ return table;
181
+ };
182
+ this.actionHandlers.DATABASE_ROW_INSERT.forEach(({ filter }) => checkTable(filter?.tableName, "Row-insert"));
183
+ this.actionHandlers.DATABASE_ROW_DELETE.forEach(({ filter }) => checkTable(filter?.tableName, "Row-delete"));
184
+ this.actionHandlers.DATABASE_COLUMN_UPDATE.forEach(({ filter }) => {
185
+ const table = checkTable(filter?.tableName, "Column-change");
186
+ if (!table || !filter || table.notify === "ALL") return;
187
+ const notifyColumns = table.notify.map((entry) => entry.replace(/^!/, ""));
188
+ for (const column of filter.columns) {
189
+ if (column === "*") continue;
190
+ if (!notifyColumns.includes(column)) {
191
+ problems.push(
192
+ `Column-change handler on "${filter.tableName}" filters column "${column}" which is not in the table's notify list \u2014 it will never fire`
193
+ );
194
+ }
195
+ }
196
+ });
197
+ if (!problems.length) return;
198
+ if (mode === "strict") {
199
+ throw new Error(`Notify handler validation failed:
200
+ - ${problems.join("\n- ")}`);
201
+ }
202
+ problems.forEach((problem) => logger.warn(`Notify handler validation: ${problem}`));
203
+ }
156
204
  hasHandlersForEventType(eventType, filter, triggerResult) {
157
205
  if (filter) {
158
206
  switch (eventType) {
@@ -433,15 +481,17 @@ var ResponseValidator = class _ResponseValidator {
433
481
  );
434
482
  return { validator: "any", isOptionalOrNullable: false };
435
483
  }
484
+ const isOptionalOrNullable = isNullable || column.roles.length > 0 || column.scopes.length > 0 || column.isNullable;
485
+ const columnType = column.type.toLowerCase();
486
+ if (columnType === "json" || columnType === "jsonb") {
487
+ return { validator: "object", isOptionalOrNullable };
488
+ }
436
489
  let validator = SqlUtils.convertDatabaseTypeToTypescript(
437
490
  column.type,
438
491
  column.value
439
492
  );
440
493
  if (!_ResponseValidator.validatorIsValidString(validator)) validator = this.parseValidationEnum(validator);
441
- return {
442
- validator,
443
- isOptionalOrNullable: isNullable || column.roles.length > 0 || column.scopes.length > 0 || column.isNullable
444
- };
494
+ return { validator, isOptionalOrNullable };
445
495
  }
446
496
  parseValidationEnum(validator) {
447
497
  let terms = validator.split("|");
@@ -1810,12 +1860,23 @@ var resturaConfigSchema = z3.object({
1810
1860
  customApiFolderPath: z3.string().default(process.cwd() + customApiFolderPath),
1811
1861
  generatedTypesPath: z3.string().default(process.cwd() + "/src/@types"),
1812
1862
  fileTempCachePath: z3.string().optional(),
1813
- scratchDatabaseSuffix: z3.string().optional()
1863
+ scratchDatabaseSuffix: z3.string().optional(),
1864
+ eventDelivery: z3.enum(["direct", "outbox"]).default("direct"),
1865
+ eventOutbox: z3.object({
1866
+ channel: z3.string().default("restura_outbox"),
1867
+ pollIntervalMs: z3.number().default(15e3),
1868
+ batchSize: z3.number().default(50),
1869
+ maxAttempts: z3.number().default(5),
1870
+ pruneAfterDays: z3.number().default(7)
1871
+ }).prefault({}),
1872
+ notifyValidation: z3.enum(["off", "warn", "strict"]).default("warn"),
1873
+ queryMetadataKeys: z3.union([z3.literal("ALL"), z3.array(z3.string())]).optional()
1814
1874
  });
1815
1875
 
1816
- // src/restura/sql/PsqlEngine.ts
1817
- import { ObjectUtils as ObjectUtils3 } from "@redskytech/core-utils";
1818
- import pg3 from "pg";
1876
+ // src/restura/sql/PsqlConnection.ts
1877
+ import crypto from "crypto";
1878
+ import { format as sqlFormat } from "sql-formatter";
1879
+ import { z as z4 } from "zod";
1819
1880
 
1820
1881
  // src/restura/sql/PsqlUtils.ts
1821
1882
  import format from "pg-format";
@@ -1906,6 +1967,140 @@ function toSqlLiteral(value) {
1906
1967
  return format.literal(value);
1907
1968
  }
1908
1969
 
1970
+ // src/restura/sql/PsqlConnection.ts
1971
+ var DEFAULT_QUERY_METADATA_KEYS = [
1972
+ "userId",
1973
+ "adminUserId",
1974
+ "isSystemUser",
1975
+ "role",
1976
+ "host",
1977
+ "ipAddress",
1978
+ "storeId",
1979
+ "storeCustomerId",
1980
+ "appTokenId",
1981
+ "requestId"
1982
+ ];
1983
+ var PsqlConnection = class _PsqlConnection {
1984
+ static queryMetadataKeys = DEFAULT_QUERY_METADATA_KEYS;
1985
+ static setQueryMetadataKeys(keys) {
1986
+ _PsqlConnection.queryMetadataKeys = keys;
1987
+ }
1988
+ instanceId;
1989
+ constructor(instanceId) {
1990
+ this.instanceId = instanceId || crypto.randomUUID();
1991
+ }
1992
+ buildQueryMetadata(requesterDetails) {
1993
+ if (_PsqlConnection.queryMetadataKeys === "ALL") {
1994
+ return { connectionInstanceId: this.instanceId, ...requesterDetails };
1995
+ }
1996
+ const meta = { connectionInstanceId: this.instanceId };
1997
+ const details = requesterDetails ?? {};
1998
+ for (const key of _PsqlConnection.queryMetadataKeys) {
1999
+ if (details[key] !== void 0) meta[key] = details[key];
2000
+ }
2001
+ return meta;
2002
+ }
2003
+ async queryOne(query, options, requesterDetails) {
2004
+ const formattedQuery = questionMarksToOrderedParams(query);
2005
+ const meta = this.buildQueryMetadata(requesterDetails);
2006
+ const queryMetadata = `--QUERY_METADATA(${JSON.stringify(meta)})
2007
+ `;
2008
+ const startTime = process.hrtime();
2009
+ try {
2010
+ const response = await this.query(queryMetadata + formattedQuery, options);
2011
+ if (response.rows.length === 0) throw new RsError("NOT_FOUND", "No results found");
2012
+ else if (response.rows.length > 1) throw new RsError("DUPLICATE", "More than one result found");
2013
+ this.logSqlStatement(formattedQuery, options, meta, startTime);
2014
+ return response.rows[0];
2015
+ } catch (error) {
2016
+ this.logSqlStatement(formattedQuery, options, meta, startTime);
2017
+ if (RsError.isRsError(error)) throw error;
2018
+ if (error?.routine === "_bt_check_unique") {
2019
+ throw new RsError("DUPLICATE", error.message);
2020
+ }
2021
+ throw new RsError("DATABASE_ERROR", `${error.message}`);
2022
+ }
2023
+ }
2024
+ async queryOneSchema(query, params, requesterDetails, zodSchema) {
2025
+ const result = await this.queryOne(query, params, requesterDetails);
2026
+ try {
2027
+ return zodSchema.parse(result);
2028
+ } catch (error) {
2029
+ if (error instanceof z4.ZodError) {
2030
+ logger.error("Invalid data returned from database:");
2031
+ logger.trace("\n" + JSON.stringify(result, null, 2));
2032
+ logger.error("\n" + z4.prettifyError(error));
2033
+ } else {
2034
+ logger.error(error);
2035
+ }
2036
+ throw new RsError("DATABASE_ERROR", `Invalid data returned from database`);
2037
+ }
2038
+ }
2039
+ async runQuery(query, options, requesterDetails) {
2040
+ const formattedQuery = questionMarksToOrderedParams(query);
2041
+ const meta = this.buildQueryMetadata(requesterDetails);
2042
+ const queryMetadata = `--QUERY_METADATA(${JSON.stringify(meta)})
2043
+ `;
2044
+ const startTime = process.hrtime();
2045
+ try {
2046
+ const response = await this.query(queryMetadata + formattedQuery, options);
2047
+ this.logSqlStatement(formattedQuery, options, meta, startTime);
2048
+ return response.rows;
2049
+ } catch (error) {
2050
+ this.logSqlStatement(formattedQuery, options, meta, startTime);
2051
+ if (error?.routine === "_bt_check_unique") {
2052
+ throw new RsError("DUPLICATE", error.message);
2053
+ }
2054
+ throw new RsError("DATABASE_ERROR", `${error.message}`);
2055
+ }
2056
+ }
2057
+ async runQuerySchema(query, params, requesterDetails, zodSchema) {
2058
+ const result = await this.runQuery(query, params, requesterDetails);
2059
+ try {
2060
+ return z4.array(zodSchema).parse(result);
2061
+ } catch (error) {
2062
+ if (error instanceof z4.ZodError) {
2063
+ logger.error("Invalid data returned from database:");
2064
+ logger.trace("\n" + JSON.stringify(result, null, 2));
2065
+ logger.error("\n" + z4.prettifyError(error));
2066
+ } else {
2067
+ logger.error(error);
2068
+ }
2069
+ throw new RsError("DATABASE_ERROR", `Invalid data returned from database`);
2070
+ }
2071
+ }
2072
+ logSqlStatement(query, options, queryMetadata, startTime, prefix = "") {
2073
+ if (logger.level !== "trace") return;
2074
+ const sqlStatement = query.replace(/\$(\d+)/g, (_, num) => {
2075
+ const paramIndex = parseInt(num) - 1;
2076
+ if (paramIndex >= options.length) return "INVALID_PARAM_INDEX";
2077
+ return toSqlLiteral(options[paramIndex]);
2078
+ });
2079
+ const formattedSql = sqlFormat(sqlStatement, {
2080
+ language: "postgresql",
2081
+ linesBetweenQueries: 2,
2082
+ indentStyle: "standard",
2083
+ keywordCase: "upper",
2084
+ useTabs: true,
2085
+ tabWidth: 4
2086
+ });
2087
+ const [seconds, nanoseconds] = process.hrtime(startTime);
2088
+ const durationMs = seconds * 1e3 + nanoseconds / 1e6;
2089
+ let initiator = "Anonymous";
2090
+ const initiatorId = queryMetadata.userId ?? queryMetadata.adminUserId;
2091
+ if (initiatorId) initiator = `User Id (${initiatorId.toString()})`;
2092
+ if ("isSystemUser" in queryMetadata && queryMetadata.isSystemUser) initiator = "SYSTEM";
2093
+ logger.trace(`${prefix}query by ${initiator}, Query ->
2094
+ ${formattedSql}`, {
2095
+ durationMs
2096
+ });
2097
+ }
2098
+ };
2099
+
2100
+ // src/restura/sql/PsqlEngine.ts
2101
+ import { ObjectUtils as ObjectUtils3 } from "@redskytech/core-utils";
2102
+ import pg4 from "pg";
2103
+
1909
2104
  // src/restura/sql/SqlEngine.ts
1910
2105
  import { ObjectUtils as ObjectUtils2 } from "@redskytech/core-utils";
1911
2106
  var SqlEngine = class {
@@ -2035,893 +2230,931 @@ var SqlEngine = class {
2035
2230
  }
2036
2231
  };
2037
2232
 
2038
- // src/restura/sql/filterPsqlParser.ts
2039
- import peg from "pegjs";
2040
- var initializers = `
2041
- // Quotes a SQL identifier (column/table name) with double quotes, escaping any embedded quotes
2042
- function quoteSqlIdentity(value) {
2043
- return '"' + value.replace(/"/g, '""') + '"';
2044
- }
2045
-
2046
- // Unescape special characters in values: \\, -> , | \\| -> | | \\\\ -> \\
2047
- function unescapeValue(str) {
2048
- var result = '';
2049
- for (var i = 0; i < str.length; i++) {
2050
- if (str[i] === '\\\\' && i + 1 < str.length) {
2051
- var next = str[i + 1];
2052
- if (next === ',' || next === '|' || next === '\\\\') {
2053
- result += next;
2054
- i++;
2055
- } else {
2056
- result += str[i];
2057
- }
2058
- } else {
2059
- result += str[i];
2060
- }
2061
- }
2062
- return result;
2233
+ // src/restura/sql/PsqlTransaction.ts
2234
+ import pg from "pg";
2235
+ var { Client } = pg;
2236
+ var PsqlTransaction = class extends PsqlConnection {
2237
+ constructor(clientConfig, instanceId) {
2238
+ super(instanceId);
2239
+ this.clientConfig = clientConfig;
2240
+ this.client = new Client(clientConfig);
2241
+ this.connectPromise = this.client.connect();
2242
+ this.beginTransactionPromise = this.beginTransaction();
2243
+ }
2244
+ client;
2245
+ beginTransactionPromise;
2246
+ connectPromise;
2247
+ async close() {
2248
+ if (this.client) {
2249
+ await this.client.end();
2063
2250
  }
2251
+ }
2252
+ async beginTransaction() {
2253
+ await this.connectPromise;
2254
+ return this.client.query("BEGIN");
2255
+ }
2256
+ async rollback() {
2257
+ return this.query("ROLLBACK");
2258
+ }
2259
+ async commit() {
2260
+ return this.query("COMMIT");
2261
+ }
2262
+ async release() {
2263
+ return this.client.end();
2264
+ }
2265
+ async query(query, values) {
2266
+ await this.connectPromise;
2267
+ await this.beginTransactionPromise;
2268
+ return this.client.query(query, values);
2269
+ }
2270
+ };
2064
2271
 
2065
- // Split pipe-separated values respecting escaped pipes
2066
- function splitPipeValues(str) {
2067
- var values = [];
2068
- var current = '';
2069
- for (var i = 0; i < str.length; i++) {
2070
- if (str[i] === '\\\\' && i + 1 < str.length && str[i + 1] === '|') {
2071
- current += '|';
2072
- i++;
2073
- } else if (str[i] === '|') {
2074
- values.push(unescapeValue(current));
2075
- current = '';
2076
- } else {
2077
- current += str[i];
2078
- }
2079
- }
2080
- if (current.length > 0) {
2081
- values.push(unescapeValue(current));
2082
- }
2083
- return values;
2084
- }
2272
+ // src/restura/sql/psqlSchemaUtils.ts
2273
+ import getDiff from "@wmfs/pg-diff-sync";
2274
+ import pgInfo from "@wmfs/pg-info";
2275
+ import pg3 from "pg";
2085
2276
 
2086
- // Build SQL IN clause from pipe-separated values
2087
- function buildInClause(column, rawValue) {
2088
- var values = splitPipeValues(rawValue);
2089
- var literals = values.map(function(v) { return formatValue(v); });
2090
- return column + ' IN (' + literals.join(', ') + ')';
2277
+ // src/restura/sql/PsqlPool.ts
2278
+ import pg2 from "pg";
2279
+ var { Pool } = pg2;
2280
+ var PsqlPool = class extends PsqlConnection {
2281
+ constructor(poolConfig) {
2282
+ super();
2283
+ this.poolConfig = poolConfig;
2284
+ if (poolConfig.connectionString) {
2285
+ let url;
2286
+ try {
2287
+ url = new URL(poolConfig.connectionString);
2288
+ } catch {
2289
+ throw new Error(`Invalid connectionString: ${poolConfig.connectionString}`);
2290
+ }
2291
+ poolConfig.host = url.hostname;
2292
+ poolConfig.port = url.port ? parseInt(url.port) : 5432;
2293
+ poolConfig.user = url.username;
2294
+ poolConfig.password = url.password;
2295
+ poolConfig.database = url.pathname.replace(/^\//, "");
2091
2296
  }
2297
+ this.pool = new Pool(poolConfig);
2298
+ this.queryOne("SELECT NOW();", [], {
2299
+ isSystemUser: true,
2300
+ role: "",
2301
+ host: "localhost",
2302
+ ipAddress: "",
2303
+ scopes: []
2304
+ }).then(() => {
2305
+ logger.info("Connected to PostgreSQL database");
2306
+ }).catch((error) => {
2307
+ logger.error("Error connecting to database", error);
2308
+ process.exit(1);
2309
+ });
2310
+ }
2311
+ pool;
2312
+ async query(query, values) {
2313
+ return this.pool.query(query, values);
2314
+ }
2315
+ };
2092
2316
 
2093
- // Check if a value is numeric and format appropriately
2094
- function formatValue(value) {
2095
- // Check if the value is a valid number (integer or decimal)
2096
- if (/^-?\\d+(\\.\\d+)?$/.test(value)) {
2097
- return value; // Return as-is without quotes
2098
- }
2099
- return format.literal(value);
2100
- }
2101
-
2102
- // Format a value with optional type cast
2103
- function formatValueWithCast(rawValue, cast) {
2104
- var formatted = formatValue(unescapeValue(rawValue));
2105
- return cast ? formatted + '::' + cast : formatted;
2106
- }
2107
-
2108
- // Build SQL IN clause from pipe-separated values with optional cast
2109
- function buildInClauseWithCast(column, rawValue, cast) {
2110
- var values = splitPipeValues(rawValue);
2111
- var literals = values.map(function(v) {
2112
- var formatted = formatValue(v);
2113
- return cast ? formatted + '::' + cast : formatted;
2114
- });
2115
- return column + ' IN (' + literals.join(', ') + ')';
2317
+ // src/restura/sql/psqlSchemaUtils.ts
2318
+ var { Client: Client2 } = pg3;
2319
+ var systemUser = {
2320
+ role: "",
2321
+ scopes: [],
2322
+ host: "",
2323
+ ipAddress: "",
2324
+ isSystemUser: true
2325
+ };
2326
+ function schemaToPsqlType(column) {
2327
+ if (column.hasAutoIncrement) return "BIGSERIAL";
2328
+ if (column.type === "ENUM") return "TEXT";
2329
+ if (column.type === "DATETIME") return "TIMESTAMPTZ";
2330
+ if (column.type === "MEDIUMINT") return "INT";
2331
+ return column.type;
2332
+ }
2333
+ var OUTBOX_TABLE_NAME = "dbEventOutbox";
2334
+ var DEFAULT_OUTBOX_CHANNEL = "restura_outbox";
2335
+ var SENSITIVE_COLUMN_PATTERN = /password|token|secret|credential|guid|key$/i;
2336
+ function isSensitiveColumnName(columnName) {
2337
+ return SENSITIVE_COLUMN_PATTERN.test(columnName);
2338
+ }
2339
+ function createOutboxTableSql() {
2340
+ return `
2341
+ CREATE TABLE IF NOT EXISTS "${OUTBOX_TABLE_NAME}"
2342
+ (
2343
+ "id" BIGSERIAL PRIMARY KEY,
2344
+ "createdOn" TIMESTAMPTZ NOT NULL DEFAULT now(),
2345
+ "tableName" TEXT NOT NULL,
2346
+ "operation" TEXT NOT NULL,
2347
+ "recordId" BIGINT NULL,
2348
+ "record" JSONB NULL,
2349
+ "previousRecord" JSONB NULL,
2350
+ "queryMetadata" JSONB NULL,
2351
+ "processedOn" TIMESTAMPTZ NULL,
2352
+ "attempts" INT NOT NULL DEFAULT 0,
2353
+ "nextAttemptOn" TIMESTAMPTZ NULL,
2354
+ "isDeadLetter" BOOLEAN NOT NULL DEFAULT FALSE
2355
+ );
2356
+ CREATE INDEX IF NOT EXISTS "${OUTBOX_TABLE_NAME}_unprocessed_index" ON "${OUTBOX_TABLE_NAME}" ("id") WHERE "processedOn" IS NULL AND "isDeadLetter" = FALSE;
2357
+ CREATE INDEX IF NOT EXISTS "${OUTBOX_TABLE_NAME}_processedOn_index" ON "${OUTBOX_TABLE_NAME}" ("processedOn") WHERE "processedOn" IS NOT NULL;
2358
+ `;
2359
+ }
2360
+ function resolveNotifyColumns(tableName, notify, tableColumns) {
2361
+ if (!notify) return [];
2362
+ if (notify === "ALL") {
2363
+ if (!tableColumns) {
2364
+ throw new Error(
2365
+ `notify: "ALL" on table "${tableName}" requires tableColumns so the sensitive-column denylist can be applied. Pass options.tableColumns or list columns explicitly.`
2366
+ );
2116
2367
  }
2117
-
2118
- // Format column with optional cast
2119
- function formatColumn(col) {
2120
- if (!col.cast) {
2121
- return col.sql;
2122
- }
2123
- // Wrap JSON field extractions in parentheses when casting
2124
- // because :: has higher precedence than ->>
2125
- if (col.isJsonField) {
2126
- return '(' + col.sql + ')::' + col.cast;
2127
- }
2128
- return col.sql + '::' + col.cast;
2368
+ return tableColumns.filter((column) => !isSensitiveColumnName(column));
2369
+ }
2370
+ return notify.map((entry) => {
2371
+ if (entry.startsWith("!")) return entry.slice(1);
2372
+ if (isSensitiveColumnName(entry)) {
2373
+ throw new Error(
2374
+ `Refusing to include sensitive column "${tableName}"."${entry}" in notify trigger payloads. Remove it from the notify list or prefix it with '!' to force-include it.`
2375
+ );
2129
2376
  }
2130
- `;
2131
- var entryGrammar = `
2132
- {
2133
- ${initializers}
2377
+ return entry;
2378
+ });
2134
2379
  }
2135
-
2136
- Start
2137
- = sql:StartOld { return { sql: sql, usedOldSyntax: true }; }
2138
- / sql:StartNew { return { sql: sql, usedOldSyntax: false }; }
2380
+ function sqlStringLiteral(value) {
2381
+ return value.replace(/'/g, "''");
2382
+ }
2383
+ function buildRowJson(rowVariable, columns) {
2384
+ return `jsonb_build_object(
2385
+ ${columns.map((column) => `'${sqlStringLiteral(column)}', ${rowVariable}."${column}"`).join(",\n ")}
2386
+ )`;
2387
+ }
2388
+ var QUERY_METADATA_DECLARE_BLOCK = `
2389
+ SELECT INTO query_metadata
2390
+ (regexp_match(
2391
+ current_query(),
2392
+ '^--QUERY_METADATA\\(({.*})', 'n'
2393
+ ))[1]::json;
2139
2394
  `;
2140
- var oldGrammar = `
2141
- StartOld
2142
- = OldExpressionList
2143
- _
2144
- = [ \\t\\r\\n]* // Matches spaces, tabs, and line breaks
2145
-
2146
- OldExpressionList
2147
- = leftExpression:OldExpression _ operator:OldOperator _ rightExpression:OldExpressionList
2148
- { return \`\${leftExpression} \${operator} \${rightExpression}\`;}
2149
- / OldExpression
2150
-
2151
- OldExpression
2152
- = negate:OldNegate? _ "(" _ "column" _ ":" column:OldColumn _ ","? _ value:OldValue? ","? _ type:OldType? _ ")"_
2153
- {return \`\${negate? " NOT " : ""}(\${type ? type(column, value) : (value == null ? \`\${column} IS NULL\` : \`\${column} = \${formatValue(value)}\`)})\`;}
2154
- /
2155
- negate:OldNegate?"("expression:OldExpressionList")" { return \`\${negate? " NOT " : ""}(\${expression})\`; }
2156
-
2157
- OldNegate
2158
- = "!"
2159
-
2160
- OldOperator
2161
- = "and"i / "or"i
2162
-
2163
- OldColumn
2164
- = first:OldColumnPart rest:("." OldColumnPart)* {
2165
- const partsArray = [first];
2166
- if (rest && rest.length > 0) {
2167
- partsArray.push(...rest.map(item => item[1]));
2168
- }
2169
-
2170
- if (partsArray.length > 3) {
2171
- throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
2172
- }
2173
-
2174
- if (partsArray.length === 1) {
2175
- return quoteSqlIdentity(partsArray[0]);
2176
- }
2177
- const tableName = quoteSqlIdentity(partsArray[0]);
2178
-
2179
- // If we only have two parts (table.column), use regular dot notation
2180
- if (partsArray.length === 2) {
2181
- return tableName + "." + quoteSqlIdentity(partsArray[1]);
2182
- }
2183
-
2184
- // For JSON paths (more than 2 parts), first part is a column, last part uses ->>
2185
- const jsonColumn = quoteSqlIdentity(partsArray[1]);
2186
- const lastPart = partsArray[partsArray.length - 1];
2187
- const escapedLast = lastPart.replace(/'/g, "''");
2188
- const result = tableName + "." + jsonColumn + "->>'" + escapedLast + "'";
2189
- return result;
2190
- }
2395
+ function buildTriggerFunctionSql(tableName, operation, notify, options) {
2396
+ if (!notify) return "";
2397
+ const columns = resolveNotifyColumns(tableName, notify, options?.tableColumns);
2398
+ const delivery = options?.delivery || "direct";
2399
+ const channel = options?.channel || DEFAULT_OUTBOX_CHANNEL;
2400
+ const tableNameLiteral = sqlStringLiteral(tableName);
2401
+ const channelLiteral = sqlStringLiteral(channel);
2402
+ const functionName = `notify_${tableName}_${operation}`;
2403
+ const rowVariable = operation === "delete" ? "OLD" : "NEW";
2404
+ let body;
2405
+ if (delivery === "outbox") {
2406
+ const recordJson = operation === "delete" ? "NULL" : buildRowJson("NEW", columns);
2407
+ const previousRecordJson = operation === "insert" ? "NULL" : buildRowJson("OLD", columns);
2408
+ body = ` INSERT INTO "${OUTBOX_TABLE_NAME}" ("tableName", "operation", "recordId", "record", "previousRecord", "queryMetadata")
2409
+ VALUES ('${tableNameLiteral}', '${operation.toUpperCase()}', ${rowVariable}.id, ${recordJson}, ${previousRecordJson}, query_metadata::jsonb)
2410
+ RETURNING "id" INTO outbox_id;
2191
2411
 
2192
- OldColumnPart
2193
- = text:[a-z0-9 \\t\\r\\n\\-_:@']i+ {
2194
- return text.join("");
2195
- }
2412
+ PERFORM pg_notify('${channelLiteral}', outbox_id::text);`;
2413
+ } else {
2414
+ const idField = operation === "insert" ? `'insertedId', NEW.id` : operation === "update" ? `'changedId', NEW.id` : `'deletedId', OLD.id`;
2415
+ const payloadFields = [`'table', '${tableNameLiteral}'`, `'queryMetadata', query_metadata`, idField];
2416
+ if (operation !== "delete") payloadFields.push(`'record', ${buildRowJson("NEW", columns)}`);
2417
+ if (operation !== "insert") payloadFields.push(`'previousRecord', ${buildRowJson("OLD", columns)}`);
2418
+ body = ` PERFORM pg_notify(
2419
+ '${operation}',
2420
+ json_build_object(
2421
+ ${payloadFields.join(",\n ")}
2422
+ )::text
2423
+ );`;
2424
+ }
2425
+ const outboxDeclare = delivery === "outbox" ? "\n outbox_id BIGINT;" : "";
2426
+ const legacyDrop = operation === "update" && tableName !== tableName.toLowerCase() ? `DROP TRIGGER IF EXISTS ${tableName}_update ON "${tableName}";
2427
+ ` : "";
2428
+ const updateGuard = operation === "update" ? "\n WHEN (OLD.* IS DISTINCT FROM NEW.*)" : "";
2429
+ return `
2430
+ CREATE OR REPLACE FUNCTION ${functionName}()
2431
+ RETURNS TRIGGER AS $$
2432
+ DECLARE
2433
+ query_metadata JSON;${outboxDeclare}
2434
+ BEGIN
2435
+ ${QUERY_METADATA_DECLARE_BLOCK}
2436
+ ${body}
2196
2437
 
2197
- OldText
2198
- = text:[a-z0-9 \\t\\r\\n\\-_:@'.]i+ {
2199
- return text.join("");
2200
- }
2438
+ RETURN ${rowVariable};
2439
+ END;
2440
+ $$ LANGUAGE plpgsql;
2201
2441
 
2202
- OldType
2203
- = "type" _ ":" _ type:OldTypeString {
2204
- return type;
2442
+ ${legacyDrop}CREATE OR REPLACE TRIGGER "${tableName}_${operation}"
2443
+ AFTER ${operation.toUpperCase()} ON "${tableName}"
2444
+ FOR EACH ROW${updateGuard}
2445
+ EXECUTE FUNCTION ${functionName}();
2446
+ `;
2447
+ }
2448
+ function createInsertTriggerSql(tableName, notify, options) {
2449
+ return buildTriggerFunctionSql(tableName, "insert", notify, options);
2450
+ }
2451
+ function createUpdateTriggerSql(tableName, notify, options) {
2452
+ return buildTriggerFunctionSql(tableName, "update", notify, options);
2453
+ }
2454
+ function createDeleteTriggerSql(tableName, notify, options) {
2455
+ return buildTriggerFunctionSql(tableName, "delete", notify, options);
2456
+ }
2457
+ function generateNotifyTriggersSql(schema, options) {
2458
+ const statements = [];
2459
+ const hasNotifyTables = schema.database.some((table) => table.notify);
2460
+ if (options?.eventDelivery === "outbox" && hasNotifyTables) {
2461
+ statements.push(createOutboxTableSql());
2462
+ }
2463
+ for (const table of schema.database) {
2464
+ if (!table.notify) continue;
2465
+ const triggerOptions = {
2466
+ delivery: options?.eventDelivery,
2467
+ channel: options?.outboxChannel,
2468
+ tableColumns: table.columns.map((column) => column.name)
2469
+ };
2470
+ statements.push(createInsertTriggerSql(table.name, table.notify, triggerOptions));
2471
+ statements.push(createUpdateTriggerSql(table.name, table.notify, triggerOptions));
2472
+ statements.push(createDeleteTriggerSql(table.name, table.notify, triggerOptions));
2473
+ }
2474
+ return statements;
2475
+ }
2476
+ function generateDatabaseSchemaFromSchema(schema, options) {
2477
+ const sqlStatements = [];
2478
+ const indexes = [];
2479
+ const triggers = generateNotifyTriggersSql(schema, options);
2480
+ for (const table of schema.database) {
2481
+ let sql = `CREATE TABLE "${table.name}"
2482
+ ( `;
2483
+ const tableColumns = [];
2484
+ for (const column of table.columns) {
2485
+ let columnSql = "";
2486
+ columnSql += ` "${column.name}" ${schemaToPsqlType(column)}`;
2487
+ let value = column.value;
2488
+ if (column.type === "JSON") value = "";
2489
+ if (column.type === "JSONB") value = "";
2490
+ if (column.type === "DECIMAL" && value) {
2491
+ value = value.replace("-", ",").replace(/['"]/g, "");
2492
+ }
2493
+ if (value && column.type !== "ENUM") {
2494
+ columnSql += `(${value})`;
2495
+ } else if (column.length) columnSql += `(${column.length})`;
2496
+ if (column.isPrimary) {
2497
+ columnSql += " PRIMARY KEY ";
2498
+ }
2499
+ if (column.isUnique) {
2500
+ columnSql += ` CONSTRAINT "${table.name}_${column.name}_unique_index" UNIQUE `;
2501
+ }
2502
+ if (column.isNullable) columnSql += " NULL";
2503
+ else columnSql += " NOT NULL";
2504
+ if (column.default) columnSql += ` DEFAULT ${column.default}`;
2505
+ if (value && column.type === "ENUM") {
2506
+ columnSql += ` CHECK ("${column.name}" IN (${value}))`;
2507
+ }
2508
+ tableColumns.push(columnSql);
2205
2509
  }
2206
-
2207
- OldTypeString
2208
- = text:"startsWith" { return function(column, value) { return \`\${column} ILIKE '\${format.literal(value).slice(1,-1)}%'\`; } }
2209
- / text:"endsWith" { return function(column, value) { return \`\${column} ILIKE '%\${format.literal(value).slice(1,-1)}'\`; } }
2210
- / text:"contains" { return function(column, value) { return \`\${column} ILIKE '%\${format.literal(value).slice(1,-1)}%'\`; } }
2211
- / text:"exact" { return function(column, value) { return \`\${column} = \${formatValue(value)}\`; } }
2212
- / text:"greaterThanEqual" { return function(column, value) { return \`\${column} >= \${formatValue(value)}\`; } }
2213
- / text:"greaterThan" { return function(column, value) { return \`\${column} > \${formatValue(value)}\`; } }
2214
- / text:"lessThanEqual" { return function(column, value) { return \`\${column} <= \${formatValue(value)}\`; } }
2215
- / text:"lessThan" { return function(column, value) { return \`\${column} < \${formatValue(value)}\`; } }
2216
- / text:"isNull" { return function(column, value) { return \`\${column} IS NULL\`; } }
2217
-
2218
- OldValue
2219
- = "value" _ ":" value:OldText {
2220
- return value;
2510
+ sql += tableColumns.join(", \n");
2511
+ for (const index of table.indexes) {
2512
+ if (!index.isPrimaryKey) {
2513
+ let unique = " ";
2514
+ if (index.isUnique) unique = "UNIQUE ";
2515
+ let indexSQL = ` CREATE ${unique}INDEX "${index.name}" ON "${table.name}"`;
2516
+ indexSQL += ` (${index.columns.map((item) => `"${item}" ${index.order}`).join(", ")})`;
2517
+ indexSQL += index.where ? ` WHERE ${index.where}` : "";
2518
+ indexSQL += ";";
2519
+ indexes.push(indexSQL);
2520
+ }
2221
2521
  }
2222
- `;
2223
- var newGrammar = `
2224
- StartNew
2225
- = ExpressionList
2226
-
2227
- ExpressionList
2228
- = left:Expression _ op:("and"i / "or"i) _ right:ExpressionList
2229
- { return left + ' ' + op.toUpperCase() + ' ' + right; }
2230
- / Expression
2231
-
2232
- Expression
2233
- = negate:"!"? _ "(" _ inner:SimpleExprList _ ")" _
2234
- { return (negate ? 'NOT ' : '') + '(' + inner + ')'; }
2235
- / SimpleExpr
2236
-
2237
- SimpleExprList
2238
- = left:SimpleExpr _ op:("and"i / "or"i) _ right:SimpleExprList
2239
- { return left + ' ' + op.toUpperCase() + ' ' + right; }
2240
- / SimpleExpr
2241
-
2242
- SimpleExpr
2243
- = negate:"!"? _ "(" _ col:Column _ "," _ op:OperatorWithValue _ ")" _
2244
- { return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
2245
- / negate:"!"? _ "(" _ col:Column _ "," _ op:NullOperator _ ")" _
2246
- { return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
2247
- / negate:"!"? _ "(" _ col:Column _ "," _ val:CastedValue _ ")" _
2248
- { return (negate ? 'NOT ' : '') + '(' + formatColumn(col) + ' = ' + formatValueWithCast(val.value, val.cast) + ')'; }
2249
-
2250
- Column
2251
- = first:ColPart rest:("." ColPart)* cast:TypeCast? {
2252
- const partsArray = [first];
2253
- if (rest && rest.length > 0) {
2254
- partsArray.push(...rest.map(item => item[1]));
2255
- }
2256
-
2257
- if (partsArray.length > 3) {
2258
- throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
2259
- }
2260
-
2261
- var sql;
2262
- var isJsonField = false;
2263
- if (partsArray.length === 1) {
2264
- sql = quoteSqlIdentity(partsArray[0]);
2265
- } else {
2266
- const tableName = quoteSqlIdentity(partsArray[0]);
2267
-
2268
- if (partsArray.length === 2) {
2269
- sql = tableName + '.' + quoteSqlIdentity(partsArray[1]);
2270
- } else {
2271
- const jsonColumn = quoteSqlIdentity(partsArray[1]);
2272
- const lastPart = partsArray[partsArray.length - 1];
2273
- const escapedLast = lastPart.replace(/'/g, "''");
2274
- sql = tableName + '.' + jsonColumn + "->>'" + escapedLast + "'";
2275
- isJsonField = true;
2276
- }
2277
- }
2278
-
2279
- return { sql: sql, cast: cast, isJsonField: isJsonField };
2522
+ sql += "\n);";
2523
+ sqlStatements.push(sql);
2524
+ }
2525
+ for (const table of schema.database) {
2526
+ if (!table.foreignKeys.length) continue;
2527
+ const sql = `ALTER TABLE "${table.name}" `;
2528
+ const constraints = [];
2529
+ for (const foreignKey of table.foreignKeys) {
2530
+ let constraint = ` ADD CONSTRAINT "${foreignKey.name}"
2531
+ FOREIGN KEY ("${foreignKey.column}") REFERENCES "${foreignKey.refTable}" ("${foreignKey.refColumn}")`;
2532
+ constraint += ` ON DELETE ${foreignKey.onDelete}`;
2533
+ constraint += ` ON UPDATE ${foreignKey.onUpdate}`;
2534
+ constraints.push(constraint);
2280
2535
  }
2281
-
2282
- ColPart
2283
- = chars:[a-zA-Z0-9_]+ { return chars.join(''); }
2284
-
2285
- NullOperator
2286
- = "notnull"i { return function(col) { return formatColumn(col) + ' IS NOT NULL'; }; }
2287
- / "null"i { return function(col) { return formatColumn(col) + ' IS NULL'; }; }
2288
-
2289
- OperatorWithValue
2290
- = "in"i _ "," _ vals:InValueList cast:TypeCast? { return function(col) {
2291
- var formattedCol = formatColumn(col);
2292
- var literals = vals.map(function(v) {
2293
- var unescaped = unescapeValue(v);
2294
- var formatted = formatValue(unescaped);
2295
- return cast ? formatted + '::' + cast : formatted;
2296
- });
2297
- return formattedCol + ' IN (' + literals.join(', ') + ')';
2298
- }; }
2299
- / "ne"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <> ' + formatValueWithCast(val.value, val.cast); }; }
2300
- / "gte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' >= ' + formatValueWithCast(val.value, val.cast); }; }
2301
- / "gt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' > ' + formatValueWithCast(val.value, val.cast); }; }
2302
- / "lte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <= ' + formatValueWithCast(val.value, val.cast); }; }
2303
- / "lt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' < ' + formatValueWithCast(val.value, val.cast); }; }
2304
- / "has"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
2305
- / "sw"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal(unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
2306
- / "ew"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value)); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
2307
-
2308
- InValueList
2309
- = first:InValue rest:("|" InValue)* {
2310
- var values = [first];
2311
- for (var i = 0; i < rest.length; i++) {
2312
- values.push(rest[i][1]);
2313
- }
2314
- return values;
2536
+ sqlStatements.push(sql + constraints.join(",\n") + ";");
2537
+ }
2538
+ for (const table of schema.database) {
2539
+ if (!table.checkConstraints.length) continue;
2540
+ const sql = `ALTER TABLE "${table.name}" `;
2541
+ const constraints = [];
2542
+ for (const check of table.checkConstraints) {
2543
+ const constraint = `ADD CONSTRAINT "${check.name}" CHECK (${check.check})`;
2544
+ constraints.push(constraint);
2315
2545
  }
2546
+ sqlStatements.push(sql + constraints.join(",\n") + ";");
2547
+ }
2548
+ sqlStatements.push(indexes.join("\n"));
2549
+ sqlStatements.push(triggers.join("\n"));
2550
+ return sqlStatements.join("\n\n");
2551
+ }
2552
+ async function getNewPublicSchemaAndScratchPool(targetPool, scratchDbName) {
2553
+ const scratchDbExists = await targetPool.runQuery(
2554
+ `SELECT * FROM pg_database WHERE datname = ?;`,
2555
+ [scratchDbName],
2556
+ systemUser
2557
+ );
2558
+ if (scratchDbExists.length === 0) {
2559
+ await targetPool.runQuery(`CREATE DATABASE ${escapeColumnName(scratchDbName)};`, [], systemUser);
2560
+ }
2561
+ const scratchPool = new PsqlPool({
2562
+ host: targetPool.poolConfig.host,
2563
+ port: targetPool.poolConfig.port,
2564
+ user: targetPool.poolConfig.user,
2565
+ database: scratchDbName,
2566
+ password: targetPool.poolConfig.password,
2567
+ max: targetPool.poolConfig.max,
2568
+ idleTimeoutMillis: targetPool.poolConfig.idleTimeoutMillis,
2569
+ connectionTimeoutMillis: targetPool.poolConfig.connectionTimeoutMillis
2570
+ });
2571
+ await scratchPool.runQuery(`DROP SCHEMA public CASCADE;`, [], systemUser);
2572
+ await scratchPool.runQuery(
2573
+ `CREATE SCHEMA public AUTHORIZATION ${escapeColumnName(targetPool.poolConfig.user)};`,
2574
+ [],
2575
+ systemUser
2576
+ );
2577
+ const schemaComment = await targetPool.runQuery(
2578
+ `
2579
+ SELECT pg_description.description
2580
+ FROM pg_description
2581
+ JOIN pg_namespace ON pg_namespace.oid = pg_description.objoid
2582
+ WHERE pg_namespace.nspname = 'public';`,
2583
+ [],
2584
+ systemUser
2585
+ );
2586
+ if (schemaComment[0]?.description) {
2587
+ const escaped = schemaComment[0].description.replace(/'/g, "''");
2588
+ await scratchPool.runQuery(`COMMENT ON SCHEMA public IS '${escaped}';`, [], systemUser);
2589
+ }
2590
+ return scratchPool;
2591
+ }
2592
+ async function diffDatabaseToSchema(schema, targetPool, scratchDbName, options) {
2593
+ let scratchPool;
2594
+ let originalClient;
2595
+ let scratchClient;
2596
+ try {
2597
+ scratchPool = await getNewPublicSchemaAndScratchPool(targetPool, scratchDbName);
2598
+ const sqlFullStatement = generateDatabaseSchemaFromSchema(schema, options);
2599
+ await scratchPool.runQuery(sqlFullStatement, [], systemUser);
2600
+ const connectionConfig = {
2601
+ host: targetPool.poolConfig.host,
2602
+ port: targetPool.poolConfig.port,
2603
+ user: targetPool.poolConfig.user,
2604
+ password: targetPool.poolConfig.password,
2605
+ ssl: targetPool.poolConfig.ssl
2606
+ };
2607
+ originalClient = new Client2({ ...connectionConfig, database: targetPool.poolConfig.database });
2608
+ scratchClient = new Client2({ ...connectionConfig, database: scratchDbName });
2609
+ await Promise.all([originalClient.connect(), scratchClient.connect()]);
2610
+ const [info1, info2] = await Promise.all([
2611
+ pgInfo({ client: originalClient }),
2612
+ pgInfo({ client: scratchClient })
2613
+ ]);
2614
+ const diff = getDiff(info1, info2);
2615
+ return diff.join("\n");
2616
+ } finally {
2617
+ const cleanups = [];
2618
+ if (originalClient) cleanups.push(originalClient.end());
2619
+ if (scratchClient) cleanups.push(scratchClient.end());
2620
+ if (scratchPool) cleanups.push(scratchPool.pool.end());
2621
+ await Promise.allSettled(cleanups);
2622
+ }
2623
+ }
2316
2624
 
2317
- InValue
2318
- = QuotedString
2319
- / chars:InValueChar+ { return chars.join(''); }
2320
-
2321
- InValueChar
2322
- = "\\\\\\\\" { return '\\\\\\\\'; }
2323
- / "\\\\," { return '\\\\,'; }
2324
- / "\\\\|" { return '\\\\|'; }
2325
- / [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" ]
2326
- / c:":" !":" { return c; }
2327
-
2328
- CastedValue
2329
- = val:Value cast:TypeCast? { return { value: val, cast: cast }; }
2330
-
2331
- CastedValueWithPipes
2332
- = val:ValueWithPipes cast:TypeCast? { return { value: val, cast: cast }; }
2333
-
2334
- TypeCast
2335
- = "::" type:("timestamptz"i / "timestamp"i / "boolean"i / "numeric"i / "bigint"i / "text"i / "date"i / "int"i)
2336
- { return type.toLowerCase(); }
2337
-
2338
- QuotedString
2339
- = '"' chars:DoubleQuotedChar* '"' { return chars.join(''); }
2340
- / "'" chars:SingleQuotedChar* "'" { return chars.join(''); }
2341
-
2342
- DoubleQuotedChar
2343
- = '\\\\"' { return '"'; }
2344
- / '\\\\\\\\' { return '\\\\'; }
2345
- / [^"\\\\]
2346
-
2347
- SingleQuotedChar
2348
- = "\\\\'" { return "'"; }
2349
- / '\\\\\\\\' { return '\\\\'; }
2350
- / [^'\\\\]
2351
-
2352
- Value
2353
- = QuotedString
2354
- / chars:ValueChar+ { return chars.join(''); }
2355
-
2356
- ValueChar
2357
- = "\\\\\\\\" { return '\\\\\\\\'; }
2358
- / "\\\\," { return '\\\\,'; }
2359
- / "\\\\|" { return '\\\\|'; }
2360
- / [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" ]
2361
- / c:":" !":" { return c; }
2362
-
2363
- ValueWithPipes
2364
- = QuotedString
2365
- / chars:ValueWithPipesChar+ { return chars.join(''); }
2366
-
2367
- ValueWithPipesChar
2368
- = "\\\\\\\\" { return '\\\\\\\\'; }
2369
- / "\\\\," { return '\\\\,'; }
2370
- / "\\\\|" { return '\\\\|'; }
2371
- / [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" |]
2372
- / c:":" !":" { return c; }
2373
- `;
2374
- var fullGrammar = entryGrammar + oldGrammar + newGrammar;
2375
- var filterPsqlParser = peg.generate(fullGrammar, {
2376
- format: "commonjs",
2377
- dependencies: { format: "pg-format" }
2378
- });
2379
- var filterPsqlParser_default = filterPsqlParser;
2380
-
2381
- // src/restura/sql/psqlSchemaUtils.ts
2382
- import getDiff from "@wmfs/pg-diff-sync";
2383
- import pgInfo from "@wmfs/pg-info";
2384
- import pg2 from "pg";
2385
-
2386
- // src/restura/sql/PsqlPool.ts
2387
- import pg from "pg";
2388
-
2389
- // src/restura/sql/PsqlConnection.ts
2390
- import crypto from "crypto";
2391
- import { format as sqlFormat } from "sql-formatter";
2392
- import { z as z4 } from "zod";
2393
- var PsqlConnection = class {
2394
- instanceId;
2395
- constructor(instanceId) {
2396
- this.instanceId = instanceId || crypto.randomUUID();
2625
+ // src/restura/sql/eventOutbox.ts
2626
+ var defaultEventOutboxOptions = {
2627
+ channel: DEFAULT_OUTBOX_CHANNEL,
2628
+ pollIntervalMs: 15e3,
2629
+ batchSize: 50,
2630
+ maxAttempts: 5,
2631
+ pruneAfterDays: 7
2632
+ };
2633
+ var PRUNE_INTERVAL_MS = 60 * 60 * 1e3;
2634
+ var EventOutboxConsumer = class {
2635
+ constructor(psqlConnectionPool, options) {
2636
+ this.psqlConnectionPool = psqlConnectionPool;
2637
+ this.options = { ...defaultEventOutboxOptions, ...options };
2397
2638
  }
2398
- async queryOne(query, options, requesterDetails) {
2399
- const formattedQuery = questionMarksToOrderedParams(query);
2400
- const meta = { connectionInstanceId: this.instanceId, ...requesterDetails };
2401
- const queryMetadata = `--QUERY_METADATA(${JSON.stringify(meta)})
2402
- `;
2403
- const startTime = process.hrtime();
2404
- try {
2405
- const response = await this.query(queryMetadata + formattedQuery, options);
2406
- if (response.rows.length === 0) throw new RsError("NOT_FOUND", "No results found");
2407
- else if (response.rows.length > 1) throw new RsError("DUPLICATE", "More than one result found");
2408
- this.logSqlStatement(formattedQuery, options, meta, startTime);
2409
- return response.rows[0];
2410
- } catch (error) {
2411
- this.logSqlStatement(formattedQuery, options, meta, startTime);
2412
- if (RsError.isRsError(error)) throw error;
2413
- if (error?.routine === "_bt_check_unique") {
2414
- throw new RsError("DUPLICATE", error.message);
2415
- }
2416
- throw new RsError("DATABASE_ERROR", `${error.message}`);
2639
+ options;
2640
+ isDraining = false;
2641
+ drainRequested = false;
2642
+ isStopped = false;
2643
+ pollTimer;
2644
+ pruneTimer;
2645
+ activeDrain = Promise.resolve();
2646
+ lastDrainOn = null;
2647
+ get channel() {
2648
+ return this.options.channel;
2649
+ }
2650
+ start() {
2651
+ this.isStopped = false;
2652
+ this.pollTimer = setInterval(() => void this.drain(), this.options.pollIntervalMs);
2653
+ this.pruneTimer = setInterval(() => void this.prune(), PRUNE_INTERVAL_MS);
2654
+ void this.drain();
2655
+ void this.prune();
2656
+ }
2657
+ async stop() {
2658
+ this.isStopped = true;
2659
+ if (this.pollTimer) clearInterval(this.pollTimer);
2660
+ if (this.pruneTimer) clearInterval(this.pruneTimer);
2661
+ await this.activeDrain;
2662
+ }
2663
+ /** Safe to call at any frequency; overlapping calls coalesce into one extra pass. */
2664
+ async drain() {
2665
+ if (this.isDraining) {
2666
+ this.drainRequested = true;
2667
+ return;
2417
2668
  }
2669
+ this.isDraining = true;
2670
+ this.activeDrain = (async () => {
2671
+ try {
2672
+ do {
2673
+ this.drainRequested = false;
2674
+ let processedCount = 0;
2675
+ do {
2676
+ processedCount = await this.processBatch();
2677
+ } while (processedCount > 0 && !this.isStopped);
2678
+ } while (this.drainRequested && !this.isStopped);
2679
+ this.lastDrainOn = (/* @__PURE__ */ new Date()).toISOString();
2680
+ } catch (error) {
2681
+ logger.error(`Event outbox drain failed: ${error}`);
2682
+ } finally {
2683
+ this.isDraining = false;
2684
+ }
2685
+ })();
2686
+ await this.activeDrain;
2418
2687
  }
2419
- async queryOneSchema(query, params, requesterDetails, zodSchema) {
2420
- const result = await this.queryOne(query, params, requesterDetails);
2688
+ async getStats() {
2689
+ const result = await this.psqlConnectionPool.runQuery(
2690
+ `SELECT
2691
+ COUNT(*) FILTER (WHERE "processedOn" IS NULL AND "isDeadLetter" = FALSE) AS "pendingCount",
2692
+ COUNT(*) FILTER (WHERE "isDeadLetter" = TRUE) AS "deadLetterCount"
2693
+ FROM "${OUTBOX_TABLE_NAME}";`,
2694
+ [],
2695
+ systemUser
2696
+ );
2697
+ return {
2698
+ pendingCount: Number(result[0]?.pendingCount || 0),
2699
+ deadLetterCount: Number(result[0]?.deadLetterCount || 0),
2700
+ lastDrainOn: this.lastDrainOn
2701
+ };
2702
+ }
2703
+ async processBatch() {
2704
+ const transaction = new PsqlTransaction({
2705
+ host: this.psqlConnectionPool.poolConfig.host,
2706
+ port: this.psqlConnectionPool.poolConfig.port,
2707
+ user: this.psqlConnectionPool.poolConfig.user,
2708
+ password: this.psqlConnectionPool.poolConfig.password,
2709
+ database: this.psqlConnectionPool.poolConfig.database,
2710
+ connectionTimeoutMillis: this.psqlConnectionPool.poolConfig.connectionTimeoutMillis
2711
+ });
2421
2712
  try {
2422
- return zodSchema.parse(result);
2713
+ const rows = await transaction.runQuery(
2714
+ `SELECT * FROM "${OUTBOX_TABLE_NAME}"
2715
+ WHERE "processedOn" IS NULL AND "isDeadLetter" = FALSE
2716
+ AND ("nextAttemptOn" IS NULL OR "nextAttemptOn" <= now())
2717
+ ORDER BY "id"
2718
+ LIMIT ?
2719
+ FOR UPDATE SKIP LOCKED;`,
2720
+ [this.options.batchSize],
2721
+ systemUser
2722
+ );
2723
+ for (const row of rows) {
2724
+ await this.processRow(transaction, row);
2725
+ }
2726
+ await transaction.commit();
2727
+ return rows.length;
2423
2728
  } catch (error) {
2424
- if (error instanceof z4.ZodError) {
2425
- logger.error("Invalid data returned from database:");
2426
- logger.trace("\n" + JSON.stringify(result, null, 2));
2427
- logger.error("\n" + z4.prettifyError(error));
2428
- } else {
2429
- logger.error(error);
2729
+ try {
2730
+ await transaction.rollback();
2731
+ } catch {
2430
2732
  }
2431
- throw new RsError("DATABASE_ERROR", `Invalid data returned from database`);
2733
+ throw error;
2734
+ } finally {
2735
+ await transaction.release();
2432
2736
  }
2433
2737
  }
2434
- async runQuery(query, options, requesterDetails) {
2435
- const formattedQuery = questionMarksToOrderedParams(query);
2436
- const meta = { connectionInstanceId: this.instanceId, ...requesterDetails };
2437
- const queryMetadata = `--QUERY_METADATA(${JSON.stringify(meta)})
2438
- `;
2439
- const startTime = process.hrtime();
2738
+ async processRow(transaction, row) {
2440
2739
  try {
2441
- const response = await this.query(queryMetadata + formattedQuery, options);
2442
- this.logSqlStatement(formattedQuery, options, meta, startTime);
2443
- return response.rows;
2740
+ const triggerResult = {
2741
+ table: row.tableName,
2742
+ insertedId: row.operation === "INSERT" ? row.recordId ?? void 0 : void 0,
2743
+ changedId: row.operation === "UPDATE" ? row.recordId ?? void 0 : void 0,
2744
+ deletedId: row.operation === "DELETE" ? row.recordId ?? void 0 : void 0,
2745
+ queryMetadata: row.queryMetadata ?? {},
2746
+ record: row.record ?? {},
2747
+ previousRecord: row.previousRecord ?? {},
2748
+ requesterId: 0
2749
+ };
2750
+ await eventManager_default.fireActionFromDbTrigger(
2751
+ { mutationType: row.operation, queryMetadata: triggerResult.queryMetadata },
2752
+ triggerResult,
2753
+ { rethrowHandlerErrors: true }
2754
+ );
2755
+ await transaction.runQuery(
2756
+ `UPDATE "${OUTBOX_TABLE_NAME}" SET "processedOn" = now() WHERE "id" = ?;`,
2757
+ [row.id],
2758
+ systemUser
2759
+ );
2444
2760
  } catch (error) {
2445
- this.logSqlStatement(formattedQuery, options, meta, startTime);
2446
- if (error?.routine === "_bt_check_unique") {
2447
- throw new RsError("DUPLICATE", error.message);
2761
+ const attempts = row.attempts + 1;
2762
+ const isDeadLetter = attempts >= this.options.maxAttempts;
2763
+ if (isDeadLetter) {
2764
+ logger.error(
2765
+ `Event outbox row ${row.id} (${row.tableName} ${row.operation}) moved to dead letter after ${attempts} attempts: ${error}`
2766
+ );
2767
+ } else {
2768
+ logger.warn(
2769
+ `Event outbox row ${row.id} (${row.tableName} ${row.operation}) attempt ${attempts} failed: ${error}`
2770
+ );
2448
2771
  }
2449
- throw new RsError("DATABASE_ERROR", `${error.message}`);
2772
+ const backoffSeconds = 30 * Math.pow(2, row.attempts);
2773
+ await transaction.runQuery(
2774
+ `UPDATE "${OUTBOX_TABLE_NAME}"
2775
+ SET "attempts" = ?, "isDeadLetter" = ?, "nextAttemptOn" = now() + (? || ' seconds')::interval
2776
+ WHERE "id" = ?;`,
2777
+ [attempts, isDeadLetter, backoffSeconds, row.id],
2778
+ systemUser
2779
+ );
2450
2780
  }
2451
2781
  }
2452
- async runQuerySchema(query, params, requesterDetails, zodSchema) {
2453
- const result = await this.runQuery(query, params, requesterDetails);
2782
+ async prune() {
2454
2783
  try {
2455
- return z4.array(zodSchema).parse(result);
2784
+ await this.psqlConnectionPool.runQuery(
2785
+ `DELETE FROM "${OUTBOX_TABLE_NAME}"
2786
+ WHERE "processedOn" IS NOT NULL
2787
+ AND "isDeadLetter" = FALSE
2788
+ AND "processedOn" < now() - (? || ' days')::interval;`,
2789
+ [this.options.pruneAfterDays],
2790
+ systemUser
2791
+ );
2456
2792
  } catch (error) {
2457
- if (error instanceof z4.ZodError) {
2458
- logger.error("Invalid data returned from database:");
2459
- logger.trace("\n" + JSON.stringify(result, null, 2));
2460
- logger.error("\n" + z4.prettifyError(error));
2461
- } else {
2462
- logger.error(error);
2463
- }
2464
- throw new RsError("DATABASE_ERROR", `Invalid data returned from database`);
2793
+ logger.error(`Event outbox prune failed: ${error}`);
2465
2794
  }
2466
2795
  }
2467
- logSqlStatement(query, options, queryMetadata, startTime, prefix = "") {
2468
- if (logger.level !== "trace") return;
2469
- const sqlStatement = query.replace(/\$(\d+)/g, (_, num) => {
2470
- const paramIndex = parseInt(num) - 1;
2471
- if (paramIndex >= options.length) return "INVALID_PARAM_INDEX";
2472
- return toSqlLiteral(options[paramIndex]);
2473
- });
2474
- const formattedSql = sqlFormat(sqlStatement, {
2475
- language: "postgresql",
2476
- linesBetweenQueries: 2,
2477
- indentStyle: "standard",
2478
- keywordCase: "upper",
2479
- useTabs: true,
2480
- tabWidth: 4
2481
- });
2482
- const [seconds, nanoseconds] = process.hrtime(startTime);
2483
- const durationMs = seconds * 1e3 + nanoseconds / 1e6;
2484
- let initiator = "Anonymous";
2485
- if ("userId" in queryMetadata && queryMetadata.userId)
2486
- initiator = `User Id (${queryMetadata.userId.toString()})`;
2487
- if ("isSystemUser" in queryMetadata && queryMetadata.isSystemUser) initiator = "SYSTEM";
2488
- logger.trace(`${prefix}query by ${initiator}, Query ->
2489
- ${formattedSql}`, {
2490
- durationMs
2491
- });
2492
- }
2493
2796
  };
2494
2797
 
2495
- // src/restura/sql/PsqlPool.ts
2496
- var { Pool } = pg;
2497
- var PsqlPool = class extends PsqlConnection {
2498
- constructor(poolConfig) {
2499
- super();
2500
- this.poolConfig = poolConfig;
2501
- if (poolConfig.connectionString) {
2502
- let url;
2503
- try {
2504
- url = new URL(poolConfig.connectionString);
2505
- } catch {
2506
- throw new Error(`Invalid connectionString: ${poolConfig.connectionString}`);
2507
- }
2508
- poolConfig.host = url.hostname;
2509
- poolConfig.port = url.port ? parseInt(url.port) : 5432;
2510
- poolConfig.user = url.username;
2511
- poolConfig.password = url.password;
2512
- poolConfig.database = url.pathname.replace(/^\//, "");
2798
+ // src/restura/sql/filterPsqlParser.ts
2799
+ import peg from "pegjs";
2800
+ var initializers = `
2801
+ // Quotes a SQL identifier (column/table name) with double quotes, escaping any embedded quotes
2802
+ function quoteSqlIdentity(value) {
2803
+ return '"' + value.replace(/"/g, '""') + '"';
2804
+ }
2805
+
2806
+ // Unescape special characters in values: \\, -> , | \\| -> | | \\\\ -> \\
2807
+ function unescapeValue(str) {
2808
+ var result = '';
2809
+ for (var i = 0; i < str.length; i++) {
2810
+ if (str[i] === '\\\\' && i + 1 < str.length) {
2811
+ var next = str[i + 1];
2812
+ if (next === ',' || next === '|' || next === '\\\\') {
2813
+ result += next;
2814
+ i++;
2815
+ } else {
2816
+ result += str[i];
2817
+ }
2818
+ } else {
2819
+ result += str[i];
2820
+ }
2821
+ }
2822
+ return result;
2823
+ }
2824
+
2825
+ // Split pipe-separated values respecting escaped pipes
2826
+ function splitPipeValues(str) {
2827
+ var values = [];
2828
+ var current = '';
2829
+ for (var i = 0; i < str.length; i++) {
2830
+ if (str[i] === '\\\\' && i + 1 < str.length && str[i + 1] === '|') {
2831
+ current += '|';
2832
+ i++;
2833
+ } else if (str[i] === '|') {
2834
+ values.push(unescapeValue(current));
2835
+ current = '';
2836
+ } else {
2837
+ current += str[i];
2838
+ }
2839
+ }
2840
+ if (current.length > 0) {
2841
+ values.push(unescapeValue(current));
2842
+ }
2843
+ return values;
2844
+ }
2845
+
2846
+ // Build SQL IN clause from pipe-separated values
2847
+ function buildInClause(column, rawValue) {
2848
+ var values = splitPipeValues(rawValue);
2849
+ var literals = values.map(function(v) { return formatValue(v); });
2850
+ return column + ' IN (' + literals.join(', ') + ')';
2851
+ }
2852
+
2853
+ // Check if a value is numeric and format appropriately
2854
+ function formatValue(value) {
2855
+ // Check if the value is a valid number (integer or decimal)
2856
+ if (/^-?\\d+(\\.\\d+)?$/.test(value)) {
2857
+ return value; // Return as-is without quotes
2858
+ }
2859
+ return format.literal(value);
2860
+ }
2861
+
2862
+ // Format a value with optional type cast
2863
+ function formatValueWithCast(rawValue, cast) {
2864
+ var formatted = formatValue(unescapeValue(rawValue));
2865
+ return cast ? formatted + '::' + cast : formatted;
2866
+ }
2867
+
2868
+ // Build SQL IN clause from pipe-separated values with optional cast
2869
+ function buildInClauseWithCast(column, rawValue, cast) {
2870
+ var values = splitPipeValues(rawValue);
2871
+ var literals = values.map(function(v) {
2872
+ var formatted = formatValue(v);
2873
+ return cast ? formatted + '::' + cast : formatted;
2874
+ });
2875
+ return column + ' IN (' + literals.join(', ') + ')';
2876
+ }
2877
+
2878
+ // Format column with optional cast
2879
+ function formatColumn(col) {
2880
+ if (!col.cast) {
2881
+ return col.sql;
2882
+ }
2883
+ // Wrap JSON field extractions in parentheses when casting
2884
+ // because :: has higher precedence than ->>
2885
+ if (col.isJsonField) {
2886
+ return '(' + col.sql + ')::' + col.cast;
2887
+ }
2888
+ return col.sql + '::' + col.cast;
2889
+ }
2890
+ `;
2891
+ var entryGrammar = `
2892
+ {
2893
+ ${initializers}
2894
+ }
2895
+
2896
+ Start
2897
+ = sql:StartOld { return { sql: sql, usedOldSyntax: true }; }
2898
+ / sql:StartNew { return { sql: sql, usedOldSyntax: false }; }
2899
+ `;
2900
+ var oldGrammar = `
2901
+ StartOld
2902
+ = OldExpressionList
2903
+ _
2904
+ = [ \\t\\r\\n]* // Matches spaces, tabs, and line breaks
2905
+
2906
+ OldExpressionList
2907
+ = leftExpression:OldExpression _ operator:OldOperator _ rightExpression:OldExpressionList
2908
+ { return \`\${leftExpression} \${operator} \${rightExpression}\`;}
2909
+ / OldExpression
2910
+
2911
+ OldExpression
2912
+ = negate:OldNegate? _ "(" _ "column" _ ":" column:OldColumn _ ","? _ value:OldValue? ","? _ type:OldType? _ ")"_
2913
+ {return \`\${negate? " NOT " : ""}(\${type ? type(column, value) : (value == null ? \`\${column} IS NULL\` : \`\${column} = \${formatValue(value)}\`)})\`;}
2914
+ /
2915
+ negate:OldNegate?"("expression:OldExpressionList")" { return \`\${negate? " NOT " : ""}(\${expression})\`; }
2916
+
2917
+ OldNegate
2918
+ = "!"
2919
+
2920
+ OldOperator
2921
+ = "and"i / "or"i
2922
+
2923
+ OldColumn
2924
+ = first:OldColumnPart rest:("." OldColumnPart)* {
2925
+ const partsArray = [first];
2926
+ if (rest && rest.length > 0) {
2927
+ partsArray.push(...rest.map(item => item[1]));
2928
+ }
2929
+
2930
+ if (partsArray.length > 3) {
2931
+ throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
2932
+ }
2933
+
2934
+ if (partsArray.length === 1) {
2935
+ return quoteSqlIdentity(partsArray[0]);
2936
+ }
2937
+ const tableName = quoteSqlIdentity(partsArray[0]);
2938
+
2939
+ // If we only have two parts (table.column), use regular dot notation
2940
+ if (partsArray.length === 2) {
2941
+ return tableName + "." + quoteSqlIdentity(partsArray[1]);
2942
+ }
2943
+
2944
+ // For JSON paths (more than 2 parts), first part is a column, last part uses ->>
2945
+ const jsonColumn = quoteSqlIdentity(partsArray[1]);
2946
+ const lastPart = partsArray[partsArray.length - 1];
2947
+ const escapedLast = lastPart.replace(/'/g, "''");
2948
+ const result = tableName + "." + jsonColumn + "->>'" + escapedLast + "'";
2949
+ return result;
2950
+ }
2951
+
2952
+ OldColumnPart
2953
+ = text:[a-z0-9 \\t\\r\\n\\-_:@']i+ {
2954
+ return text.join("");
2513
2955
  }
2514
- this.pool = new Pool(poolConfig);
2515
- this.queryOne("SELECT NOW();", [], {
2516
- isSystemUser: true,
2517
- role: "",
2518
- host: "localhost",
2519
- ipAddress: "",
2520
- scopes: []
2521
- }).then(() => {
2522
- logger.info("Connected to PostgreSQL database");
2523
- }).catch((error) => {
2524
- logger.error("Error connecting to database", error);
2525
- process.exit(1);
2526
- });
2527
- }
2528
- pool;
2529
- async query(query, values) {
2530
- return this.pool.query(query, values);
2531
- }
2532
- };
2533
2956
 
2534
- // src/restura/sql/psqlSchemaUtils.ts
2535
- var { Client } = pg2;
2536
- var systemUser = {
2537
- role: "",
2538
- scopes: [],
2539
- host: "",
2540
- ipAddress: "",
2541
- isSystemUser: true
2542
- };
2543
- function schemaToPsqlType(column) {
2544
- if (column.hasAutoIncrement) return "BIGSERIAL";
2545
- if (column.type === "ENUM") return "TEXT";
2546
- if (column.type === "DATETIME") return "TIMESTAMPTZ";
2547
- if (column.type === "MEDIUMINT") return "INT";
2548
- return column.type;
2549
- }
2550
- function createInsertTriggerSql(tableName, notify) {
2551
- if (!notify) return "";
2552
- if (notify === "ALL") {
2553
- return `
2554
- CREATE OR REPLACE FUNCTION notify_${tableName}_insert()
2555
- RETURNS TRIGGER AS $$
2556
- DECLARE
2557
- query_metadata JSON;
2558
- BEGIN
2559
- SELECT INTO query_metadata
2560
- (regexp_match(
2561
- current_query(),
2562
- '^--QUERY_METADATA\\(({.*})', 'n'
2563
- ))[1]::json;
2957
+ OldText
2958
+ = text:[a-z0-9 \\t\\r\\n\\-_:@'.]i+ {
2959
+ return text.join("");
2960
+ }
2564
2961
 
2565
- PERFORM pg_notify(
2566
- 'insert',
2567
- json_build_object(
2568
- 'table', '${tableName}',
2569
- 'queryMetadata', query_metadata,
2570
- 'insertedId', NEW.id,
2571
- 'record', NEW
2572
- )::text
2573
- );
2962
+ OldType
2963
+ = "type" _ ":" _ type:OldTypeString {
2964
+ return type;
2965
+ }
2574
2966
 
2575
- RETURN NEW;
2576
- END;
2577
- $$ LANGUAGE plpgsql;
2967
+ OldTypeString
2968
+ = text:"startsWith" { return function(column, value) { return \`\${column} ILIKE '\${format.literal(value).slice(1,-1)}%'\`; } }
2969
+ / text:"endsWith" { return function(column, value) { return \`\${column} ILIKE '%\${format.literal(value).slice(1,-1)}'\`; } }
2970
+ / text:"contains" { return function(column, value) { return \`\${column} ILIKE '%\${format.literal(value).slice(1,-1)}%'\`; } }
2971
+ / text:"exact" { return function(column, value) { return \`\${column} = \${formatValue(value)}\`; } }
2972
+ / text:"greaterThanEqual" { return function(column, value) { return \`\${column} >= \${formatValue(value)}\`; } }
2973
+ / text:"greaterThan" { return function(column, value) { return \`\${column} > \${formatValue(value)}\`; } }
2974
+ / text:"lessThanEqual" { return function(column, value) { return \`\${column} <= \${formatValue(value)}\`; } }
2975
+ / text:"lessThan" { return function(column, value) { return \`\${column} < \${formatValue(value)}\`; } }
2976
+ / text:"isNull" { return function(column, value) { return \`\${column} IS NULL\`; } }
2578
2977
 
2579
- CREATE OR REPLACE TRIGGER "${tableName}_insert"
2580
- AFTER INSERT ON "${tableName}"
2581
- FOR EACH ROW
2582
- EXECUTE FUNCTION notify_${tableName}_insert();
2978
+ OldValue
2979
+ = "value" _ ":" value:OldText {
2980
+ return value;
2981
+ }
2583
2982
  `;
2584
- }
2585
- const notifyColumnNewBuildString = notify.map((column) => `'${column}', NEW."${column}"`).join(",\n");
2586
- return `
2587
- CREATE OR REPLACE FUNCTION notify_${tableName}_insert()
2588
- RETURNS TRIGGER AS $$
2589
- DECLARE
2590
- query_metadata JSON;
2591
- BEGIN
2592
- SELECT INTO query_metadata
2593
- (regexp_match(
2594
- current_query(),
2595
- '^--QUERY_METADATA\\(({.*})', 'n'
2596
- ))[1]::json;
2983
+ var newGrammar = `
2984
+ StartNew
2985
+ = ExpressionList
2597
2986
 
2598
- PERFORM pg_notify(
2599
- 'insert',
2600
- json_build_object(
2601
- 'table', '${tableName}',
2602
- 'queryMetadata', query_metadata,
2603
- 'insertedId', NEW.id,
2604
- 'record', json_build_object(
2605
- ${notifyColumnNewBuildString}
2606
- )
2607
- )::text
2608
- );
2987
+ ExpressionList
2988
+ = left:Expression _ op:("and"i / "or"i) _ right:ExpressionList
2989
+ { return left + ' ' + op.toUpperCase() + ' ' + right; }
2990
+ / Expression
2609
2991
 
2610
- RETURN NEW;
2611
- END;
2612
- $$ LANGUAGE plpgsql;
2992
+ Expression
2993
+ = negate:"!"? _ "(" _ inner:SimpleExprList _ ")" _
2994
+ { return (negate ? 'NOT ' : '') + '(' + inner + ')'; }
2995
+ / SimpleExpr
2613
2996
 
2614
- CREATE OR REPLACE TRIGGER "${tableName}_insert"
2615
- AFTER INSERT ON "${tableName}"
2616
- FOR EACH ROW
2617
- EXECUTE FUNCTION notify_${tableName}_insert();
2618
- `;
2619
- }
2620
- function createUpdateTriggerSql(tableName, notify) {
2621
- if (!notify) return "";
2622
- if (notify === "ALL") {
2623
- return `
2624
- CREATE OR REPLACE FUNCTION notify_${tableName}_update()
2625
- RETURNS TRIGGER AS $$
2626
- DECLARE
2627
- query_metadata JSON;
2628
- BEGIN
2629
- SELECT INTO query_metadata
2630
- (regexp_match(
2631
- current_query(),
2632
- '^--QUERY_METADATA\\(({.*})', 'n'
2633
- ))[1]::json;
2997
+ SimpleExprList
2998
+ = left:SimpleExpr _ op:("and"i / "or"i) _ right:SimpleExprList
2999
+ { return left + ' ' + op.toUpperCase() + ' ' + right; }
3000
+ / SimpleExpr
2634
3001
 
2635
- PERFORM pg_notify(
2636
- 'update',
2637
- json_build_object(
2638
- 'table', '${tableName}',
2639
- 'queryMetadata', query_metadata,
2640
- 'changedId', NEW.id,
2641
- 'record', NEW,
2642
- 'previousRecord', OLD
2643
- )::text
2644
- );
2645
- RETURN NEW;
2646
- END;
2647
- $$ LANGUAGE plpgsql;
3002
+ SimpleExpr
3003
+ = negate:"!"? _ "(" _ col:Column _ "," _ op:OperatorWithValue _ ")" _
3004
+ { return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
3005
+ / negate:"!"? _ "(" _ col:Column _ "," _ op:NullOperator _ ")" _
3006
+ { return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
3007
+ / negate:"!"? _ "(" _ col:Column _ "," _ val:CastedValue _ ")" _
3008
+ { return (negate ? 'NOT ' : '') + '(' + formatColumn(col) + ' = ' + formatValueWithCast(val.value, val.cast) + ')'; }
2648
3009
 
2649
- CREATE OR REPLACE TRIGGER ${tableName}_update
2650
- AFTER UPDATE ON "${tableName}"
2651
- FOR EACH ROW
2652
- EXECUTE FUNCTION notify_${tableName}_update();
2653
- `;
2654
- }
2655
- const notifyColumnNewBuildString = notify.map((column) => `'${column}', NEW."${column}"`).join(",\n");
2656
- const notifyColumnOldBuildString = notify.map((column) => `'${column}', OLD."${column}"`).join(",\n");
2657
- return `
2658
- CREATE OR REPLACE FUNCTION notify_${tableName}_update()
2659
- RETURNS TRIGGER AS $$
2660
- DECLARE
2661
- query_metadata JSON;
2662
- BEGIN
2663
- SELECT INTO query_metadata
2664
- (regexp_match(
2665
- current_query(),
2666
- '^--QUERY_METADATA\\(({.*})', 'n'
2667
- ))[1]::json;
3010
+ Column
3011
+ = first:ColPart rest:("." ColPart)* cast:TypeCast? {
3012
+ const partsArray = [first];
3013
+ if (rest && rest.length > 0) {
3014
+ partsArray.push(...rest.map(item => item[1]));
3015
+ }
3016
+
3017
+ if (partsArray.length > 3) {
3018
+ throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
3019
+ }
3020
+
3021
+ var sql;
3022
+ var isJsonField = false;
3023
+ if (partsArray.length === 1) {
3024
+ sql = quoteSqlIdentity(partsArray[0]);
3025
+ } else {
3026
+ const tableName = quoteSqlIdentity(partsArray[0]);
3027
+
3028
+ if (partsArray.length === 2) {
3029
+ sql = tableName + '.' + quoteSqlIdentity(partsArray[1]);
3030
+ } else {
3031
+ const jsonColumn = quoteSqlIdentity(partsArray[1]);
3032
+ const lastPart = partsArray[partsArray.length - 1];
3033
+ const escapedLast = lastPart.replace(/'/g, "''");
3034
+ sql = tableName + '.' + jsonColumn + "->>'" + escapedLast + "'";
3035
+ isJsonField = true;
3036
+ }
3037
+ }
3038
+
3039
+ return { sql: sql, cast: cast, isJsonField: isJsonField };
3040
+ }
2668
3041
 
2669
- PERFORM pg_notify(
2670
- 'update',
2671
- json_build_object(
2672
- 'table', '${tableName}',
2673
- 'queryMetadata', query_metadata,
2674
- 'changedId', NEW.id,
2675
- 'record', json_build_object(
2676
- ${notifyColumnNewBuildString}
2677
- ),
2678
- 'previousRecord', json_build_object(
2679
- ${notifyColumnOldBuildString}
2680
- )
2681
- )::text
2682
- );
2683
- RETURN NEW;
2684
- END;
2685
- $$ LANGUAGE plpgsql;
3042
+ ColPart
3043
+ = chars:[a-zA-Z0-9_]+ { return chars.join(''); }
2686
3044
 
2687
- CREATE OR REPLACE TRIGGER ${tableName}_update
2688
- AFTER UPDATE ON "${tableName}"
2689
- FOR EACH ROW
2690
- EXECUTE FUNCTION notify_${tableName}_update();
2691
- `;
2692
- }
2693
- function createDeleteTriggerSql(tableName, notify) {
2694
- if (!notify) return "";
2695
- if (notify === "ALL") {
2696
- return `
2697
- CREATE OR REPLACE FUNCTION notify_${tableName}_delete()
2698
- RETURNS TRIGGER AS $$
2699
- DECLARE
2700
- query_metadata JSON;
2701
- BEGIN
2702
- SELECT INTO query_metadata
2703
- (regexp_match(
2704
- current_query(),
2705
- '^--QUERY_METADATA\\(({.*})', 'n'
2706
- ))[1]::json;
3045
+ NullOperator
3046
+ = "notnull"i { return function(col) { return formatColumn(col) + ' IS NOT NULL'; }; }
3047
+ / "null"i { return function(col) { return formatColumn(col) + ' IS NULL'; }; }
3048
+
3049
+ OperatorWithValue
3050
+ = "in"i _ "," _ vals:InValueList cast:TypeCast? { return function(col) {
3051
+ var formattedCol = formatColumn(col);
3052
+ var literals = vals.map(function(v) {
3053
+ var unescaped = unescapeValue(v);
3054
+ var formatted = formatValue(unescaped);
3055
+ return cast ? formatted + '::' + cast : formatted;
3056
+ });
3057
+ return formattedCol + ' IN (' + literals.join(', ') + ')';
3058
+ }; }
3059
+ / "ne"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <> ' + formatValueWithCast(val.value, val.cast); }; }
3060
+ / "gte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' >= ' + formatValueWithCast(val.value, val.cast); }; }
3061
+ / "gt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' > ' + formatValueWithCast(val.value, val.cast); }; }
3062
+ / "lte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <= ' + formatValueWithCast(val.value, val.cast); }; }
3063
+ / "lt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' < ' + formatValueWithCast(val.value, val.cast); }; }
3064
+ / "has"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
3065
+ / "sw"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal(unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
3066
+ / "ew"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value)); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
3067
+
3068
+ InValueList
3069
+ = first:InValue rest:("|" InValue)* {
3070
+ var values = [first];
3071
+ for (var i = 0; i < rest.length; i++) {
3072
+ values.push(rest[i][1]);
3073
+ }
3074
+ return values;
3075
+ }
3076
+
3077
+ InValue
3078
+ = QuotedString
3079
+ / chars:InValueChar+ { return chars.join(''); }
3080
+
3081
+ InValueChar
3082
+ = "\\\\\\\\" { return '\\\\\\\\'; }
3083
+ / "\\\\," { return '\\\\,'; }
3084
+ / "\\\\|" { return '\\\\|'; }
3085
+ / [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" ]
3086
+ / c:":" !":" { return c; }
3087
+
3088
+ CastedValue
3089
+ = val:Value cast:TypeCast? { return { value: val, cast: cast }; }
3090
+
3091
+ CastedValueWithPipes
3092
+ = val:ValueWithPipes cast:TypeCast? { return { value: val, cast: cast }; }
3093
+
3094
+ TypeCast
3095
+ = "::" type:("timestamptz"i / "timestamp"i / "boolean"i / "numeric"i / "bigint"i / "text"i / "date"i / "int"i)
3096
+ { return type.toLowerCase(); }
3097
+
3098
+ QuotedString
3099
+ = '"' chars:DoubleQuotedChar* '"' { return chars.join(''); }
3100
+ / "'" chars:SingleQuotedChar* "'" { return chars.join(''); }
3101
+
3102
+ DoubleQuotedChar
3103
+ = '\\\\"' { return '"'; }
3104
+ / '\\\\\\\\' { return '\\\\'; }
3105
+ / [^"\\\\]
3106
+
3107
+ SingleQuotedChar
3108
+ = "\\\\'" { return "'"; }
3109
+ / '\\\\\\\\' { return '\\\\'; }
3110
+ / [^'\\\\]
3111
+
3112
+ Value
3113
+ = QuotedString
3114
+ / chars:ValueChar+ { return chars.join(''); }
3115
+
3116
+ ValueChar
3117
+ = "\\\\\\\\" { return '\\\\\\\\'; }
3118
+ / "\\\\," { return '\\\\,'; }
3119
+ / "\\\\|" { return '\\\\|'; }
3120
+ / [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" ]
3121
+ / c:":" !":" { return c; }
2707
3122
 
2708
- PERFORM pg_notify(
2709
- 'delete',
2710
- json_build_object(
2711
- 'table', '${tableName}',
2712
- 'queryMetadata', query_metadata,
2713
- 'deletedId', OLD.id,
2714
- 'previousRecord', OLD
2715
- )::text
2716
- );
2717
- RETURN OLD;
2718
- END;
2719
- $$ LANGUAGE plpgsql;
3123
+ ValueWithPipes
3124
+ = QuotedString
3125
+ / chars:ValueWithPipesChar+ { return chars.join(''); }
2720
3126
 
2721
- CREATE OR REPLACE TRIGGER "${tableName}_delete"
2722
- AFTER DELETE ON "${tableName}"
2723
- FOR EACH ROW
2724
- EXECUTE FUNCTION notify_${tableName}_delete();
3127
+ ValueWithPipesChar
3128
+ = "\\\\\\\\" { return '\\\\\\\\'; }
3129
+ / "\\\\," { return '\\\\,'; }
3130
+ / "\\\\|" { return '\\\\|'; }
3131
+ / [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" |]
3132
+ / c:":" !":" { return c; }
2725
3133
  `;
2726
- }
2727
- const notifyColumnOldBuildString = notify.map((column) => `'${column}', OLD."${column}"`).join(",\n");
2728
- return `
2729
- CREATE OR REPLACE FUNCTION notify_${tableName}_delete()
2730
- RETURNS TRIGGER AS $$
2731
- DECLARE
2732
- query_metadata JSON;
2733
- BEGIN
2734
- SELECT INTO query_metadata
2735
- (regexp_match(
2736
- current_query(),
2737
- '^--QUERY_METADATA\\(({.*})', 'n'
2738
- ))[1]::json;
2739
-
2740
- PERFORM pg_notify(
2741
- 'delete',
2742
- json_build_object(
2743
- 'table', '${tableName}',
2744
- 'queryMetadata', query_metadata,
2745
- 'deletedId', OLD.id,
2746
- 'previousRecord', json_build_object(
2747
- ${notifyColumnOldBuildString}
2748
- )
2749
- )::text
2750
- );
2751
- RETURN OLD;
2752
- END;
2753
- $$ LANGUAGE plpgsql;
2754
-
2755
- CREATE OR REPLACE TRIGGER "${tableName}_delete"
2756
- AFTER DELETE ON "${tableName}"
2757
- FOR EACH ROW
2758
- EXECUTE FUNCTION notify_${tableName}_delete();
2759
- `;
2760
- }
2761
- function generateDatabaseSchemaFromSchema(schema) {
2762
- const sqlStatements = [];
2763
- const indexes = [];
2764
- const triggers = [];
2765
- for (const table of schema.database) {
2766
- if (table.notify) {
2767
- triggers.push(createInsertTriggerSql(table.name, table.notify));
2768
- triggers.push(createUpdateTriggerSql(table.name, table.notify));
2769
- triggers.push(createDeleteTriggerSql(table.name, table.notify));
2770
- }
2771
- let sql = `CREATE TABLE "${table.name}"
2772
- ( `;
2773
- const tableColumns = [];
2774
- for (const column of table.columns) {
2775
- let columnSql = "";
2776
- columnSql += ` "${column.name}" ${schemaToPsqlType(column)}`;
2777
- let value = column.value;
2778
- if (column.type === "JSON") value = "";
2779
- if (column.type === "JSONB") value = "";
2780
- if (column.type === "DECIMAL" && value) {
2781
- value = value.replace("-", ",").replace(/['"]/g, "");
2782
- }
2783
- if (value && column.type !== "ENUM") {
2784
- columnSql += `(${value})`;
2785
- } else if (column.length) columnSql += `(${column.length})`;
2786
- if (column.isPrimary) {
2787
- columnSql += " PRIMARY KEY ";
2788
- }
2789
- if (column.isUnique) {
2790
- columnSql += ` CONSTRAINT "${table.name}_${column.name}_unique_index" UNIQUE `;
2791
- }
2792
- if (column.isNullable) columnSql += " NULL";
2793
- else columnSql += " NOT NULL";
2794
- if (column.default) columnSql += ` DEFAULT ${column.default}`;
2795
- if (value && column.type === "ENUM") {
2796
- columnSql += ` CHECK ("${column.name}" IN (${value}))`;
2797
- }
2798
- tableColumns.push(columnSql);
2799
- }
2800
- sql += tableColumns.join(", \n");
2801
- for (const index of table.indexes) {
2802
- if (!index.isPrimaryKey) {
2803
- let unique = " ";
2804
- if (index.isUnique) unique = "UNIQUE ";
2805
- let indexSQL = ` CREATE ${unique}INDEX "${index.name}" ON "${table.name}"`;
2806
- indexSQL += ` (${index.columns.map((item) => `"${item}" ${index.order}`).join(", ")})`;
2807
- indexSQL += index.where ? ` WHERE ${index.where}` : "";
2808
- indexSQL += ";";
2809
- indexes.push(indexSQL);
2810
- }
2811
- }
2812
- sql += "\n);";
2813
- sqlStatements.push(sql);
2814
- }
2815
- for (const table of schema.database) {
2816
- if (!table.foreignKeys.length) continue;
2817
- const sql = `ALTER TABLE "${table.name}" `;
2818
- const constraints = [];
2819
- for (const foreignKey of table.foreignKeys) {
2820
- let constraint = ` ADD CONSTRAINT "${foreignKey.name}"
2821
- FOREIGN KEY ("${foreignKey.column}") REFERENCES "${foreignKey.refTable}" ("${foreignKey.refColumn}")`;
2822
- constraint += ` ON DELETE ${foreignKey.onDelete}`;
2823
- constraint += ` ON UPDATE ${foreignKey.onUpdate}`;
2824
- constraints.push(constraint);
2825
- }
2826
- sqlStatements.push(sql + constraints.join(",\n") + ";");
2827
- }
2828
- for (const table of schema.database) {
2829
- if (!table.checkConstraints.length) continue;
2830
- const sql = `ALTER TABLE "${table.name}" `;
2831
- const constraints = [];
2832
- for (const check of table.checkConstraints) {
2833
- const constraint = `ADD CONSTRAINT "${check.name}" CHECK (${check.check})`;
2834
- constraints.push(constraint);
2835
- }
2836
- sqlStatements.push(sql + constraints.join(",\n") + ";");
2837
- }
2838
- sqlStatements.push(indexes.join("\n"));
2839
- sqlStatements.push(triggers.join("\n"));
2840
- return sqlStatements.join("\n\n");
2841
- }
2842
- async function getNewPublicSchemaAndScratchPool(targetPool, scratchDbName) {
2843
- const scratchDbExists = await targetPool.runQuery(
2844
- `SELECT * FROM pg_database WHERE datname = ?;`,
2845
- [scratchDbName],
2846
- systemUser
2847
- );
2848
- if (scratchDbExists.length === 0) {
2849
- await targetPool.runQuery(`CREATE DATABASE ${escapeColumnName(scratchDbName)};`, [], systemUser);
2850
- }
2851
- const scratchPool = new PsqlPool({
2852
- host: targetPool.poolConfig.host,
2853
- port: targetPool.poolConfig.port,
2854
- user: targetPool.poolConfig.user,
2855
- database: scratchDbName,
2856
- password: targetPool.poolConfig.password,
2857
- max: targetPool.poolConfig.max,
2858
- idleTimeoutMillis: targetPool.poolConfig.idleTimeoutMillis,
2859
- connectionTimeoutMillis: targetPool.poolConfig.connectionTimeoutMillis
2860
- });
2861
- await scratchPool.runQuery(`DROP SCHEMA public CASCADE;`, [], systemUser);
2862
- await scratchPool.runQuery(
2863
- `CREATE SCHEMA public AUTHORIZATION ${escapeColumnName(targetPool.poolConfig.user)};`,
2864
- [],
2865
- systemUser
2866
- );
2867
- const schemaComment = await targetPool.runQuery(
2868
- `
2869
- SELECT pg_description.description
2870
- FROM pg_description
2871
- JOIN pg_namespace ON pg_namespace.oid = pg_description.objoid
2872
- WHERE pg_namespace.nspname = 'public';`,
2873
- [],
2874
- systemUser
2875
- );
2876
- if (schemaComment[0]?.description) {
2877
- const escaped = schemaComment[0].description.replace(/'/g, "''");
2878
- await scratchPool.runQuery(`COMMENT ON SCHEMA public IS '${escaped}';`, [], systemUser);
2879
- }
2880
- return scratchPool;
2881
- }
2882
- async function diffDatabaseToSchema(schema, targetPool, scratchDbName) {
2883
- let scratchPool;
2884
- let originalClient;
2885
- let scratchClient;
2886
- try {
2887
- scratchPool = await getNewPublicSchemaAndScratchPool(targetPool, scratchDbName);
2888
- const sqlFullStatement = generateDatabaseSchemaFromSchema(schema);
2889
- await scratchPool.runQuery(sqlFullStatement, [], systemUser);
2890
- const connectionConfig = {
2891
- host: targetPool.poolConfig.host,
2892
- port: targetPool.poolConfig.port,
2893
- user: targetPool.poolConfig.user,
2894
- password: targetPool.poolConfig.password,
2895
- ssl: targetPool.poolConfig.ssl
2896
- };
2897
- originalClient = new Client({ ...connectionConfig, database: targetPool.poolConfig.database });
2898
- scratchClient = new Client({ ...connectionConfig, database: scratchDbName });
2899
- await Promise.all([originalClient.connect(), scratchClient.connect()]);
2900
- const [info1, info2] = await Promise.all([
2901
- pgInfo({ client: originalClient }),
2902
- pgInfo({ client: scratchClient })
2903
- ]);
2904
- const diff = getDiff(info1, info2);
2905
- return diff.join("\n");
2906
- } finally {
2907
- const cleanups = [];
2908
- if (originalClient) cleanups.push(originalClient.end());
2909
- if (scratchClient) cleanups.push(scratchClient.end());
2910
- if (scratchPool) cleanups.push(scratchPool.pool.end());
2911
- await Promise.allSettled(cleanups);
2912
- }
2913
- }
3134
+ var fullGrammar = entryGrammar + oldGrammar + newGrammar;
3135
+ var filterPsqlParser = peg.generate(fullGrammar, {
3136
+ format: "commonjs",
3137
+ dependencies: { format: "pg-format" }
3138
+ });
3139
+ var filterPsqlParser_default = filterPsqlParser;
2914
3140
 
2915
3141
  // src/restura/sql/PsqlEngine.ts
2916
- var { Client: Client2, types } = pg3;
3142
+ var { Client: Client3, types } = pg4;
2917
3143
  var PsqlEngine = class extends SqlEngine {
2918
- // 5 seconds
2919
- constructor(psqlConnectionPool, shouldListenForDbTriggers = false, scratchDatabaseSuffix = "") {
3144
+ constructor(psqlConnectionPool, shouldListenForDbTriggers = false, scratchDatabaseSuffix = "", eventDelivery) {
2920
3145
  super();
2921
3146
  this.psqlConnectionPool = psqlConnectionPool;
3147
+ this.eventDelivery = eventDelivery || { mode: "direct" };
2922
3148
  this.setupPgReturnTypes();
3149
+ if (this.eventDelivery.mode === "outbox") {
3150
+ this.outboxConsumer = new EventOutboxConsumer(psqlConnectionPool, this.eventDelivery.outbox);
3151
+ }
2923
3152
  if (shouldListenForDbTriggers) {
2924
- this.setupTriggerListeners = this.listenForDbTriggers();
3153
+ this.setupTriggerListeners = this.listenForDbTriggers().catch((error) => {
3154
+ logger.error(`Failed to setup trigger listeners: ${error}`);
3155
+ void this.reconnectTriggerClient();
3156
+ });
3157
+ this.outboxConsumer?.start();
2925
3158
  }
2926
3159
  this.scratchDbName = `${psqlConnectionPool.poolConfig.database}_scratch${scratchDatabaseSuffix ? `_${scratchDatabaseSuffix}` : ""}`;
2927
3160
  }
@@ -2929,12 +3162,34 @@ var PsqlEngine = class extends SqlEngine {
2929
3162
  triggerClient;
2930
3163
  scratchDbName = "";
2931
3164
  reconnectAttempts = 0;
2932
- MAX_RECONNECT_ATTEMPTS = 5;
2933
3165
  INITIAL_RECONNECT_DELAY = 5e3;
3166
+ MAX_RECONNECT_DELAY = 6e4;
3167
+ HEARTBEAT_INTERVAL_MS = 3e4;
3168
+ eventDelivery;
3169
+ outboxConsumer;
3170
+ heartbeatTimer;
3171
+ isListenerConnected = false;
3172
+ isReconnecting = false;
3173
+ isClosed = false;
3174
+ lastHeartbeatOn = null;
2934
3175
  async close() {
3176
+ this.isClosed = true;
3177
+ if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
3178
+ if (this.outboxConsumer) await this.outboxConsumer.stop();
2935
3179
  if (this.triggerClient) {
2936
3180
  await this.triggerClient.end();
2937
3181
  }
3182
+ this.isListenerConnected = false;
3183
+ }
3184
+ getEventListenerHealth() {
3185
+ return {
3186
+ connected: this.isListenerConnected,
3187
+ lastHeartbeatOn: this.lastHeartbeatOn,
3188
+ reconnectAttempts: this.reconnectAttempts
3189
+ };
3190
+ }
3191
+ getOutboxConsumer() {
3192
+ return this.outboxConsumer;
2938
3193
  }
2939
3194
  /**
2940
3195
  * Setup the return types for the PostgreSQL connection.
@@ -2957,35 +3212,55 @@ var PsqlEngine = class extends SqlEngine {
2957
3212
  types.setTypeParser(PG_TYPE_OID.TIMESTAMPTZ, (val) => val === null ? null : new Date(val).toISOString());
2958
3213
  }
2959
3214
  async reconnectTriggerClient() {
2960
- if (this.reconnectAttempts >= this.MAX_RECONNECT_ATTEMPTS) {
2961
- logger.error("Max reconnection attempts reached for trigger client. Stopping reconnection attempts.");
2962
- return;
3215
+ if (this.isReconnecting || this.isClosed) return;
3216
+ this.isReconnecting = true;
3217
+ this.isListenerConnected = false;
3218
+ if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
3219
+ try {
3220
+ while (!this.isClosed) {
3221
+ if (this.triggerClient) {
3222
+ try {
3223
+ await this.triggerClient.end();
3224
+ } catch (error) {
3225
+ logger.error(`Error closing trigger client: ${error}`);
3226
+ }
3227
+ this.triggerClient = void 0;
3228
+ }
3229
+ const exponentialDelay = this.INITIAL_RECONNECT_DELAY * Math.pow(2, this.reconnectAttempts);
3230
+ const delay = Math.min(exponentialDelay, this.MAX_RECONNECT_DELAY) + Math.floor(Math.random() * 1e3);
3231
+ logger.info(
3232
+ `Attempting to reconnect trigger client in ${Math.round(delay / 1e3)} seconds... (attempt ${this.reconnectAttempts + 1})`
3233
+ );
3234
+ await new Promise((resolve2) => setTimeout(resolve2, delay));
3235
+ if (this.isClosed) return;
3236
+ this.reconnectAttempts++;
3237
+ try {
3238
+ await this.listenForDbTriggers();
3239
+ this.reconnectAttempts = 0;
3240
+ return;
3241
+ } catch (error) {
3242
+ logger.error(`Reconnection attempt ${this.reconnectAttempts} failed: ${error}`);
3243
+ }
3244
+ }
3245
+ } finally {
3246
+ this.isReconnecting = false;
2963
3247
  }
2964
- if (this.triggerClient) {
3248
+ }
3249
+ startHeartbeat() {
3250
+ if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
3251
+ this.heartbeatTimer = setInterval(async () => {
3252
+ if (!this.triggerClient || this.isClosed) return;
2965
3253
  try {
2966
- await this.triggerClient.end();
3254
+ await this.triggerClient.query("SELECT 1");
3255
+ this.lastHeartbeatOn = (/* @__PURE__ */ new Date()).toISOString();
2967
3256
  } catch (error) {
2968
- logger.error(`Error closing trigger client: ${error}`);
2969
- }
2970
- }
2971
- const delay = this.INITIAL_RECONNECT_DELAY * Math.pow(2, this.reconnectAttempts);
2972
- logger.info(
2973
- `Attempting to reconnect trigger client in ${delay / 1e3} seconds... (Attempt ${this.reconnectAttempts + 1}/${this.MAX_RECONNECT_ATTEMPTS})`
2974
- );
2975
- await new Promise((resolve2) => setTimeout(resolve2, delay));
2976
- this.reconnectAttempts++;
2977
- try {
2978
- await this.listenForDbTriggers();
2979
- this.reconnectAttempts = 0;
2980
- } catch (error) {
2981
- logger.error(`Reconnection attempt ${this.reconnectAttempts} failed: ${error}`);
2982
- if (this.reconnectAttempts < this.MAX_RECONNECT_ATTEMPTS) {
2983
- await this.reconnectTriggerClient();
3257
+ logger.error(`Trigger client heartbeat failed, reconnecting: ${error}`);
3258
+ void this.reconnectTriggerClient();
2984
3259
  }
2985
- }
3260
+ }, this.HEARTBEAT_INTERVAL_MS);
2986
3261
  }
2987
3262
  async listenForDbTriggers() {
2988
- this.triggerClient = new Client2({
3263
+ const client = new Client3({
2989
3264
  user: this.psqlConnectionPool.poolConfig.user,
2990
3265
  host: this.psqlConnectionPool.poolConfig.host,
2991
3266
  database: this.psqlConnectionPool.poolConfig.database,
@@ -2993,25 +3268,52 @@ var PsqlEngine = class extends SqlEngine {
2993
3268
  port: this.psqlConnectionPool.poolConfig.port,
2994
3269
  connectionTimeoutMillis: this.psqlConnectionPool.poolConfig.connectionTimeoutMillis
2995
3270
  });
3271
+ this.triggerClient = client;
3272
+ client.on("error", (error) => {
3273
+ logger.error(`Trigger client error: ${error}`);
3274
+ void this.reconnectTriggerClient();
3275
+ });
3276
+ client.on("notification", async (msg) => {
3277
+ if (this.outboxConsumer && msg.channel === this.outboxConsumer.channel) {
3278
+ await this.outboxConsumer.drain();
3279
+ } else if (msg.channel === "insert" || msg.channel === "update" || msg.channel === "delete") {
3280
+ const payload = ObjectUtils3.safeParse(msg.payload);
3281
+ await this.handleTrigger(payload, msg.channel.toUpperCase());
3282
+ }
3283
+ });
3284
+ await client.connect();
3285
+ const channels = ["insert", "update", "delete"];
3286
+ if (this.outboxConsumer) channels.push(this.outboxConsumer.channel);
3287
+ for (const channel of channels) {
3288
+ await client.query(`LISTEN ${escapeColumnName(channel)}`);
3289
+ }
3290
+ this.isListenerConnected = true;
3291
+ this.lastHeartbeatOn = (/* @__PURE__ */ new Date()).toISOString();
3292
+ this.startHeartbeat();
3293
+ logger.info("Successfully connected to database triggers");
3294
+ void this.warnIfOutboxBacklogInDirectMode();
3295
+ }
3296
+ async warnIfOutboxBacklogInDirectMode() {
3297
+ if (this.eventDelivery.mode !== "direct") return;
2996
3298
  try {
2997
- await this.triggerClient.connect();
2998
- for (const channel of ["insert", "update", "delete"]) {
2999
- await this.triggerClient.query(`LISTEN ${channel}`);
3299
+ const tableExists = await this.psqlConnectionPool.runQuery(
3300
+ `SELECT to_regclass('public."dbEventOutbox"') AS exists;`,
3301
+ [],
3302
+ systemUser
3303
+ );
3304
+ if (!tableExists[0]?.exists) return;
3305
+ const pending = await this.psqlConnectionPool.runQuery(
3306
+ `SELECT COUNT(*)::int AS count FROM "dbEventOutbox" WHERE "processedOn" IS NULL;`,
3307
+ [],
3308
+ systemUser
3309
+ );
3310
+ if (Number(pending[0]?.count) > 0) {
3311
+ logger.warn(
3312
+ `eventDelivery is 'direct' but "dbEventOutbox" exists with ${pending[0].count} unprocessed rows \u2014 the database appears to have outbox-mode triggers installed. Switch eventDelivery to 'outbox' or regenerate direct-mode triggers.`
3313
+ );
3000
3314
  }
3001
- this.triggerClient.on("error", async (error) => {
3002
- logger.error(`Trigger client error: ${error}`);
3003
- await this.reconnectTriggerClient();
3004
- });
3005
- this.triggerClient.on("notification", async (msg) => {
3006
- if (msg.channel === "insert" || msg.channel === "update" || msg.channel === "delete") {
3007
- const payload = ObjectUtils3.safeParse(msg.payload);
3008
- await this.handleTrigger(payload, msg.channel.toUpperCase());
3009
- }
3010
- });
3011
- logger.info("Successfully connected to database triggers");
3012
3315
  } catch (error) {
3013
- logger.error(`Failed to setup trigger listeners: ${error}`);
3014
- await this.reconnectTriggerClient();
3316
+ logger.warn(`Could not check dbEventOutbox backlog: ${error}`);
3015
3317
  }
3016
3318
  }
3017
3319
  async handleTrigger(payload, mutationType) {
@@ -3025,10 +3327,21 @@ var PsqlEngine = class extends SqlEngine {
3025
3327
  return sqlFullStatement;
3026
3328
  }
3027
3329
  generateDatabaseSchemaFromSchema(schema) {
3028
- return generateDatabaseSchemaFromSchema(schema);
3330
+ return generateDatabaseSchemaFromSchema(schema, this.schemaGenerationOptions());
3029
3331
  }
3030
3332
  async diffDatabaseToSchema(schema) {
3031
- return diffDatabaseToSchema(schema, this.psqlConnectionPool, this.scratchDbName);
3333
+ return diffDatabaseToSchema(
3334
+ schema,
3335
+ this.psqlConnectionPool,
3336
+ this.scratchDbName,
3337
+ this.schemaGenerationOptions()
3338
+ );
3339
+ }
3340
+ schemaGenerationOptions() {
3341
+ return {
3342
+ eventDelivery: this.eventDelivery.mode,
3343
+ outboxChannel: this.eventDelivery.outbox?.channel
3344
+ };
3032
3345
  }
3033
3346
  createNestedSelect(req, schema, item, routeData, sqlParams) {
3034
3347
  if (!item.subquery) return "";
@@ -3466,7 +3779,13 @@ var ResturaEngine = class {
3466
3779
  this.multerCommonUpload = getMulterUpload(this.resturaConfig.fileTempCachePath);
3467
3780
  new TempCache(this.resturaConfig.fileTempCachePath);
3468
3781
  this.psqlConnectionPool = psqlConnectionPool;
3469
- this.psqlEngine = new PsqlEngine(this.psqlConnectionPool, true, this.resturaConfig.scratchDatabaseSuffix);
3782
+ if (this.resturaConfig.queryMetadataKeys) {
3783
+ PsqlConnection.setQueryMetadataKeys(this.resturaConfig.queryMetadataKeys);
3784
+ }
3785
+ this.psqlEngine = new PsqlEngine(this.psqlConnectionPool, true, this.resturaConfig.scratchDatabaseSuffix, {
3786
+ mode: this.resturaConfig.eventDelivery,
3787
+ outbox: this.resturaConfig.eventOutbox
3788
+ });
3470
3789
  await customApiFactory_default.loadApiFiles(this.resturaConfig.customApiFolderPath);
3471
3790
  this.authenticationHandler = authenticationHandler;
3472
3791
  app.use(compression());
@@ -3492,6 +3811,7 @@ var ResturaEngine = class {
3492
3811
  this.expressApp = app;
3493
3812
  await this.reloadEndpoints();
3494
3813
  await this.initializeGeneratedTypesFolder();
3814
+ eventManager_default.validateHandlersAgainstSchema(this.schema, this.resturaConfig.notifyValidation);
3495
3815
  logger.info("Restura Engine Initialized");
3496
3816
  }
3497
3817
  /**
@@ -3621,6 +3941,7 @@ var ResturaEngine = class {
3621
3941
  }
3622
3942
  async updateSchema(req, res) {
3623
3943
  try {
3944
+ eventManager_default.validateHandlersAgainstSchema(req.data, this.resturaConfig.notifyValidation);
3624
3945
  this.schema = sortObjectKeysAlphabetically(req.data);
3625
3946
  await this.storeFileSystemSchema();
3626
3947
  await this.reloadEndpoints();
@@ -4429,47 +4750,12 @@ function defaultsMatch(desired, live, column) {
4429
4750
  const normalizedLive = live.replace(/::[a-z][a-z0-9_ ]*(\[\])?$/gi, "").trim();
4430
4751
  return lowercaseOutsideStrings(desired.trim()) === lowercaseOutsideStrings(normalizedLive);
4431
4752
  }
4432
-
4433
- // src/restura/sql/PsqlTransaction.ts
4434
- import pg4 from "pg";
4435
- var { Client: Client3 } = pg4;
4436
- var PsqlTransaction = class extends PsqlConnection {
4437
- constructor(clientConfig, instanceId) {
4438
- super(instanceId);
4439
- this.clientConfig = clientConfig;
4440
- this.client = new Client3(clientConfig);
4441
- this.connectPromise = this.client.connect();
4442
- this.beginTransactionPromise = this.beginTransaction();
4443
- }
4444
- client;
4445
- beginTransactionPromise;
4446
- connectPromise;
4447
- async close() {
4448
- if (this.client) {
4449
- await this.client.end();
4450
- }
4451
- }
4452
- async beginTransaction() {
4453
- await this.connectPromise;
4454
- return this.client.query("BEGIN");
4455
- }
4456
- async rollback() {
4457
- return this.query("ROLLBACK");
4458
- }
4459
- async commit() {
4460
- return this.query("COMMIT");
4461
- }
4462
- async release() {
4463
- return this.client.end();
4464
- }
4465
- async query(query, values) {
4466
- await this.connectPromise;
4467
- await this.beginTransactionPromise;
4468
- return this.client.query(query, values);
4469
- }
4470
- };
4471
4753
  export {
4754
+ DEFAULT_OUTBOX_CHANNEL,
4755
+ EventManager,
4756
+ EventOutboxConsumer,
4472
4757
  HtmlStatusCodes,
4758
+ OUTBOX_TABLE_NAME,
4473
4759
  PsqlConnection,
4474
4760
  PsqlEngine,
4475
4761
  PsqlPool,
@@ -4479,17 +4765,21 @@ export {
4479
4765
  apiGenerator,
4480
4766
  createDeleteTriggerSql,
4481
4767
  createInsertTriggerSql,
4768
+ createOutboxTableSql,
4482
4769
  createUpdateTriggerSql,
4770
+ defaultEventOutboxOptions,
4483
4771
  diffDatabaseToSchema,
4484
4772
  diffSchemaToDatabase,
4485
4773
  escapeColumnName,
4486
4774
  eventManager_default as eventManager,
4487
4775
  filterPsqlParser_default as filterPsqlParser,
4488
4776
  generateDatabaseSchemaFromSchema,
4777
+ generateNotifyTriggersSql,
4489
4778
  getNewPublicSchemaAndScratchPool,
4490
4779
  insertObjectQuery,
4491
4780
  introspectDatabase,
4492
4781
  isSchemaValid,
4782
+ isSensitiveColumnName,
4493
4783
  isValueNumber,
4494
4784
  logger,
4495
4785
  modelGenerator,