@restura/core 2.0.3 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +166 -57
- package/dist/index.js +1225 -923
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -83,16 +83,20 @@ var EventManager = class {
|
|
|
83
83
|
filter
|
|
84
84
|
});
|
|
85
85
|
}
|
|
86
|
-
async fireActionFromDbTrigger(sqlMutationData, result) {
|
|
86
|
+
async fireActionFromDbTrigger(sqlMutationData, result, options) {
|
|
87
|
+
const handlerErrors = [];
|
|
87
88
|
if (sqlMutationData.mutationType === "INSERT") {
|
|
88
|
-
await this.fireInsertActions(sqlMutationData, result);
|
|
89
|
+
await this.fireInsertActions(sqlMutationData, result, handlerErrors);
|
|
89
90
|
} else if (sqlMutationData.mutationType === "UPDATE") {
|
|
90
|
-
await this.fireUpdateActions(sqlMutationData, result);
|
|
91
|
+
await this.fireUpdateActions(sqlMutationData, result, handlerErrors);
|
|
91
92
|
} else if (sqlMutationData.mutationType === "DELETE") {
|
|
92
|
-
await this.fireDeleteActions(sqlMutationData, result);
|
|
93
|
+
await this.fireDeleteActions(sqlMutationData, result, handlerErrors);
|
|
94
|
+
}
|
|
95
|
+
if (options?.rethrowHandlerErrors && handlerErrors.length) {
|
|
96
|
+
throw handlerErrors[0];
|
|
93
97
|
}
|
|
94
98
|
}
|
|
95
|
-
async fireInsertActions(data, triggerResult) {
|
|
99
|
+
async fireInsertActions(data, triggerResult, handlerErrors) {
|
|
96
100
|
await Bluebird.map(
|
|
97
101
|
this.actionHandlers.DATABASE_ROW_INSERT,
|
|
98
102
|
async ({ callback, filter }) => {
|
|
@@ -107,12 +111,13 @@ var EventManager = class {
|
|
|
107
111
|
await callback(insertData, data.queryMetadata);
|
|
108
112
|
} catch (error) {
|
|
109
113
|
logger.error(`Error firing insert action for table ${triggerResult.table}`, error);
|
|
114
|
+
handlerErrors.push(error);
|
|
110
115
|
}
|
|
111
116
|
},
|
|
112
117
|
{ concurrency: 10 }
|
|
113
118
|
);
|
|
114
119
|
}
|
|
115
|
-
async fireDeleteActions(data, triggerResult) {
|
|
120
|
+
async fireDeleteActions(data, triggerResult, handlerErrors) {
|
|
116
121
|
await Bluebird.map(
|
|
117
122
|
this.actionHandlers.DATABASE_ROW_DELETE,
|
|
118
123
|
async ({ callback, filter }) => {
|
|
@@ -127,12 +132,13 @@ var EventManager = class {
|
|
|
127
132
|
await callback(deleteData, data.queryMetadata);
|
|
128
133
|
} catch (error) {
|
|
129
134
|
logger.error(`Error firing delete action for table ${triggerResult.table}`, error);
|
|
135
|
+
handlerErrors.push(error);
|
|
130
136
|
}
|
|
131
137
|
},
|
|
132
138
|
{ concurrency: 10 }
|
|
133
139
|
);
|
|
134
140
|
}
|
|
135
|
-
async fireUpdateActions(data, triggerResult) {
|
|
141
|
+
async fireUpdateActions(data, triggerResult, handlerErrors) {
|
|
136
142
|
await Bluebird.map(
|
|
137
143
|
this.actionHandlers.DATABASE_COLUMN_UPDATE,
|
|
138
144
|
async ({ callback, filter }) => {
|
|
@@ -148,11 +154,53 @@ var EventManager = class {
|
|
|
148
154
|
await callback(columnChangeData, data.queryMetadata);
|
|
149
155
|
} catch (error) {
|
|
150
156
|
logger.error(`Error firing update action for table ${triggerResult.table}`, error);
|
|
157
|
+
handlerErrors.push(error);
|
|
151
158
|
}
|
|
152
159
|
},
|
|
153
160
|
{ concurrency: 10 }
|
|
154
161
|
);
|
|
155
162
|
}
|
|
163
|
+
// A handler whose filter doesn't match the schema's notify config silently never fires — surface it at startup
|
|
164
|
+
validateHandlersAgainstSchema(schema, mode) {
|
|
165
|
+
if (mode === "off") return;
|
|
166
|
+
const problems = [];
|
|
167
|
+
const checkTable = (tableName, handlerKind) => {
|
|
168
|
+
if (!tableName) return void 0;
|
|
169
|
+
const table = schema.database.find((item) => item.name === tableName);
|
|
170
|
+
if (!table) {
|
|
171
|
+
problems.push(`${handlerKind} handler registered for unknown table "${tableName}"`);
|
|
172
|
+
return void 0;
|
|
173
|
+
}
|
|
174
|
+
if (!table.notify) {
|
|
175
|
+
problems.push(
|
|
176
|
+
`${handlerKind} handler registered for table "${tableName}" which has no notify config \u2014 it will never fire`
|
|
177
|
+
);
|
|
178
|
+
return void 0;
|
|
179
|
+
}
|
|
180
|
+
return table;
|
|
181
|
+
};
|
|
182
|
+
this.actionHandlers.DATABASE_ROW_INSERT.forEach(({ filter }) => checkTable(filter?.tableName, "Row-insert"));
|
|
183
|
+
this.actionHandlers.DATABASE_ROW_DELETE.forEach(({ filter }) => checkTable(filter?.tableName, "Row-delete"));
|
|
184
|
+
this.actionHandlers.DATABASE_COLUMN_UPDATE.forEach(({ filter }) => {
|
|
185
|
+
const table = checkTable(filter?.tableName, "Column-change");
|
|
186
|
+
if (!table || !filter || table.notify === "ALL") return;
|
|
187
|
+
const notifyColumns = table.notify.map((entry) => entry.replace(/^!/, ""));
|
|
188
|
+
for (const column of filter.columns) {
|
|
189
|
+
if (column === "*") continue;
|
|
190
|
+
if (!notifyColumns.includes(column)) {
|
|
191
|
+
problems.push(
|
|
192
|
+
`Column-change handler on "${filter.tableName}" filters column "${column}" which is not in the table's notify list \u2014 it will never fire`
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
if (!problems.length) return;
|
|
198
|
+
if (mode === "strict") {
|
|
199
|
+
throw new Error(`Notify handler validation failed:
|
|
200
|
+
- ${problems.join("\n- ")}`);
|
|
201
|
+
}
|
|
202
|
+
problems.forEach((problem) => logger.warn(`Notify handler validation: ${problem}`));
|
|
203
|
+
}
|
|
156
204
|
hasHandlersForEventType(eventType, filter, triggerResult) {
|
|
157
205
|
if (filter) {
|
|
158
206
|
switch (eventType) {
|
|
@@ -211,7 +259,7 @@ var SqlUtils = class _SqlUtils {
|
|
|
211
259
|
if (type.startsWith("decimal") || type.startsWith("numeric")) return "string";
|
|
212
260
|
if (type.indexOf("int") > -1 || type.startsWith("double") || type.startsWith("float") || type.indexOf("serial") > -1 || type.startsWith("real") || type.startsWith("double precision"))
|
|
213
261
|
return "number";
|
|
214
|
-
if (type === "json") {
|
|
262
|
+
if (type === "json" || type === "jsonb") {
|
|
215
263
|
if (!value) return "object";
|
|
216
264
|
return value.split(",").map((val) => {
|
|
217
265
|
return val.replace(/['"]/g, "");
|
|
@@ -1812,12 +1860,23 @@ var resturaConfigSchema = z3.object({
|
|
|
1812
1860
|
customApiFolderPath: z3.string().default(process.cwd() + customApiFolderPath),
|
|
1813
1861
|
generatedTypesPath: z3.string().default(process.cwd() + "/src/@types"),
|
|
1814
1862
|
fileTempCachePath: z3.string().optional(),
|
|
1815
|
-
scratchDatabaseSuffix: z3.string().optional()
|
|
1863
|
+
scratchDatabaseSuffix: z3.string().optional(),
|
|
1864
|
+
eventDelivery: z3.enum(["direct", "outbox"]).default("direct"),
|
|
1865
|
+
eventOutbox: z3.object({
|
|
1866
|
+
channel: z3.string().default("restura_outbox"),
|
|
1867
|
+
pollIntervalMs: z3.number().default(15e3),
|
|
1868
|
+
batchSize: z3.number().default(50),
|
|
1869
|
+
maxAttempts: z3.number().default(5),
|
|
1870
|
+
pruneAfterDays: z3.number().default(7)
|
|
1871
|
+
}).prefault({}),
|
|
1872
|
+
notifyValidation: z3.enum(["off", "warn", "strict"]).default("warn"),
|
|
1873
|
+
queryMetadataKeys: z3.union([z3.literal("ALL"), z3.array(z3.string())]).optional()
|
|
1816
1874
|
});
|
|
1817
1875
|
|
|
1818
|
-
// src/restura/sql/
|
|
1819
|
-
import
|
|
1820
|
-
import
|
|
1876
|
+
// src/restura/sql/PsqlConnection.ts
|
|
1877
|
+
import crypto from "crypto";
|
|
1878
|
+
import { format as sqlFormat } from "sql-formatter";
|
|
1879
|
+
import { z as z4 } from "zod";
|
|
1821
1880
|
|
|
1822
1881
|
// src/restura/sql/PsqlUtils.ts
|
|
1823
1882
|
import format from "pg-format";
|
|
@@ -1908,6 +1967,140 @@ function toSqlLiteral(value) {
|
|
|
1908
1967
|
return format.literal(value);
|
|
1909
1968
|
}
|
|
1910
1969
|
|
|
1970
|
+
// src/restura/sql/PsqlConnection.ts
|
|
1971
|
+
var DEFAULT_QUERY_METADATA_KEYS = [
|
|
1972
|
+
"userId",
|
|
1973
|
+
"adminUserId",
|
|
1974
|
+
"isSystemUser",
|
|
1975
|
+
"role",
|
|
1976
|
+
"host",
|
|
1977
|
+
"ipAddress",
|
|
1978
|
+
"storeId",
|
|
1979
|
+
"storeCustomerId",
|
|
1980
|
+
"appTokenId",
|
|
1981
|
+
"requestId"
|
|
1982
|
+
];
|
|
1983
|
+
var PsqlConnection = class _PsqlConnection {
|
|
1984
|
+
static queryMetadataKeys = DEFAULT_QUERY_METADATA_KEYS;
|
|
1985
|
+
static setQueryMetadataKeys(keys) {
|
|
1986
|
+
_PsqlConnection.queryMetadataKeys = keys;
|
|
1987
|
+
}
|
|
1988
|
+
instanceId;
|
|
1989
|
+
constructor(instanceId) {
|
|
1990
|
+
this.instanceId = instanceId || crypto.randomUUID();
|
|
1991
|
+
}
|
|
1992
|
+
buildQueryMetadata(requesterDetails) {
|
|
1993
|
+
if (_PsqlConnection.queryMetadataKeys === "ALL") {
|
|
1994
|
+
return { connectionInstanceId: this.instanceId, ...requesterDetails };
|
|
1995
|
+
}
|
|
1996
|
+
const meta = { connectionInstanceId: this.instanceId };
|
|
1997
|
+
const details = requesterDetails ?? {};
|
|
1998
|
+
for (const key of _PsqlConnection.queryMetadataKeys) {
|
|
1999
|
+
if (details[key] !== void 0) meta[key] = details[key];
|
|
2000
|
+
}
|
|
2001
|
+
return meta;
|
|
2002
|
+
}
|
|
2003
|
+
async queryOne(query, options, requesterDetails) {
|
|
2004
|
+
const formattedQuery = questionMarksToOrderedParams(query);
|
|
2005
|
+
const meta = this.buildQueryMetadata(requesterDetails);
|
|
2006
|
+
const queryMetadata = `--QUERY_METADATA(${JSON.stringify(meta)})
|
|
2007
|
+
`;
|
|
2008
|
+
const startTime = process.hrtime();
|
|
2009
|
+
try {
|
|
2010
|
+
const response = await this.query(queryMetadata + formattedQuery, options);
|
|
2011
|
+
if (response.rows.length === 0) throw new RsError("NOT_FOUND", "No results found");
|
|
2012
|
+
else if (response.rows.length > 1) throw new RsError("DUPLICATE", "More than one result found");
|
|
2013
|
+
this.logSqlStatement(formattedQuery, options, meta, startTime);
|
|
2014
|
+
return response.rows[0];
|
|
2015
|
+
} catch (error) {
|
|
2016
|
+
this.logSqlStatement(formattedQuery, options, meta, startTime);
|
|
2017
|
+
if (RsError.isRsError(error)) throw error;
|
|
2018
|
+
if (error?.routine === "_bt_check_unique") {
|
|
2019
|
+
throw new RsError("DUPLICATE", error.message);
|
|
2020
|
+
}
|
|
2021
|
+
throw new RsError("DATABASE_ERROR", `${error.message}`);
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
async queryOneSchema(query, params, requesterDetails, zodSchema) {
|
|
2025
|
+
const result = await this.queryOne(query, params, requesterDetails);
|
|
2026
|
+
try {
|
|
2027
|
+
return zodSchema.parse(result);
|
|
2028
|
+
} catch (error) {
|
|
2029
|
+
if (error instanceof z4.ZodError) {
|
|
2030
|
+
logger.error("Invalid data returned from database:");
|
|
2031
|
+
logger.trace("\n" + JSON.stringify(result, null, 2));
|
|
2032
|
+
logger.error("\n" + z4.prettifyError(error));
|
|
2033
|
+
} else {
|
|
2034
|
+
logger.error(error);
|
|
2035
|
+
}
|
|
2036
|
+
throw new RsError("DATABASE_ERROR", `Invalid data returned from database`);
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
async runQuery(query, options, requesterDetails) {
|
|
2040
|
+
const formattedQuery = questionMarksToOrderedParams(query);
|
|
2041
|
+
const meta = this.buildQueryMetadata(requesterDetails);
|
|
2042
|
+
const queryMetadata = `--QUERY_METADATA(${JSON.stringify(meta)})
|
|
2043
|
+
`;
|
|
2044
|
+
const startTime = process.hrtime();
|
|
2045
|
+
try {
|
|
2046
|
+
const response = await this.query(queryMetadata + formattedQuery, options);
|
|
2047
|
+
this.logSqlStatement(formattedQuery, options, meta, startTime);
|
|
2048
|
+
return response.rows;
|
|
2049
|
+
} catch (error) {
|
|
2050
|
+
this.logSqlStatement(formattedQuery, options, meta, startTime);
|
|
2051
|
+
if (error?.routine === "_bt_check_unique") {
|
|
2052
|
+
throw new RsError("DUPLICATE", error.message);
|
|
2053
|
+
}
|
|
2054
|
+
throw new RsError("DATABASE_ERROR", `${error.message}`);
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
async runQuerySchema(query, params, requesterDetails, zodSchema) {
|
|
2058
|
+
const result = await this.runQuery(query, params, requesterDetails);
|
|
2059
|
+
try {
|
|
2060
|
+
return z4.array(zodSchema).parse(result);
|
|
2061
|
+
} catch (error) {
|
|
2062
|
+
if (error instanceof z4.ZodError) {
|
|
2063
|
+
logger.error("Invalid data returned from database:");
|
|
2064
|
+
logger.trace("\n" + JSON.stringify(result, null, 2));
|
|
2065
|
+
logger.error("\n" + z4.prettifyError(error));
|
|
2066
|
+
} else {
|
|
2067
|
+
logger.error(error);
|
|
2068
|
+
}
|
|
2069
|
+
throw new RsError("DATABASE_ERROR", `Invalid data returned from database`);
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
2072
|
+
logSqlStatement(query, options, queryMetadata, startTime, prefix = "") {
|
|
2073
|
+
if (logger.level !== "trace") return;
|
|
2074
|
+
const sqlStatement = query.replace(/\$(\d+)/g, (_, num) => {
|
|
2075
|
+
const paramIndex = parseInt(num) - 1;
|
|
2076
|
+
if (paramIndex >= options.length) return "INVALID_PARAM_INDEX";
|
|
2077
|
+
return toSqlLiteral(options[paramIndex]);
|
|
2078
|
+
});
|
|
2079
|
+
const formattedSql = sqlFormat(sqlStatement, {
|
|
2080
|
+
language: "postgresql",
|
|
2081
|
+
linesBetweenQueries: 2,
|
|
2082
|
+
indentStyle: "standard",
|
|
2083
|
+
keywordCase: "upper",
|
|
2084
|
+
useTabs: true,
|
|
2085
|
+
tabWidth: 4
|
|
2086
|
+
});
|
|
2087
|
+
const [seconds, nanoseconds] = process.hrtime(startTime);
|
|
2088
|
+
const durationMs = seconds * 1e3 + nanoseconds / 1e6;
|
|
2089
|
+
let initiator = "Anonymous";
|
|
2090
|
+
const initiatorId = queryMetadata.userId ?? queryMetadata.adminUserId;
|
|
2091
|
+
if (initiatorId) initiator = `User Id (${initiatorId.toString()})`;
|
|
2092
|
+
if ("isSystemUser" in queryMetadata && queryMetadata.isSystemUser) initiator = "SYSTEM";
|
|
2093
|
+
logger.trace(`${prefix}query by ${initiator}, Query ->
|
|
2094
|
+
${formattedSql}`, {
|
|
2095
|
+
durationMs
|
|
2096
|
+
});
|
|
2097
|
+
}
|
|
2098
|
+
};
|
|
2099
|
+
|
|
2100
|
+
// src/restura/sql/PsqlEngine.ts
|
|
2101
|
+
import { ObjectUtils as ObjectUtils3 } from "@redskytech/core-utils";
|
|
2102
|
+
import pg4 from "pg";
|
|
2103
|
+
|
|
1911
2104
|
// src/restura/sql/SqlEngine.ts
|
|
1912
2105
|
import { ObjectUtils as ObjectUtils2 } from "@redskytech/core-utils";
|
|
1913
2106
|
var SqlEngine = class {
|
|
@@ -2037,893 +2230,945 @@ var SqlEngine = class {
|
|
|
2037
2230
|
}
|
|
2038
2231
|
};
|
|
2039
2232
|
|
|
2040
|
-
// src/restura/sql/
|
|
2041
|
-
import
|
|
2042
|
-
var
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
} else {
|
|
2058
|
-
result += str[i];
|
|
2059
|
-
}
|
|
2060
|
-
} else {
|
|
2061
|
-
result += str[i];
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
|
-
return result;
|
|
2233
|
+
// src/restura/sql/PsqlTransaction.ts
|
|
2234
|
+
import pg from "pg";
|
|
2235
|
+
var { Client } = pg;
|
|
2236
|
+
var PsqlTransaction = class extends PsqlConnection {
|
|
2237
|
+
constructor(clientConfig, instanceId) {
|
|
2238
|
+
super(instanceId);
|
|
2239
|
+
this.clientConfig = clientConfig;
|
|
2240
|
+
this.client = new Client(clientConfig);
|
|
2241
|
+
this.connectPromise = this.client.connect();
|
|
2242
|
+
this.beginTransactionPromise = this.beginTransaction();
|
|
2243
|
+
}
|
|
2244
|
+
client;
|
|
2245
|
+
beginTransactionPromise;
|
|
2246
|
+
connectPromise;
|
|
2247
|
+
async close() {
|
|
2248
|
+
if (this.client) {
|
|
2249
|
+
await this.client.end();
|
|
2065
2250
|
}
|
|
2251
|
+
}
|
|
2252
|
+
async beginTransaction() {
|
|
2253
|
+
await this.connectPromise;
|
|
2254
|
+
return this.client.query("BEGIN");
|
|
2255
|
+
}
|
|
2256
|
+
async rollback() {
|
|
2257
|
+
return this.query("ROLLBACK");
|
|
2258
|
+
}
|
|
2259
|
+
async commit() {
|
|
2260
|
+
return this.query("COMMIT");
|
|
2261
|
+
}
|
|
2262
|
+
async release() {
|
|
2263
|
+
return this.client.end();
|
|
2264
|
+
}
|
|
2265
|
+
async query(query, values) {
|
|
2266
|
+
await this.connectPromise;
|
|
2267
|
+
await this.beginTransactionPromise;
|
|
2268
|
+
return this.client.query(query, values);
|
|
2269
|
+
}
|
|
2270
|
+
};
|
|
2066
2271
|
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
for (var i = 0; i < str.length; i++) {
|
|
2072
|
-
if (str[i] === '\\\\' && i + 1 < str.length && str[i + 1] === '|') {
|
|
2073
|
-
current += '|';
|
|
2074
|
-
i++;
|
|
2075
|
-
} else if (str[i] === '|') {
|
|
2076
|
-
values.push(unescapeValue(current));
|
|
2077
|
-
current = '';
|
|
2078
|
-
} else {
|
|
2079
|
-
current += str[i];
|
|
2080
|
-
}
|
|
2081
|
-
}
|
|
2082
|
-
if (current.length > 0) {
|
|
2083
|
-
values.push(unescapeValue(current));
|
|
2084
|
-
}
|
|
2085
|
-
return values;
|
|
2086
|
-
}
|
|
2272
|
+
// src/restura/sql/psqlSchemaUtils.ts
|
|
2273
|
+
import getDiff from "@wmfs/pg-diff-sync";
|
|
2274
|
+
import pgInfo from "@wmfs/pg-info";
|
|
2275
|
+
import pg3 from "pg";
|
|
2087
2276
|
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2277
|
+
// src/restura/sql/PsqlPool.ts
|
|
2278
|
+
import pg2 from "pg";
|
|
2279
|
+
var { Pool } = pg2;
|
|
2280
|
+
var PsqlPool = class extends PsqlConnection {
|
|
2281
|
+
constructor(poolConfig) {
|
|
2282
|
+
super();
|
|
2283
|
+
this.poolConfig = poolConfig;
|
|
2284
|
+
if (poolConfig.connectionString) {
|
|
2285
|
+
let url;
|
|
2286
|
+
try {
|
|
2287
|
+
url = new URL(poolConfig.connectionString);
|
|
2288
|
+
} catch {
|
|
2289
|
+
throw new Error(`Invalid connectionString: ${poolConfig.connectionString}`);
|
|
2290
|
+
}
|
|
2291
|
+
poolConfig.host = url.hostname;
|
|
2292
|
+
poolConfig.port = url.port ? parseInt(url.port) : 5432;
|
|
2293
|
+
poolConfig.user = url.username;
|
|
2294
|
+
poolConfig.password = url.password;
|
|
2295
|
+
poolConfig.database = url.pathname.replace(/^\//, "");
|
|
2093
2296
|
}
|
|
2297
|
+
this.pool = new Pool(poolConfig);
|
|
2298
|
+
this.queryOne("SELECT NOW();", [], {
|
|
2299
|
+
isSystemUser: true,
|
|
2300
|
+
role: "",
|
|
2301
|
+
host: "localhost",
|
|
2302
|
+
ipAddress: "",
|
|
2303
|
+
scopes: []
|
|
2304
|
+
}).then(() => {
|
|
2305
|
+
logger.info("Connected to PostgreSQL database");
|
|
2306
|
+
}).catch((error) => {
|
|
2307
|
+
logger.error("Error connecting to database", error);
|
|
2308
|
+
process.exit(1);
|
|
2309
|
+
});
|
|
2310
|
+
}
|
|
2311
|
+
pool;
|
|
2312
|
+
async query(query, values) {
|
|
2313
|
+
return this.pool.query(query, values);
|
|
2314
|
+
}
|
|
2315
|
+
};
|
|
2094
2316
|
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
// Build SQL IN clause from pipe-separated values with optional cast
|
|
2111
|
-
function buildInClauseWithCast(column, rawValue, cast) {
|
|
2112
|
-
var values = splitPipeValues(rawValue);
|
|
2113
|
-
var literals = values.map(function(v) {
|
|
2114
|
-
var formatted = formatValue(v);
|
|
2115
|
-
return cast ? formatted + '::' + cast : formatted;
|
|
2116
|
-
});
|
|
2117
|
-
return column + ' IN (' + literals.join(', ') + ')';
|
|
2118
|
-
}
|
|
2119
|
-
|
|
2120
|
-
// Format column with optional cast
|
|
2121
|
-
function formatColumn(col) {
|
|
2122
|
-
if (!col.cast) {
|
|
2123
|
-
return col.sql;
|
|
2124
|
-
}
|
|
2125
|
-
// Wrap JSON field extractions in parentheses when casting
|
|
2126
|
-
// because :: has higher precedence than ->>
|
|
2127
|
-
if (col.isJsonField) {
|
|
2128
|
-
return '(' + col.sql + ')::' + col.cast;
|
|
2129
|
-
}
|
|
2130
|
-
return col.sql + '::' + col.cast;
|
|
2131
|
-
}
|
|
2132
|
-
`;
|
|
2133
|
-
var entryGrammar = `
|
|
2134
|
-
{
|
|
2135
|
-
${initializers}
|
|
2317
|
+
// src/restura/sql/psqlSchemaUtils.ts
|
|
2318
|
+
var { Client: Client2 } = pg3;
|
|
2319
|
+
var systemUser = {
|
|
2320
|
+
role: "",
|
|
2321
|
+
scopes: [],
|
|
2322
|
+
host: "",
|
|
2323
|
+
ipAddress: "",
|
|
2324
|
+
isSystemUser: true
|
|
2325
|
+
};
|
|
2326
|
+
function schemaToPsqlType(column) {
|
|
2327
|
+
if (column.hasAutoIncrement) return "BIGSERIAL";
|
|
2328
|
+
if (column.type === "ENUM") return "TEXT";
|
|
2329
|
+
if (column.type === "DATETIME") return "TIMESTAMPTZ";
|
|
2330
|
+
if (column.type === "MEDIUMINT") return "INT";
|
|
2331
|
+
return column.type;
|
|
2136
2332
|
}
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2333
|
+
var OUTBOX_TABLE_NAME = "dbEventOutbox";
|
|
2334
|
+
var DEFAULT_OUTBOX_CHANNEL = "restura_outbox";
|
|
2335
|
+
var SENSITIVE_COLUMN_PATTERN = /password|token|secret|credential|guid|key$/i;
|
|
2336
|
+
function isSensitiveColumnName(columnName) {
|
|
2337
|
+
return SENSITIVE_COLUMN_PATTERN.test(columnName);
|
|
2338
|
+
}
|
|
2339
|
+
function createOutboxTableSql() {
|
|
2340
|
+
return `
|
|
2341
|
+
CREATE TABLE IF NOT EXISTS "${OUTBOX_TABLE_NAME}"
|
|
2342
|
+
(
|
|
2343
|
+
"id" BIGSERIAL PRIMARY KEY,
|
|
2344
|
+
"createdOn" TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2345
|
+
"tableName" TEXT NOT NULL,
|
|
2346
|
+
"operation" TEXT NOT NULL,
|
|
2347
|
+
"recordId" BIGINT NULL,
|
|
2348
|
+
"record" JSONB NULL,
|
|
2349
|
+
"previousRecord" JSONB NULL,
|
|
2350
|
+
"queryMetadata" JSONB NULL,
|
|
2351
|
+
"processedOn" TIMESTAMPTZ NULL,
|
|
2352
|
+
"attempts" INT NOT NULL DEFAULT 0,
|
|
2353
|
+
"nextAttemptOn" TIMESTAMPTZ NULL,
|
|
2354
|
+
"isDeadLetter" BOOLEAN NOT NULL DEFAULT FALSE
|
|
2355
|
+
);
|
|
2356
|
+
CREATE INDEX IF NOT EXISTS "${OUTBOX_TABLE_NAME}_unprocessed_index" ON "${OUTBOX_TABLE_NAME}" ("id") WHERE "processedOn" IS NULL AND "isDeadLetter" = FALSE;
|
|
2357
|
+
CREATE INDEX IF NOT EXISTS "${OUTBOX_TABLE_NAME}_processedOn_index" ON "${OUTBOX_TABLE_NAME}" ("processedOn") WHERE "processedOn" IS NOT NULL;
|
|
2141
2358
|
`;
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
/ OldExpression
|
|
2152
|
-
|
|
2153
|
-
OldExpression
|
|
2154
|
-
= negate:OldNegate? _ "(" _ "column" _ ":" column:OldColumn _ ","? _ value:OldValue? ","? _ type:OldType? _ ")"_
|
|
2155
|
-
{return \`\${negate? " NOT " : ""}(\${type ? type(column, value) : (value == null ? \`\${column} IS NULL\` : \`\${column} = \${formatValue(value)}\`)})\`;}
|
|
2156
|
-
/
|
|
2157
|
-
negate:OldNegate?"("expression:OldExpressionList")" { return \`\${negate? " NOT " : ""}(\${expression})\`; }
|
|
2158
|
-
|
|
2159
|
-
OldNegate
|
|
2160
|
-
= "!"
|
|
2161
|
-
|
|
2162
|
-
OldOperator
|
|
2163
|
-
= "and"i / "or"i
|
|
2164
|
-
|
|
2165
|
-
OldColumn
|
|
2166
|
-
= first:OldColumnPart rest:("." OldColumnPart)* {
|
|
2167
|
-
const partsArray = [first];
|
|
2168
|
-
if (rest && rest.length > 0) {
|
|
2169
|
-
partsArray.push(...rest.map(item => item[1]));
|
|
2170
|
-
}
|
|
2171
|
-
|
|
2172
|
-
if (partsArray.length > 3) {
|
|
2173
|
-
throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
|
|
2174
|
-
}
|
|
2175
|
-
|
|
2176
|
-
if (partsArray.length === 1) {
|
|
2177
|
-
return quoteSqlIdentity(partsArray[0]);
|
|
2178
|
-
}
|
|
2179
|
-
const tableName = quoteSqlIdentity(partsArray[0]);
|
|
2180
|
-
|
|
2181
|
-
// If we only have two parts (table.column), use regular dot notation
|
|
2182
|
-
if (partsArray.length === 2) {
|
|
2183
|
-
return tableName + "." + quoteSqlIdentity(partsArray[1]);
|
|
2184
|
-
}
|
|
2185
|
-
|
|
2186
|
-
// For JSON paths (more than 2 parts), first part is a column, last part uses ->>
|
|
2187
|
-
const jsonColumn = quoteSqlIdentity(partsArray[1]);
|
|
2188
|
-
const lastPart = partsArray[partsArray.length - 1];
|
|
2189
|
-
const escapedLast = lastPart.replace(/'/g, "''");
|
|
2190
|
-
const result = tableName + "." + jsonColumn + "->>'" + escapedLast + "'";
|
|
2191
|
-
return result;
|
|
2359
|
+
}
|
|
2360
|
+
function resolveNotifyColumns(tableName, notify, tableColumns) {
|
|
2361
|
+
if (!notify) return [];
|
|
2362
|
+
let columns;
|
|
2363
|
+
if (notify === "ALL") {
|
|
2364
|
+
if (!tableColumns) {
|
|
2365
|
+
throw new Error(
|
|
2366
|
+
`notify: "ALL" on table "${tableName}" requires tableColumns so the sensitive-column denylist can be applied. Pass options.tableColumns or list columns explicitly.`
|
|
2367
|
+
);
|
|
2192
2368
|
}
|
|
2369
|
+
columns = tableColumns.filter((column) => !isSensitiveColumnName(column));
|
|
2370
|
+
} else {
|
|
2371
|
+
columns = notify.map((entry) => {
|
|
2372
|
+
if (entry.startsWith("!")) return entry.slice(1);
|
|
2373
|
+
if (isSensitiveColumnName(entry)) {
|
|
2374
|
+
throw new Error(
|
|
2375
|
+
`Refusing to include sensitive column "${tableName}"."${entry}" in notify trigger payloads. Remove it from the notify list or prefix it with '!' to force-include it.`
|
|
2376
|
+
);
|
|
2377
|
+
}
|
|
2378
|
+
return entry;
|
|
2379
|
+
});
|
|
2380
|
+
}
|
|
2381
|
+
if (!columns.length) {
|
|
2382
|
+
throw new Error(
|
|
2383
|
+
`notify config on table "${tableName}" resolves to no columns \u2014 remove the notify key or list at least one column.`
|
|
2384
|
+
);
|
|
2385
|
+
}
|
|
2386
|
+
return columns;
|
|
2387
|
+
}
|
|
2388
|
+
function sqlStringLiteral(value) {
|
|
2389
|
+
return value.replace(/'/g, "''");
|
|
2390
|
+
}
|
|
2391
|
+
function buildRowJson(rowVariable, columns) {
|
|
2392
|
+
return `jsonb_build_object(
|
|
2393
|
+
${columns.map((column) => `'${sqlStringLiteral(column)}', ${rowVariable}."${column}"`).join(",\n ")}
|
|
2394
|
+
)`;
|
|
2395
|
+
}
|
|
2396
|
+
var QUERY_METADATA_DECLARE_BLOCK = `
|
|
2397
|
+
SELECT INTO query_metadata
|
|
2398
|
+
(regexp_match(
|
|
2399
|
+
current_query(),
|
|
2400
|
+
'^--QUERY_METADATA\\(({.*})', 'n'
|
|
2401
|
+
))[1]::json;
|
|
2402
|
+
`;
|
|
2403
|
+
function buildTriggerFunctionSql(tableName, operation, notify, options) {
|
|
2404
|
+
if (!notify) return "";
|
|
2405
|
+
const columns = resolveNotifyColumns(tableName, notify, options?.tableColumns);
|
|
2406
|
+
const delivery = options?.delivery || "direct";
|
|
2407
|
+
const channel = options?.channel || DEFAULT_OUTBOX_CHANNEL;
|
|
2408
|
+
const tableNameLiteral = sqlStringLiteral(tableName);
|
|
2409
|
+
const channelLiteral = sqlStringLiteral(channel);
|
|
2410
|
+
const functionName = `notify_${tableName}_${operation}`;
|
|
2411
|
+
const rowVariable = operation === "delete" ? "OLD" : "NEW";
|
|
2412
|
+
let body;
|
|
2413
|
+
if (delivery === "outbox") {
|
|
2414
|
+
const recordJson = operation === "delete" ? "NULL" : buildRowJson("NEW", columns);
|
|
2415
|
+
const previousRecordJson = operation === "insert" ? "NULL" : buildRowJson("OLD", columns);
|
|
2416
|
+
body = ` INSERT INTO "${OUTBOX_TABLE_NAME}" ("tableName", "operation", "recordId", "record", "previousRecord", "queryMetadata")
|
|
2417
|
+
VALUES ('${tableNameLiteral}', '${operation.toUpperCase()}', ${rowVariable}.id, ${recordJson}, ${previousRecordJson}, query_metadata::jsonb)
|
|
2418
|
+
RETURNING "id" INTO outbox_id;
|
|
2193
2419
|
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
}
|
|
2420
|
+
PERFORM pg_notify('${channelLiteral}', outbox_id::text);`;
|
|
2421
|
+
} else {
|
|
2422
|
+
const idField = operation === "insert" ? `'insertedId', NEW.id` : operation === "update" ? `'changedId', NEW.id` : `'deletedId', OLD.id`;
|
|
2423
|
+
const payloadFields = [`'table', '${tableNameLiteral}'`, `'queryMetadata', query_metadata`, idField];
|
|
2424
|
+
if (operation !== "delete") payloadFields.push(`'record', ${buildRowJson("NEW", columns)}`);
|
|
2425
|
+
if (operation !== "insert") payloadFields.push(`'previousRecord', ${buildRowJson("OLD", columns)}`);
|
|
2426
|
+
body = ` PERFORM pg_notify(
|
|
2427
|
+
'${operation}',
|
|
2428
|
+
json_build_object(
|
|
2429
|
+
${payloadFields.join(",\n ")}
|
|
2430
|
+
)::text
|
|
2431
|
+
);`;
|
|
2432
|
+
}
|
|
2433
|
+
const outboxDeclare = delivery === "outbox" ? "\n outbox_id BIGINT;" : "";
|
|
2434
|
+
const legacyDrop = operation === "update" && tableName !== tableName.toLowerCase() ? `DROP TRIGGER IF EXISTS ${tableName}_update ON "${tableName}";
|
|
2435
|
+
` : "";
|
|
2436
|
+
let updateScope = "";
|
|
2437
|
+
let updateGuard = "";
|
|
2438
|
+
if (operation === "update") {
|
|
2439
|
+
updateScope = ` OF ${columns.map((column) => `"${column}"`).join(", ")}`;
|
|
2440
|
+
updateGuard = `
|
|
2441
|
+
WHEN (${columns.map((column) => `OLD."${column}" IS DISTINCT FROM NEW."${column}"`).join(" OR ")})`;
|
|
2442
|
+
}
|
|
2443
|
+
return `
|
|
2444
|
+
CREATE OR REPLACE FUNCTION ${functionName}()
|
|
2445
|
+
RETURNS TRIGGER AS $$
|
|
2446
|
+
DECLARE
|
|
2447
|
+
query_metadata JSON;${outboxDeclare}
|
|
2448
|
+
BEGIN
|
|
2449
|
+
${QUERY_METADATA_DECLARE_BLOCK}
|
|
2450
|
+
${body}
|
|
2198
2451
|
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
}
|
|
2452
|
+
RETURN ${rowVariable};
|
|
2453
|
+
END;
|
|
2454
|
+
$$ LANGUAGE plpgsql;
|
|
2203
2455
|
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2456
|
+
${legacyDrop}CREATE OR REPLACE TRIGGER "${tableName}_${operation}"
|
|
2457
|
+
AFTER ${operation.toUpperCase()}${updateScope} ON "${tableName}"
|
|
2458
|
+
FOR EACH ROW${updateGuard}
|
|
2459
|
+
EXECUTE FUNCTION ${functionName}();
|
|
2460
|
+
`;
|
|
2461
|
+
}
|
|
2462
|
+
function createInsertTriggerSql(tableName, notify, options) {
|
|
2463
|
+
return buildTriggerFunctionSql(tableName, "insert", notify, options);
|
|
2464
|
+
}
|
|
2465
|
+
function createUpdateTriggerSql(tableName, notify, options) {
|
|
2466
|
+
return buildTriggerFunctionSql(tableName, "update", notify, options);
|
|
2467
|
+
}
|
|
2468
|
+
function createDeleteTriggerSql(tableName, notify, options) {
|
|
2469
|
+
return buildTriggerFunctionSql(tableName, "delete", notify, options);
|
|
2470
|
+
}
|
|
2471
|
+
function generateNotifyTriggersSql(schema, options) {
|
|
2472
|
+
const statements = [];
|
|
2473
|
+
const hasNotifyTables = schema.database.some((table) => table.notify);
|
|
2474
|
+
if (options?.eventDelivery === "outbox" && hasNotifyTables) {
|
|
2475
|
+
statements.push(createOutboxTableSql());
|
|
2476
|
+
}
|
|
2477
|
+
for (const table of schema.database) {
|
|
2478
|
+
if (!table.notify) continue;
|
|
2479
|
+
const triggerOptions = {
|
|
2480
|
+
delivery: options?.eventDelivery,
|
|
2481
|
+
channel: options?.outboxChannel,
|
|
2482
|
+
tableColumns: table.columns.map((column) => column.name)
|
|
2483
|
+
};
|
|
2484
|
+
statements.push(createInsertTriggerSql(table.name, table.notify, triggerOptions));
|
|
2485
|
+
statements.push(createUpdateTriggerSql(table.name, table.notify, triggerOptions));
|
|
2486
|
+
statements.push(createDeleteTriggerSql(table.name, table.notify, triggerOptions));
|
|
2487
|
+
}
|
|
2488
|
+
return statements;
|
|
2489
|
+
}
|
|
2490
|
+
function generateDatabaseSchemaFromSchema(schema, options) {
|
|
2491
|
+
const sqlStatements = [];
|
|
2492
|
+
const indexes = [];
|
|
2493
|
+
const triggers = generateNotifyTriggersSql(schema, options);
|
|
2494
|
+
for (const table of schema.database) {
|
|
2495
|
+
let sql = `CREATE TABLE "${table.name}"
|
|
2496
|
+
( `;
|
|
2497
|
+
const tableColumns = [];
|
|
2498
|
+
for (const column of table.columns) {
|
|
2499
|
+
let columnSql = "";
|
|
2500
|
+
columnSql += ` "${column.name}" ${schemaToPsqlType(column)}`;
|
|
2501
|
+
let value = column.value;
|
|
2502
|
+
if (column.type === "JSON") value = "";
|
|
2503
|
+
if (column.type === "JSONB") value = "";
|
|
2504
|
+
if (column.type === "DECIMAL" && value) {
|
|
2505
|
+
value = value.replace("-", ",").replace(/['"]/g, "");
|
|
2506
|
+
}
|
|
2507
|
+
if (value && column.type !== "ENUM") {
|
|
2508
|
+
columnSql += `(${value})`;
|
|
2509
|
+
} else if (column.length) columnSql += `(${column.length})`;
|
|
2510
|
+
if (column.isPrimary) {
|
|
2511
|
+
columnSql += " PRIMARY KEY ";
|
|
2512
|
+
}
|
|
2513
|
+
if (column.isUnique) {
|
|
2514
|
+
columnSql += ` CONSTRAINT "${table.name}_${column.name}_unique_index" UNIQUE `;
|
|
2515
|
+
}
|
|
2516
|
+
if (column.isNullable) columnSql += " NULL";
|
|
2517
|
+
else columnSql += " NOT NULL";
|
|
2518
|
+
if (column.default) columnSql += ` DEFAULT ${column.default}`;
|
|
2519
|
+
if (value && column.type === "ENUM") {
|
|
2520
|
+
columnSql += ` CHECK ("${column.name}" IN (${value}))`;
|
|
2521
|
+
}
|
|
2522
|
+
tableColumns.push(columnSql);
|
|
2207
2523
|
}
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
OldValue
|
|
2221
|
-
= "value" _ ":" value:OldText {
|
|
2222
|
-
return value;
|
|
2524
|
+
sql += tableColumns.join(", \n");
|
|
2525
|
+
for (const index of table.indexes) {
|
|
2526
|
+
if (!index.isPrimaryKey) {
|
|
2527
|
+
let unique = " ";
|
|
2528
|
+
if (index.isUnique) unique = "UNIQUE ";
|
|
2529
|
+
let indexSQL = ` CREATE ${unique}INDEX "${index.name}" ON "${table.name}"`;
|
|
2530
|
+
indexSQL += ` (${index.columns.map((item) => `"${item}" ${index.order}`).join(", ")})`;
|
|
2531
|
+
indexSQL += index.where ? ` WHERE ${index.where}` : "";
|
|
2532
|
+
indexSQL += ";";
|
|
2533
|
+
indexes.push(indexSQL);
|
|
2534
|
+
}
|
|
2223
2535
|
}
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
/ SimpleExpr
|
|
2238
|
-
|
|
2239
|
-
SimpleExprList
|
|
2240
|
-
= left:SimpleExpr _ op:("and"i / "or"i) _ right:SimpleExprList
|
|
2241
|
-
{ return left + ' ' + op.toUpperCase() + ' ' + right; }
|
|
2242
|
-
/ SimpleExpr
|
|
2243
|
-
|
|
2244
|
-
SimpleExpr
|
|
2245
|
-
= negate:"!"? _ "(" _ col:Column _ "," _ op:OperatorWithValue _ ")" _
|
|
2246
|
-
{ return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
|
|
2247
|
-
/ negate:"!"? _ "(" _ col:Column _ "," _ op:NullOperator _ ")" _
|
|
2248
|
-
{ return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
|
|
2249
|
-
/ negate:"!"? _ "(" _ col:Column _ "," _ val:CastedValue _ ")" _
|
|
2250
|
-
{ return (negate ? 'NOT ' : '') + '(' + formatColumn(col) + ' = ' + formatValueWithCast(val.value, val.cast) + ')'; }
|
|
2251
|
-
|
|
2252
|
-
Column
|
|
2253
|
-
= first:ColPart rest:("." ColPart)* cast:TypeCast? {
|
|
2254
|
-
const partsArray = [first];
|
|
2255
|
-
if (rest && rest.length > 0) {
|
|
2256
|
-
partsArray.push(...rest.map(item => item[1]));
|
|
2257
|
-
}
|
|
2258
|
-
|
|
2259
|
-
if (partsArray.length > 3) {
|
|
2260
|
-
throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
|
|
2261
|
-
}
|
|
2262
|
-
|
|
2263
|
-
var sql;
|
|
2264
|
-
var isJsonField = false;
|
|
2265
|
-
if (partsArray.length === 1) {
|
|
2266
|
-
sql = quoteSqlIdentity(partsArray[0]);
|
|
2267
|
-
} else {
|
|
2268
|
-
const tableName = quoteSqlIdentity(partsArray[0]);
|
|
2269
|
-
|
|
2270
|
-
if (partsArray.length === 2) {
|
|
2271
|
-
sql = tableName + '.' + quoteSqlIdentity(partsArray[1]);
|
|
2272
|
-
} else {
|
|
2273
|
-
const jsonColumn = quoteSqlIdentity(partsArray[1]);
|
|
2274
|
-
const lastPart = partsArray[partsArray.length - 1];
|
|
2275
|
-
const escapedLast = lastPart.replace(/'/g, "''");
|
|
2276
|
-
sql = tableName + '.' + jsonColumn + "->>'" + escapedLast + "'";
|
|
2277
|
-
isJsonField = true;
|
|
2278
|
-
}
|
|
2279
|
-
}
|
|
2280
|
-
|
|
2281
|
-
return { sql: sql, cast: cast, isJsonField: isJsonField };
|
|
2536
|
+
sql += "\n);";
|
|
2537
|
+
sqlStatements.push(sql);
|
|
2538
|
+
}
|
|
2539
|
+
for (const table of schema.database) {
|
|
2540
|
+
if (!table.foreignKeys.length) continue;
|
|
2541
|
+
const sql = `ALTER TABLE "${table.name}" `;
|
|
2542
|
+
const constraints = [];
|
|
2543
|
+
for (const foreignKey of table.foreignKeys) {
|
|
2544
|
+
let constraint = ` ADD CONSTRAINT "${foreignKey.name}"
|
|
2545
|
+
FOREIGN KEY ("${foreignKey.column}") REFERENCES "${foreignKey.refTable}" ("${foreignKey.refColumn}")`;
|
|
2546
|
+
constraint += ` ON DELETE ${foreignKey.onDelete}`;
|
|
2547
|
+
constraint += ` ON UPDATE ${foreignKey.onUpdate}`;
|
|
2548
|
+
constraints.push(constraint);
|
|
2282
2549
|
}
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
= "in"i _ "," _ vals:InValueList cast:TypeCast? { return function(col) {
|
|
2293
|
-
var formattedCol = formatColumn(col);
|
|
2294
|
-
var literals = vals.map(function(v) {
|
|
2295
|
-
var unescaped = unescapeValue(v);
|
|
2296
|
-
var formatted = formatValue(unescaped);
|
|
2297
|
-
return cast ? formatted + '::' + cast : formatted;
|
|
2298
|
-
});
|
|
2299
|
-
return formattedCol + ' IN (' + literals.join(', ') + ')';
|
|
2300
|
-
}; }
|
|
2301
|
-
/ "ne"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <> ' + formatValueWithCast(val.value, val.cast); }; }
|
|
2302
|
-
/ "gte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' >= ' + formatValueWithCast(val.value, val.cast); }; }
|
|
2303
|
-
/ "gt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' > ' + formatValueWithCast(val.value, val.cast); }; }
|
|
2304
|
-
/ "lte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <= ' + formatValueWithCast(val.value, val.cast); }; }
|
|
2305
|
-
/ "lt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' < ' + formatValueWithCast(val.value, val.cast); }; }
|
|
2306
|
-
/ "has"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
|
|
2307
|
-
/ "sw"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal(unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
|
|
2308
|
-
/ "ew"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value)); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
|
|
2309
|
-
|
|
2310
|
-
InValueList
|
|
2311
|
-
= first:InValue rest:("|" InValue)* {
|
|
2312
|
-
var values = [first];
|
|
2313
|
-
for (var i = 0; i < rest.length; i++) {
|
|
2314
|
-
values.push(rest[i][1]);
|
|
2315
|
-
}
|
|
2316
|
-
return values;
|
|
2550
|
+
sqlStatements.push(sql + constraints.join(",\n") + ";");
|
|
2551
|
+
}
|
|
2552
|
+
for (const table of schema.database) {
|
|
2553
|
+
if (!table.checkConstraints.length) continue;
|
|
2554
|
+
const sql = `ALTER TABLE "${table.name}" `;
|
|
2555
|
+
const constraints = [];
|
|
2556
|
+
for (const check of table.checkConstraints) {
|
|
2557
|
+
const constraint = `ADD CONSTRAINT "${check.name}" CHECK (${check.check})`;
|
|
2558
|
+
constraints.push(constraint);
|
|
2317
2559
|
}
|
|
2560
|
+
sqlStatements.push(sql + constraints.join(",\n") + ";");
|
|
2561
|
+
}
|
|
2562
|
+
sqlStatements.push(indexes.join("\n"));
|
|
2563
|
+
sqlStatements.push(triggers.join("\n"));
|
|
2564
|
+
return sqlStatements.join("\n\n");
|
|
2565
|
+
}
|
|
2566
|
+
async function getNewPublicSchemaAndScratchPool(targetPool, scratchDbName) {
|
|
2567
|
+
const scratchDbExists = await targetPool.runQuery(
|
|
2568
|
+
`SELECT * FROM pg_database WHERE datname = ?;`,
|
|
2569
|
+
[scratchDbName],
|
|
2570
|
+
systemUser
|
|
2571
|
+
);
|
|
2572
|
+
if (scratchDbExists.length === 0) {
|
|
2573
|
+
await targetPool.runQuery(`CREATE DATABASE ${escapeColumnName(scratchDbName)};`, [], systemUser);
|
|
2574
|
+
}
|
|
2575
|
+
const scratchPool = new PsqlPool({
|
|
2576
|
+
host: targetPool.poolConfig.host,
|
|
2577
|
+
port: targetPool.poolConfig.port,
|
|
2578
|
+
user: targetPool.poolConfig.user,
|
|
2579
|
+
database: scratchDbName,
|
|
2580
|
+
password: targetPool.poolConfig.password,
|
|
2581
|
+
max: targetPool.poolConfig.max,
|
|
2582
|
+
idleTimeoutMillis: targetPool.poolConfig.idleTimeoutMillis,
|
|
2583
|
+
connectionTimeoutMillis: targetPool.poolConfig.connectionTimeoutMillis
|
|
2584
|
+
});
|
|
2585
|
+
await scratchPool.runQuery(`DROP SCHEMA public CASCADE;`, [], systemUser);
|
|
2586
|
+
await scratchPool.runQuery(
|
|
2587
|
+
`CREATE SCHEMA public AUTHORIZATION ${escapeColumnName(targetPool.poolConfig.user)};`,
|
|
2588
|
+
[],
|
|
2589
|
+
systemUser
|
|
2590
|
+
);
|
|
2591
|
+
const schemaComment = await targetPool.runQuery(
|
|
2592
|
+
`
|
|
2593
|
+
SELECT pg_description.description
|
|
2594
|
+
FROM pg_description
|
|
2595
|
+
JOIN pg_namespace ON pg_namespace.oid = pg_description.objoid
|
|
2596
|
+
WHERE pg_namespace.nspname = 'public';`,
|
|
2597
|
+
[],
|
|
2598
|
+
systemUser
|
|
2599
|
+
);
|
|
2600
|
+
if (schemaComment[0]?.description) {
|
|
2601
|
+
const escaped = schemaComment[0].description.replace(/'/g, "''");
|
|
2602
|
+
await scratchPool.runQuery(`COMMENT ON SCHEMA public IS '${escaped}';`, [], systemUser);
|
|
2603
|
+
}
|
|
2604
|
+
return scratchPool;
|
|
2605
|
+
}
|
|
2606
|
+
async function diffDatabaseToSchema(schema, targetPool, scratchDbName, options) {
|
|
2607
|
+
let scratchPool;
|
|
2608
|
+
let originalClient;
|
|
2609
|
+
let scratchClient;
|
|
2610
|
+
try {
|
|
2611
|
+
scratchPool = await getNewPublicSchemaAndScratchPool(targetPool, scratchDbName);
|
|
2612
|
+
const sqlFullStatement = generateDatabaseSchemaFromSchema(schema, options);
|
|
2613
|
+
await scratchPool.runQuery(sqlFullStatement, [], systemUser);
|
|
2614
|
+
const connectionConfig = {
|
|
2615
|
+
host: targetPool.poolConfig.host,
|
|
2616
|
+
port: targetPool.poolConfig.port,
|
|
2617
|
+
user: targetPool.poolConfig.user,
|
|
2618
|
+
password: targetPool.poolConfig.password,
|
|
2619
|
+
ssl: targetPool.poolConfig.ssl
|
|
2620
|
+
};
|
|
2621
|
+
originalClient = new Client2({ ...connectionConfig, database: targetPool.poolConfig.database });
|
|
2622
|
+
scratchClient = new Client2({ ...connectionConfig, database: scratchDbName });
|
|
2623
|
+
await Promise.all([originalClient.connect(), scratchClient.connect()]);
|
|
2624
|
+
const [info1, info2] = await Promise.all([
|
|
2625
|
+
pgInfo({ client: originalClient }),
|
|
2626
|
+
pgInfo({ client: scratchClient })
|
|
2627
|
+
]);
|
|
2628
|
+
const diff = getDiff(info1, info2);
|
|
2629
|
+
return diff.join("\n");
|
|
2630
|
+
} finally {
|
|
2631
|
+
const cleanups = [];
|
|
2632
|
+
if (originalClient) cleanups.push(originalClient.end());
|
|
2633
|
+
if (scratchClient) cleanups.push(scratchClient.end());
|
|
2634
|
+
if (scratchPool) cleanups.push(scratchPool.pool.end());
|
|
2635
|
+
await Promise.allSettled(cleanups);
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2318
2638
|
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
=
|
|
2332
|
-
|
|
2333
|
-
CastedValueWithPipes
|
|
2334
|
-
= val:ValueWithPipes cast:TypeCast? { return { value: val, cast: cast }; }
|
|
2335
|
-
|
|
2336
|
-
TypeCast
|
|
2337
|
-
= "::" type:("timestamptz"i / "timestamp"i / "boolean"i / "numeric"i / "bigint"i / "text"i / "date"i / "int"i)
|
|
2338
|
-
{ return type.toLowerCase(); }
|
|
2339
|
-
|
|
2340
|
-
QuotedString
|
|
2341
|
-
= '"' chars:DoubleQuotedChar* '"' { return chars.join(''); }
|
|
2342
|
-
/ "'" chars:SingleQuotedChar* "'" { return chars.join(''); }
|
|
2343
|
-
|
|
2344
|
-
DoubleQuotedChar
|
|
2345
|
-
= '\\\\"' { return '"'; }
|
|
2346
|
-
/ '\\\\\\\\' { return '\\\\'; }
|
|
2347
|
-
/ [^"\\\\]
|
|
2348
|
-
|
|
2349
|
-
SingleQuotedChar
|
|
2350
|
-
= "\\\\'" { return "'"; }
|
|
2351
|
-
/ '\\\\\\\\' { return '\\\\'; }
|
|
2352
|
-
/ [^'\\\\]
|
|
2353
|
-
|
|
2354
|
-
Value
|
|
2355
|
-
= QuotedString
|
|
2356
|
-
/ chars:ValueChar+ { return chars.join(''); }
|
|
2357
|
-
|
|
2358
|
-
ValueChar
|
|
2359
|
-
= "\\\\\\\\" { return '\\\\\\\\'; }
|
|
2360
|
-
/ "\\\\," { return '\\\\,'; }
|
|
2361
|
-
/ "\\\\|" { return '\\\\|'; }
|
|
2362
|
-
/ [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" ]
|
|
2363
|
-
/ c:":" !":" { return c; }
|
|
2364
|
-
|
|
2365
|
-
ValueWithPipes
|
|
2366
|
-
= QuotedString
|
|
2367
|
-
/ chars:ValueWithPipesChar+ { return chars.join(''); }
|
|
2368
|
-
|
|
2369
|
-
ValueWithPipesChar
|
|
2370
|
-
= "\\\\\\\\" { return '\\\\\\\\'; }
|
|
2371
|
-
/ "\\\\," { return '\\\\,'; }
|
|
2372
|
-
/ "\\\\|" { return '\\\\|'; }
|
|
2373
|
-
/ [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" |]
|
|
2374
|
-
/ c:":" !":" { return c; }
|
|
2375
|
-
`;
|
|
2376
|
-
var fullGrammar = entryGrammar + oldGrammar + newGrammar;
|
|
2377
|
-
var filterPsqlParser = peg.generate(fullGrammar, {
|
|
2378
|
-
format: "commonjs",
|
|
2379
|
-
dependencies: { format: "pg-format" }
|
|
2380
|
-
});
|
|
2381
|
-
var filterPsqlParser_default = filterPsqlParser;
|
|
2382
|
-
|
|
2383
|
-
// src/restura/sql/psqlSchemaUtils.ts
|
|
2384
|
-
import getDiff from "@wmfs/pg-diff-sync";
|
|
2385
|
-
import pgInfo from "@wmfs/pg-info";
|
|
2386
|
-
import pg2 from "pg";
|
|
2387
|
-
|
|
2388
|
-
// src/restura/sql/PsqlPool.ts
|
|
2389
|
-
import pg from "pg";
|
|
2390
|
-
|
|
2391
|
-
// src/restura/sql/PsqlConnection.ts
|
|
2392
|
-
import crypto from "crypto";
|
|
2393
|
-
import { format as sqlFormat } from "sql-formatter";
|
|
2394
|
-
import { z as z4 } from "zod";
|
|
2395
|
-
var PsqlConnection = class {
|
|
2396
|
-
instanceId;
|
|
2397
|
-
constructor(instanceId) {
|
|
2398
|
-
this.instanceId = instanceId || crypto.randomUUID();
|
|
2639
|
+
// src/restura/sql/eventOutbox.ts
|
|
2640
|
+
var defaultEventOutboxOptions = {
|
|
2641
|
+
channel: DEFAULT_OUTBOX_CHANNEL,
|
|
2642
|
+
pollIntervalMs: 15e3,
|
|
2643
|
+
batchSize: 50,
|
|
2644
|
+
maxAttempts: 5,
|
|
2645
|
+
pruneAfterDays: 7
|
|
2646
|
+
};
|
|
2647
|
+
var PRUNE_INTERVAL_MS = 60 * 60 * 1e3;
|
|
2648
|
+
var EventOutboxConsumer = class {
|
|
2649
|
+
constructor(psqlConnectionPool, options) {
|
|
2650
|
+
this.psqlConnectionPool = psqlConnectionPool;
|
|
2651
|
+
this.options = { ...defaultEventOutboxOptions, ...options };
|
|
2399
2652
|
}
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2653
|
+
options;
|
|
2654
|
+
isDraining = false;
|
|
2655
|
+
drainRequested = false;
|
|
2656
|
+
isStopped = false;
|
|
2657
|
+
pollTimer;
|
|
2658
|
+
pruneTimer;
|
|
2659
|
+
activeDrain = Promise.resolve();
|
|
2660
|
+
lastDrainOn = null;
|
|
2661
|
+
get channel() {
|
|
2662
|
+
return this.options.channel;
|
|
2663
|
+
}
|
|
2664
|
+
start() {
|
|
2665
|
+
this.isStopped = false;
|
|
2666
|
+
this.pollTimer = setInterval(() => void this.drain(), this.options.pollIntervalMs);
|
|
2667
|
+
this.pruneTimer = setInterval(() => void this.prune(), PRUNE_INTERVAL_MS);
|
|
2668
|
+
void this.drain();
|
|
2669
|
+
void this.prune();
|
|
2670
|
+
}
|
|
2671
|
+
async stop() {
|
|
2672
|
+
this.isStopped = true;
|
|
2673
|
+
if (this.pollTimer) clearInterval(this.pollTimer);
|
|
2674
|
+
if (this.pruneTimer) clearInterval(this.pruneTimer);
|
|
2675
|
+
await this.activeDrain;
|
|
2676
|
+
}
|
|
2677
|
+
/** Safe to call at any frequency; overlapping calls coalesce into one extra pass. */
|
|
2678
|
+
async drain() {
|
|
2679
|
+
if (this.isDraining) {
|
|
2680
|
+
this.drainRequested = true;
|
|
2681
|
+
return;
|
|
2419
2682
|
}
|
|
2683
|
+
this.isDraining = true;
|
|
2684
|
+
this.activeDrain = (async () => {
|
|
2685
|
+
try {
|
|
2686
|
+
do {
|
|
2687
|
+
this.drainRequested = false;
|
|
2688
|
+
let processedCount = 0;
|
|
2689
|
+
do {
|
|
2690
|
+
processedCount = await this.processBatch();
|
|
2691
|
+
} while (processedCount > 0 && !this.isStopped);
|
|
2692
|
+
} while (this.drainRequested && !this.isStopped);
|
|
2693
|
+
this.lastDrainOn = (/* @__PURE__ */ new Date()).toISOString();
|
|
2694
|
+
} catch (error) {
|
|
2695
|
+
logger.error(`Event outbox drain failed: ${error}`);
|
|
2696
|
+
} finally {
|
|
2697
|
+
this.isDraining = false;
|
|
2698
|
+
}
|
|
2699
|
+
})();
|
|
2700
|
+
await this.activeDrain;
|
|
2420
2701
|
}
|
|
2421
|
-
async
|
|
2422
|
-
const result = await this.
|
|
2702
|
+
async getStats() {
|
|
2703
|
+
const result = await this.psqlConnectionPool.runQuery(
|
|
2704
|
+
`SELECT
|
|
2705
|
+
COUNT(*) FILTER (WHERE "processedOn" IS NULL AND "isDeadLetter" = FALSE) AS "pendingCount",
|
|
2706
|
+
COUNT(*) FILTER (WHERE "isDeadLetter" = TRUE) AS "deadLetterCount"
|
|
2707
|
+
FROM "${OUTBOX_TABLE_NAME}";`,
|
|
2708
|
+
[],
|
|
2709
|
+
systemUser
|
|
2710
|
+
);
|
|
2711
|
+
return {
|
|
2712
|
+
pendingCount: Number(result[0]?.pendingCount || 0),
|
|
2713
|
+
deadLetterCount: Number(result[0]?.deadLetterCount || 0),
|
|
2714
|
+
lastDrainOn: this.lastDrainOn
|
|
2715
|
+
};
|
|
2716
|
+
}
|
|
2717
|
+
async processBatch() {
|
|
2718
|
+
const transaction = new PsqlTransaction({
|
|
2719
|
+
host: this.psqlConnectionPool.poolConfig.host,
|
|
2720
|
+
port: this.psqlConnectionPool.poolConfig.port,
|
|
2721
|
+
user: this.psqlConnectionPool.poolConfig.user,
|
|
2722
|
+
password: this.psqlConnectionPool.poolConfig.password,
|
|
2723
|
+
database: this.psqlConnectionPool.poolConfig.database,
|
|
2724
|
+
connectionTimeoutMillis: this.psqlConnectionPool.poolConfig.connectionTimeoutMillis
|
|
2725
|
+
});
|
|
2423
2726
|
try {
|
|
2424
|
-
|
|
2727
|
+
const rows = await transaction.runQuery(
|
|
2728
|
+
`SELECT * FROM "${OUTBOX_TABLE_NAME}"
|
|
2729
|
+
WHERE "processedOn" IS NULL AND "isDeadLetter" = FALSE
|
|
2730
|
+
AND ("nextAttemptOn" IS NULL OR "nextAttemptOn" <= now())
|
|
2731
|
+
ORDER BY "id"
|
|
2732
|
+
LIMIT ?
|
|
2733
|
+
FOR UPDATE SKIP LOCKED;`,
|
|
2734
|
+
[this.options.batchSize],
|
|
2735
|
+
systemUser
|
|
2736
|
+
);
|
|
2737
|
+
for (const row of rows) {
|
|
2738
|
+
await this.processRow(transaction, row);
|
|
2739
|
+
}
|
|
2740
|
+
await transaction.commit();
|
|
2741
|
+
return rows.length;
|
|
2425
2742
|
} catch (error) {
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
logger.error("\n" + z4.prettifyError(error));
|
|
2430
|
-
} else {
|
|
2431
|
-
logger.error(error);
|
|
2743
|
+
try {
|
|
2744
|
+
await transaction.rollback();
|
|
2745
|
+
} catch {
|
|
2432
2746
|
}
|
|
2433
|
-
throw
|
|
2747
|
+
throw error;
|
|
2748
|
+
} finally {
|
|
2749
|
+
await transaction.release();
|
|
2434
2750
|
}
|
|
2435
2751
|
}
|
|
2436
|
-
async
|
|
2437
|
-
const formattedQuery = questionMarksToOrderedParams(query);
|
|
2438
|
-
const meta = { connectionInstanceId: this.instanceId, ...requesterDetails };
|
|
2439
|
-
const queryMetadata = `--QUERY_METADATA(${JSON.stringify(meta)})
|
|
2440
|
-
`;
|
|
2441
|
-
const startTime = process.hrtime();
|
|
2752
|
+
async processRow(transaction, row) {
|
|
2442
2753
|
try {
|
|
2443
|
-
const
|
|
2444
|
-
|
|
2445
|
-
|
|
2754
|
+
const triggerResult = {
|
|
2755
|
+
table: row.tableName,
|
|
2756
|
+
insertedId: row.operation === "INSERT" ? row.recordId ?? void 0 : void 0,
|
|
2757
|
+
changedId: row.operation === "UPDATE" ? row.recordId ?? void 0 : void 0,
|
|
2758
|
+
deletedId: row.operation === "DELETE" ? row.recordId ?? void 0 : void 0,
|
|
2759
|
+
queryMetadata: row.queryMetadata ?? {},
|
|
2760
|
+
record: row.record ?? {},
|
|
2761
|
+
previousRecord: row.previousRecord ?? {},
|
|
2762
|
+
requesterId: 0
|
|
2763
|
+
};
|
|
2764
|
+
await eventManager_default.fireActionFromDbTrigger(
|
|
2765
|
+
{ mutationType: row.operation, queryMetadata: triggerResult.queryMetadata },
|
|
2766
|
+
triggerResult,
|
|
2767
|
+
{ rethrowHandlerErrors: true }
|
|
2768
|
+
);
|
|
2769
|
+
await transaction.runQuery(
|
|
2770
|
+
`UPDATE "${OUTBOX_TABLE_NAME}" SET "processedOn" = now() WHERE "id" = ?;`,
|
|
2771
|
+
[row.id],
|
|
2772
|
+
systemUser
|
|
2773
|
+
);
|
|
2446
2774
|
} catch (error) {
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2775
|
+
const attempts = row.attempts + 1;
|
|
2776
|
+
const isDeadLetter = attempts >= this.options.maxAttempts;
|
|
2777
|
+
if (isDeadLetter) {
|
|
2778
|
+
logger.error(
|
|
2779
|
+
`Event outbox row ${row.id} (${row.tableName} ${row.operation}) moved to dead letter after ${attempts} attempts: ${error}`
|
|
2780
|
+
);
|
|
2781
|
+
} else {
|
|
2782
|
+
logger.warn(
|
|
2783
|
+
`Event outbox row ${row.id} (${row.tableName} ${row.operation}) attempt ${attempts} failed: ${error}`
|
|
2784
|
+
);
|
|
2450
2785
|
}
|
|
2451
|
-
|
|
2786
|
+
const backoffSeconds = 30 * Math.pow(2, row.attempts);
|
|
2787
|
+
await transaction.runQuery(
|
|
2788
|
+
`UPDATE "${OUTBOX_TABLE_NAME}"
|
|
2789
|
+
SET "attempts" = ?, "isDeadLetter" = ?, "nextAttemptOn" = now() + (? || ' seconds')::interval
|
|
2790
|
+
WHERE "id" = ?;`,
|
|
2791
|
+
[attempts, isDeadLetter, backoffSeconds, row.id],
|
|
2792
|
+
systemUser
|
|
2793
|
+
);
|
|
2452
2794
|
}
|
|
2453
2795
|
}
|
|
2454
|
-
async
|
|
2455
|
-
const result = await this.runQuery(query, params, requesterDetails);
|
|
2796
|
+
async prune() {
|
|
2456
2797
|
try {
|
|
2457
|
-
|
|
2798
|
+
await this.psqlConnectionPool.runQuery(
|
|
2799
|
+
`DELETE FROM "${OUTBOX_TABLE_NAME}"
|
|
2800
|
+
WHERE "processedOn" IS NOT NULL
|
|
2801
|
+
AND "isDeadLetter" = FALSE
|
|
2802
|
+
AND "processedOn" < now() - (? || ' days')::interval;`,
|
|
2803
|
+
[this.options.pruneAfterDays],
|
|
2804
|
+
systemUser
|
|
2805
|
+
);
|
|
2458
2806
|
} catch (error) {
|
|
2459
|
-
|
|
2460
|
-
logger.error("Invalid data returned from database:");
|
|
2461
|
-
logger.trace("\n" + JSON.stringify(result, null, 2));
|
|
2462
|
-
logger.error("\n" + z4.prettifyError(error));
|
|
2463
|
-
} else {
|
|
2464
|
-
logger.error(error);
|
|
2465
|
-
}
|
|
2466
|
-
throw new RsError("DATABASE_ERROR", `Invalid data returned from database`);
|
|
2807
|
+
logger.error(`Event outbox prune failed: ${error}`);
|
|
2467
2808
|
}
|
|
2468
2809
|
}
|
|
2469
|
-
logSqlStatement(query, options, queryMetadata, startTime, prefix = "") {
|
|
2470
|
-
if (logger.level !== "trace") return;
|
|
2471
|
-
const sqlStatement = query.replace(/\$(\d+)/g, (_, num) => {
|
|
2472
|
-
const paramIndex = parseInt(num) - 1;
|
|
2473
|
-
if (paramIndex >= options.length) return "INVALID_PARAM_INDEX";
|
|
2474
|
-
return toSqlLiteral(options[paramIndex]);
|
|
2475
|
-
});
|
|
2476
|
-
const formattedSql = sqlFormat(sqlStatement, {
|
|
2477
|
-
language: "postgresql",
|
|
2478
|
-
linesBetweenQueries: 2,
|
|
2479
|
-
indentStyle: "standard",
|
|
2480
|
-
keywordCase: "upper",
|
|
2481
|
-
useTabs: true,
|
|
2482
|
-
tabWidth: 4
|
|
2483
|
-
});
|
|
2484
|
-
const [seconds, nanoseconds] = process.hrtime(startTime);
|
|
2485
|
-
const durationMs = seconds * 1e3 + nanoseconds / 1e6;
|
|
2486
|
-
let initiator = "Anonymous";
|
|
2487
|
-
if ("userId" in queryMetadata && queryMetadata.userId)
|
|
2488
|
-
initiator = `User Id (${queryMetadata.userId.toString()})`;
|
|
2489
|
-
if ("isSystemUser" in queryMetadata && queryMetadata.isSystemUser) initiator = "SYSTEM";
|
|
2490
|
-
logger.trace(`${prefix}query by ${initiator}, Query ->
|
|
2491
|
-
${formattedSql}`, {
|
|
2492
|
-
durationMs
|
|
2493
|
-
});
|
|
2494
|
-
}
|
|
2495
2810
|
};
|
|
2496
2811
|
|
|
2497
|
-
// src/restura/sql/
|
|
2498
|
-
|
|
2499
|
-
var
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2812
|
+
// src/restura/sql/filterPsqlParser.ts
|
|
2813
|
+
import peg from "pegjs";
|
|
2814
|
+
var initializers = `
|
|
2815
|
+
// Quotes a SQL identifier (column/table name) with double quotes, escaping any embedded quotes
|
|
2816
|
+
function quoteSqlIdentity(value) {
|
|
2817
|
+
return '"' + value.replace(/"/g, '""') + '"';
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
// Unescape special characters in values: \\, -> , | \\| -> | | \\\\ -> \\
|
|
2821
|
+
function unescapeValue(str) {
|
|
2822
|
+
var result = '';
|
|
2823
|
+
for (var i = 0; i < str.length; i++) {
|
|
2824
|
+
if (str[i] === '\\\\' && i + 1 < str.length) {
|
|
2825
|
+
var next = str[i + 1];
|
|
2826
|
+
if (next === ',' || next === '|' || next === '\\\\') {
|
|
2827
|
+
result += next;
|
|
2828
|
+
i++;
|
|
2829
|
+
} else {
|
|
2830
|
+
result += str[i];
|
|
2831
|
+
}
|
|
2832
|
+
} else {
|
|
2833
|
+
result += str[i];
|
|
2834
|
+
}
|
|
2835
|
+
}
|
|
2836
|
+
return result;
|
|
2837
|
+
}
|
|
2838
|
+
|
|
2839
|
+
// Split pipe-separated values respecting escaped pipes
|
|
2840
|
+
function splitPipeValues(str) {
|
|
2841
|
+
var values = [];
|
|
2842
|
+
var current = '';
|
|
2843
|
+
for (var i = 0; i < str.length; i++) {
|
|
2844
|
+
if (str[i] === '\\\\' && i + 1 < str.length && str[i + 1] === '|') {
|
|
2845
|
+
current += '|';
|
|
2846
|
+
i++;
|
|
2847
|
+
} else if (str[i] === '|') {
|
|
2848
|
+
values.push(unescapeValue(current));
|
|
2849
|
+
current = '';
|
|
2850
|
+
} else {
|
|
2851
|
+
current += str[i];
|
|
2852
|
+
}
|
|
2853
|
+
}
|
|
2854
|
+
if (current.length > 0) {
|
|
2855
|
+
values.push(unescapeValue(current));
|
|
2856
|
+
}
|
|
2857
|
+
return values;
|
|
2858
|
+
}
|
|
2859
|
+
|
|
2860
|
+
// Build SQL IN clause from pipe-separated values
|
|
2861
|
+
function buildInClause(column, rawValue) {
|
|
2862
|
+
var values = splitPipeValues(rawValue);
|
|
2863
|
+
var literals = values.map(function(v) { return formatValue(v); });
|
|
2864
|
+
return column + ' IN (' + literals.join(', ') + ')';
|
|
2865
|
+
}
|
|
2866
|
+
|
|
2867
|
+
// Check if a value is numeric and format appropriately
|
|
2868
|
+
function formatValue(value) {
|
|
2869
|
+
// Check if the value is a valid number (integer or decimal)
|
|
2870
|
+
if (/^-?\\d+(\\.\\d+)?$/.test(value)) {
|
|
2871
|
+
return value; // Return as-is without quotes
|
|
2872
|
+
}
|
|
2873
|
+
return format.literal(value);
|
|
2874
|
+
}
|
|
2875
|
+
|
|
2876
|
+
// Format a value with optional type cast
|
|
2877
|
+
function formatValueWithCast(rawValue, cast) {
|
|
2878
|
+
var formatted = formatValue(unescapeValue(rawValue));
|
|
2879
|
+
return cast ? formatted + '::' + cast : formatted;
|
|
2880
|
+
}
|
|
2881
|
+
|
|
2882
|
+
// Build SQL IN clause from pipe-separated values with optional cast
|
|
2883
|
+
function buildInClauseWithCast(column, rawValue, cast) {
|
|
2884
|
+
var values = splitPipeValues(rawValue);
|
|
2885
|
+
var literals = values.map(function(v) {
|
|
2886
|
+
var formatted = formatValue(v);
|
|
2887
|
+
return cast ? formatted + '::' + cast : formatted;
|
|
2888
|
+
});
|
|
2889
|
+
return column + ' IN (' + literals.join(', ') + ')';
|
|
2890
|
+
}
|
|
2891
|
+
|
|
2892
|
+
// Format column with optional cast
|
|
2893
|
+
function formatColumn(col) {
|
|
2894
|
+
if (!col.cast) {
|
|
2895
|
+
return col.sql;
|
|
2896
|
+
}
|
|
2897
|
+
// Wrap JSON field extractions in parentheses when casting
|
|
2898
|
+
// because :: has higher precedence than ->>
|
|
2899
|
+
if (col.isJsonField) {
|
|
2900
|
+
return '(' + col.sql + ')::' + col.cast;
|
|
2901
|
+
}
|
|
2902
|
+
return col.sql + '::' + col.cast;
|
|
2903
|
+
}
|
|
2904
|
+
`;
|
|
2905
|
+
var entryGrammar = `
|
|
2906
|
+
{
|
|
2907
|
+
${initializers}
|
|
2908
|
+
}
|
|
2909
|
+
|
|
2910
|
+
Start
|
|
2911
|
+
= sql:StartOld { return { sql: sql, usedOldSyntax: true }; }
|
|
2912
|
+
/ sql:StartNew { return { sql: sql, usedOldSyntax: false }; }
|
|
2913
|
+
`;
|
|
2914
|
+
var oldGrammar = `
|
|
2915
|
+
StartOld
|
|
2916
|
+
= OldExpressionList
|
|
2917
|
+
_
|
|
2918
|
+
= [ \\t\\r\\n]* // Matches spaces, tabs, and line breaks
|
|
2919
|
+
|
|
2920
|
+
OldExpressionList
|
|
2921
|
+
= leftExpression:OldExpression _ operator:OldOperator _ rightExpression:OldExpressionList
|
|
2922
|
+
{ return \`\${leftExpression} \${operator} \${rightExpression}\`;}
|
|
2923
|
+
/ OldExpression
|
|
2924
|
+
|
|
2925
|
+
OldExpression
|
|
2926
|
+
= negate:OldNegate? _ "(" _ "column" _ ":" column:OldColumn _ ","? _ value:OldValue? ","? _ type:OldType? _ ")"_
|
|
2927
|
+
{return \`\${negate? " NOT " : ""}(\${type ? type(column, value) : (value == null ? \`\${column} IS NULL\` : \`\${column} = \${formatValue(value)}\`)})\`;}
|
|
2928
|
+
/
|
|
2929
|
+
negate:OldNegate?"("expression:OldExpressionList")" { return \`\${negate? " NOT " : ""}(\${expression})\`; }
|
|
2930
|
+
|
|
2931
|
+
OldNegate
|
|
2932
|
+
= "!"
|
|
2933
|
+
|
|
2934
|
+
OldOperator
|
|
2935
|
+
= "and"i / "or"i
|
|
2936
|
+
|
|
2937
|
+
OldColumn
|
|
2938
|
+
= first:OldColumnPart rest:("." OldColumnPart)* {
|
|
2939
|
+
const partsArray = [first];
|
|
2940
|
+
if (rest && rest.length > 0) {
|
|
2941
|
+
partsArray.push(...rest.map(item => item[1]));
|
|
2942
|
+
}
|
|
2943
|
+
|
|
2944
|
+
if (partsArray.length > 3) {
|
|
2945
|
+
throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
|
|
2946
|
+
}
|
|
2947
|
+
|
|
2948
|
+
if (partsArray.length === 1) {
|
|
2949
|
+
return quoteSqlIdentity(partsArray[0]);
|
|
2950
|
+
}
|
|
2951
|
+
const tableName = quoteSqlIdentity(partsArray[0]);
|
|
2952
|
+
|
|
2953
|
+
// If we only have two parts (table.column), use regular dot notation
|
|
2954
|
+
if (partsArray.length === 2) {
|
|
2955
|
+
return tableName + "." + quoteSqlIdentity(partsArray[1]);
|
|
2956
|
+
}
|
|
2957
|
+
|
|
2958
|
+
// For JSON paths (more than 2 parts), first part is a column, last part uses ->>
|
|
2959
|
+
const jsonColumn = quoteSqlIdentity(partsArray[1]);
|
|
2960
|
+
const lastPart = partsArray[partsArray.length - 1];
|
|
2961
|
+
const escapedLast = lastPart.replace(/'/g, "''");
|
|
2962
|
+
const result = tableName + "." + jsonColumn + "->>'" + escapedLast + "'";
|
|
2963
|
+
return result;
|
|
2964
|
+
}
|
|
2965
|
+
|
|
2966
|
+
OldColumnPart
|
|
2967
|
+
= text:[a-z0-9 \\t\\r\\n\\-_:@']i+ {
|
|
2968
|
+
return text.join("");
|
|
2515
2969
|
}
|
|
2516
|
-
this.pool = new Pool(poolConfig);
|
|
2517
|
-
this.queryOne("SELECT NOW();", [], {
|
|
2518
|
-
isSystemUser: true,
|
|
2519
|
-
role: "",
|
|
2520
|
-
host: "localhost",
|
|
2521
|
-
ipAddress: "",
|
|
2522
|
-
scopes: []
|
|
2523
|
-
}).then(() => {
|
|
2524
|
-
logger.info("Connected to PostgreSQL database");
|
|
2525
|
-
}).catch((error) => {
|
|
2526
|
-
logger.error("Error connecting to database", error);
|
|
2527
|
-
process.exit(1);
|
|
2528
|
-
});
|
|
2529
|
-
}
|
|
2530
|
-
pool;
|
|
2531
|
-
async query(query, values) {
|
|
2532
|
-
return this.pool.query(query, values);
|
|
2533
|
-
}
|
|
2534
|
-
};
|
|
2535
2970
|
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
scopes: [],
|
|
2541
|
-
host: "",
|
|
2542
|
-
ipAddress: "",
|
|
2543
|
-
isSystemUser: true
|
|
2544
|
-
};
|
|
2545
|
-
function schemaToPsqlType(column) {
|
|
2546
|
-
if (column.hasAutoIncrement) return "BIGSERIAL";
|
|
2547
|
-
if (column.type === "ENUM") return "TEXT";
|
|
2548
|
-
if (column.type === "DATETIME") return "TIMESTAMPTZ";
|
|
2549
|
-
if (column.type === "MEDIUMINT") return "INT";
|
|
2550
|
-
return column.type;
|
|
2551
|
-
}
|
|
2552
|
-
function createInsertTriggerSql(tableName, notify) {
|
|
2553
|
-
if (!notify) return "";
|
|
2554
|
-
if (notify === "ALL") {
|
|
2555
|
-
return `
|
|
2556
|
-
CREATE OR REPLACE FUNCTION notify_${tableName}_insert()
|
|
2557
|
-
RETURNS TRIGGER AS $$
|
|
2558
|
-
DECLARE
|
|
2559
|
-
query_metadata JSON;
|
|
2560
|
-
BEGIN
|
|
2561
|
-
SELECT INTO query_metadata
|
|
2562
|
-
(regexp_match(
|
|
2563
|
-
current_query(),
|
|
2564
|
-
'^--QUERY_METADATA\\(({.*})', 'n'
|
|
2565
|
-
))[1]::json;
|
|
2971
|
+
OldText
|
|
2972
|
+
= text:[a-z0-9 \\t\\r\\n\\-_:@'.]i+ {
|
|
2973
|
+
return text.join("");
|
|
2974
|
+
}
|
|
2566
2975
|
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
'queryMetadata', query_metadata,
|
|
2572
|
-
'insertedId', NEW.id,
|
|
2573
|
-
'record', NEW
|
|
2574
|
-
)::text
|
|
2575
|
-
);
|
|
2976
|
+
OldType
|
|
2977
|
+
= "type" _ ":" _ type:OldTypeString {
|
|
2978
|
+
return type;
|
|
2979
|
+
}
|
|
2576
2980
|
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2981
|
+
OldTypeString
|
|
2982
|
+
= text:"startsWith" { return function(column, value) { return \`\${column} ILIKE '\${format.literal(value).slice(1,-1)}%'\`; } }
|
|
2983
|
+
/ text:"endsWith" { return function(column, value) { return \`\${column} ILIKE '%\${format.literal(value).slice(1,-1)}'\`; } }
|
|
2984
|
+
/ text:"contains" { return function(column, value) { return \`\${column} ILIKE '%\${format.literal(value).slice(1,-1)}%'\`; } }
|
|
2985
|
+
/ text:"exact" { return function(column, value) { return \`\${column} = \${formatValue(value)}\`; } }
|
|
2986
|
+
/ text:"greaterThanEqual" { return function(column, value) { return \`\${column} >= \${formatValue(value)}\`; } }
|
|
2987
|
+
/ text:"greaterThan" { return function(column, value) { return \`\${column} > \${formatValue(value)}\`; } }
|
|
2988
|
+
/ text:"lessThanEqual" { return function(column, value) { return \`\${column} <= \${formatValue(value)}\`; } }
|
|
2989
|
+
/ text:"lessThan" { return function(column, value) { return \`\${column} < \${formatValue(value)}\`; } }
|
|
2990
|
+
/ text:"isNull" { return function(column, value) { return \`\${column} IS NULL\`; } }
|
|
2580
2991
|
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2992
|
+
OldValue
|
|
2993
|
+
= "value" _ ":" value:OldText {
|
|
2994
|
+
return value;
|
|
2995
|
+
}
|
|
2585
2996
|
`;
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
CREATE OR REPLACE FUNCTION notify_${tableName}_insert()
|
|
2590
|
-
RETURNS TRIGGER AS $$
|
|
2591
|
-
DECLARE
|
|
2592
|
-
query_metadata JSON;
|
|
2593
|
-
BEGIN
|
|
2594
|
-
SELECT INTO query_metadata
|
|
2595
|
-
(regexp_match(
|
|
2596
|
-
current_query(),
|
|
2597
|
-
'^--QUERY_METADATA\\(({.*})', 'n'
|
|
2598
|
-
))[1]::json;
|
|
2997
|
+
var newGrammar = `
|
|
2998
|
+
StartNew
|
|
2999
|
+
= ExpressionList
|
|
2599
3000
|
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
'queryMetadata', query_metadata,
|
|
2605
|
-
'insertedId', NEW.id,
|
|
2606
|
-
'record', json_build_object(
|
|
2607
|
-
${notifyColumnNewBuildString}
|
|
2608
|
-
)
|
|
2609
|
-
)::text
|
|
2610
|
-
);
|
|
3001
|
+
ExpressionList
|
|
3002
|
+
= left:Expression _ op:("and"i / "or"i) _ right:ExpressionList
|
|
3003
|
+
{ return left + ' ' + op.toUpperCase() + ' ' + right; }
|
|
3004
|
+
/ Expression
|
|
2611
3005
|
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
3006
|
+
Expression
|
|
3007
|
+
= negate:"!"? _ "(" _ inner:SimpleExprList _ ")" _
|
|
3008
|
+
{ return (negate ? 'NOT ' : '') + '(' + inner + ')'; }
|
|
3009
|
+
/ SimpleExpr
|
|
2615
3010
|
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
`;
|
|
2621
|
-
}
|
|
2622
|
-
function createUpdateTriggerSql(tableName, notify) {
|
|
2623
|
-
if (!notify) return "";
|
|
2624
|
-
if (notify === "ALL") {
|
|
2625
|
-
return `
|
|
2626
|
-
CREATE OR REPLACE FUNCTION notify_${tableName}_update()
|
|
2627
|
-
RETURNS TRIGGER AS $$
|
|
2628
|
-
DECLARE
|
|
2629
|
-
query_metadata JSON;
|
|
2630
|
-
BEGIN
|
|
2631
|
-
SELECT INTO query_metadata
|
|
2632
|
-
(regexp_match(
|
|
2633
|
-
current_query(),
|
|
2634
|
-
'^--QUERY_METADATA\\(({.*})', 'n'
|
|
2635
|
-
))[1]::json;
|
|
3011
|
+
SimpleExprList
|
|
3012
|
+
= left:SimpleExpr _ op:("and"i / "or"i) _ right:SimpleExprList
|
|
3013
|
+
{ return left + ' ' + op.toUpperCase() + ' ' + right; }
|
|
3014
|
+
/ SimpleExpr
|
|
2636
3015
|
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
'previousRecord', OLD
|
|
2645
|
-
)::text
|
|
2646
|
-
);
|
|
2647
|
-
RETURN NEW;
|
|
2648
|
-
END;
|
|
2649
|
-
$$ LANGUAGE plpgsql;
|
|
3016
|
+
SimpleExpr
|
|
3017
|
+
= negate:"!"? _ "(" _ col:Column _ "," _ op:OperatorWithValue _ ")" _
|
|
3018
|
+
{ return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
|
|
3019
|
+
/ negate:"!"? _ "(" _ col:Column _ "," _ op:NullOperator _ ")" _
|
|
3020
|
+
{ return (negate ? 'NOT ' : '') + '(' + op(col) + ')'; }
|
|
3021
|
+
/ negate:"!"? _ "(" _ col:Column _ "," _ val:CastedValue _ ")" _
|
|
3022
|
+
{ return (negate ? 'NOT ' : '') + '(' + formatColumn(col) + ' = ' + formatValueWithCast(val.value, val.cast) + ')'; }
|
|
2650
3023
|
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
3024
|
+
Column
|
|
3025
|
+
= first:ColPart rest:("." ColPart)* cast:TypeCast? {
|
|
3026
|
+
const partsArray = [first];
|
|
3027
|
+
if (rest && rest.length > 0) {
|
|
3028
|
+
partsArray.push(...rest.map(item => item[1]));
|
|
3029
|
+
}
|
|
3030
|
+
|
|
3031
|
+
if (partsArray.length > 3) {
|
|
3032
|
+
throw new SyntaxError('Column path cannot have more than 3 parts (table.column.jsonField)');
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
var sql;
|
|
3036
|
+
var isJsonField = false;
|
|
3037
|
+
if (partsArray.length === 1) {
|
|
3038
|
+
sql = quoteSqlIdentity(partsArray[0]);
|
|
3039
|
+
} else {
|
|
3040
|
+
const tableName = quoteSqlIdentity(partsArray[0]);
|
|
3041
|
+
|
|
3042
|
+
if (partsArray.length === 2) {
|
|
3043
|
+
sql = tableName + '.' + quoteSqlIdentity(partsArray[1]);
|
|
3044
|
+
} else {
|
|
3045
|
+
const jsonColumn = quoteSqlIdentity(partsArray[1]);
|
|
3046
|
+
const lastPart = partsArray[partsArray.length - 1];
|
|
3047
|
+
const escapedLast = lastPart.replace(/'/g, "''");
|
|
3048
|
+
sql = tableName + '.' + jsonColumn + "->>'" + escapedLast + "'";
|
|
3049
|
+
isJsonField = true;
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
|
|
3053
|
+
return { sql: sql, cast: cast, isJsonField: isJsonField };
|
|
3054
|
+
}
|
|
2670
3055
|
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
json_build_object(
|
|
2674
|
-
'table', '${tableName}',
|
|
2675
|
-
'queryMetadata', query_metadata,
|
|
2676
|
-
'changedId', NEW.id,
|
|
2677
|
-
'record', json_build_object(
|
|
2678
|
-
${notifyColumnNewBuildString}
|
|
2679
|
-
),
|
|
2680
|
-
'previousRecord', json_build_object(
|
|
2681
|
-
${notifyColumnOldBuildString}
|
|
2682
|
-
)
|
|
2683
|
-
)::text
|
|
2684
|
-
);
|
|
2685
|
-
RETURN NEW;
|
|
2686
|
-
END;
|
|
2687
|
-
$$ LANGUAGE plpgsql;
|
|
3056
|
+
ColPart
|
|
3057
|
+
= chars:[a-zA-Z0-9_]+ { return chars.join(''); }
|
|
2688
3058
|
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
3059
|
+
NullOperator
|
|
3060
|
+
= "notnull"i { return function(col) { return formatColumn(col) + ' IS NOT NULL'; }; }
|
|
3061
|
+
/ "null"i { return function(col) { return formatColumn(col) + ' IS NULL'; }; }
|
|
3062
|
+
|
|
3063
|
+
OperatorWithValue
|
|
3064
|
+
= "in"i _ "," _ vals:InValueList cast:TypeCast? { return function(col) {
|
|
3065
|
+
var formattedCol = formatColumn(col);
|
|
3066
|
+
var literals = vals.map(function(v) {
|
|
3067
|
+
var unescaped = unescapeValue(v);
|
|
3068
|
+
var formatted = formatValue(unescaped);
|
|
3069
|
+
return cast ? formatted + '::' + cast : formatted;
|
|
3070
|
+
});
|
|
3071
|
+
return formattedCol + ' IN (' + literals.join(', ') + ')';
|
|
3072
|
+
}; }
|
|
3073
|
+
/ "ne"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <> ' + formatValueWithCast(val.value, val.cast); }; }
|
|
3074
|
+
/ "gte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' >= ' + formatValueWithCast(val.value, val.cast); }; }
|
|
3075
|
+
/ "gt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' > ' + formatValueWithCast(val.value, val.cast); }; }
|
|
3076
|
+
/ "lte"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' <= ' + formatValueWithCast(val.value, val.cast); }; }
|
|
3077
|
+
/ "lt"i _ "," _ val:CastedValue { return function(col) { return formatColumn(col) + ' < ' + formatValueWithCast(val.value, val.cast); }; }
|
|
3078
|
+
/ "has"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
|
|
3079
|
+
/ "sw"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal(unescapeValue(val.value) + '%'); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
|
|
3080
|
+
/ "ew"i _ "," _ val:CastedValue { return function(col) { var formatted = format.literal('%' + unescapeValue(val.value)); return formatColumn(col) + ' ILIKE ' + (val.cast ? formatted + '::' + val.cast : formatted); }; }
|
|
3081
|
+
|
|
3082
|
+
InValueList
|
|
3083
|
+
= first:InValue rest:("|" InValue)* {
|
|
3084
|
+
var values = [first];
|
|
3085
|
+
for (var i = 0; i < rest.length; i++) {
|
|
3086
|
+
values.push(rest[i][1]);
|
|
3087
|
+
}
|
|
3088
|
+
return values;
|
|
3089
|
+
}
|
|
3090
|
+
|
|
3091
|
+
InValue
|
|
3092
|
+
= QuotedString
|
|
3093
|
+
/ chars:InValueChar+ { return chars.join(''); }
|
|
3094
|
+
|
|
3095
|
+
InValueChar
|
|
3096
|
+
= "\\\\\\\\" { return '\\\\\\\\'; }
|
|
3097
|
+
/ "\\\\," { return '\\\\,'; }
|
|
3098
|
+
/ "\\\\|" { return '\\\\|'; }
|
|
3099
|
+
/ [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" ]
|
|
3100
|
+
/ c:":" !":" { return c; }
|
|
3101
|
+
|
|
3102
|
+
CastedValue
|
|
3103
|
+
= val:Value cast:TypeCast? { return { value: val, cast: cast }; }
|
|
3104
|
+
|
|
3105
|
+
CastedValueWithPipes
|
|
3106
|
+
= val:ValueWithPipes cast:TypeCast? { return { value: val, cast: cast }; }
|
|
3107
|
+
|
|
3108
|
+
TypeCast
|
|
3109
|
+
= "::" type:("timestamptz"i / "timestamp"i / "boolean"i / "numeric"i / "bigint"i / "text"i / "date"i / "int"i)
|
|
3110
|
+
{ return type.toLowerCase(); }
|
|
3111
|
+
|
|
3112
|
+
QuotedString
|
|
3113
|
+
= '"' chars:DoubleQuotedChar* '"' { return chars.join(''); }
|
|
3114
|
+
/ "'" chars:SingleQuotedChar* "'" { return chars.join(''); }
|
|
3115
|
+
|
|
3116
|
+
DoubleQuotedChar
|
|
3117
|
+
= '\\\\"' { return '"'; }
|
|
3118
|
+
/ '\\\\\\\\' { return '\\\\'; }
|
|
3119
|
+
/ [^"\\\\]
|
|
3120
|
+
|
|
3121
|
+
SingleQuotedChar
|
|
3122
|
+
= "\\\\'" { return "'"; }
|
|
3123
|
+
/ '\\\\\\\\' { return '\\\\'; }
|
|
3124
|
+
/ [^'\\\\]
|
|
3125
|
+
|
|
3126
|
+
Value
|
|
3127
|
+
= QuotedString
|
|
3128
|
+
/ chars:ValueChar+ { return chars.join(''); }
|
|
3129
|
+
|
|
3130
|
+
ValueChar
|
|
3131
|
+
= "\\\\\\\\" { return '\\\\\\\\'; }
|
|
3132
|
+
/ "\\\\," { return '\\\\,'; }
|
|
3133
|
+
/ "\\\\|" { return '\\\\|'; }
|
|
3134
|
+
/ [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" ]
|
|
3135
|
+
/ c:":" !":" { return c; }
|
|
2709
3136
|
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
'table', '${tableName}',
|
|
2714
|
-
'queryMetadata', query_metadata,
|
|
2715
|
-
'deletedId', OLD.id,
|
|
2716
|
-
'previousRecord', OLD
|
|
2717
|
-
)::text
|
|
2718
|
-
);
|
|
2719
|
-
RETURN OLD;
|
|
2720
|
-
END;
|
|
2721
|
-
$$ LANGUAGE plpgsql;
|
|
3137
|
+
ValueWithPipes
|
|
3138
|
+
= QuotedString
|
|
3139
|
+
/ chars:ValueWithPipesChar+ { return chars.join(''); }
|
|
2722
3140
|
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
3141
|
+
ValueWithPipesChar
|
|
3142
|
+
= "\\\\\\\\" { return '\\\\\\\\'; }
|
|
3143
|
+
/ "\\\\," { return '\\\\,'; }
|
|
3144
|
+
/ "\\\\|" { return '\\\\|'; }
|
|
3145
|
+
/ [a-zA-Z0-9_\\-!#/@$%^&*+=<>?~.;'" |]
|
|
3146
|
+
/ c:":" !":" { return c; }
|
|
2727
3147
|
`;
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
query_metadata JSON;
|
|
2735
|
-
BEGIN
|
|
2736
|
-
SELECT INTO query_metadata
|
|
2737
|
-
(regexp_match(
|
|
2738
|
-
current_query(),
|
|
2739
|
-
'^--QUERY_METADATA\\(({.*})', 'n'
|
|
2740
|
-
))[1]::json;
|
|
2741
|
-
|
|
2742
|
-
PERFORM pg_notify(
|
|
2743
|
-
'delete',
|
|
2744
|
-
json_build_object(
|
|
2745
|
-
'table', '${tableName}',
|
|
2746
|
-
'queryMetadata', query_metadata,
|
|
2747
|
-
'deletedId', OLD.id,
|
|
2748
|
-
'previousRecord', json_build_object(
|
|
2749
|
-
${notifyColumnOldBuildString}
|
|
2750
|
-
)
|
|
2751
|
-
)::text
|
|
2752
|
-
);
|
|
2753
|
-
RETURN OLD;
|
|
2754
|
-
END;
|
|
2755
|
-
$$ LANGUAGE plpgsql;
|
|
2756
|
-
|
|
2757
|
-
CREATE OR REPLACE TRIGGER "${tableName}_delete"
|
|
2758
|
-
AFTER DELETE ON "${tableName}"
|
|
2759
|
-
FOR EACH ROW
|
|
2760
|
-
EXECUTE FUNCTION notify_${tableName}_delete();
|
|
2761
|
-
`;
|
|
2762
|
-
}
|
|
2763
|
-
function generateDatabaseSchemaFromSchema(schema) {
|
|
2764
|
-
const sqlStatements = [];
|
|
2765
|
-
const indexes = [];
|
|
2766
|
-
const triggers = [];
|
|
2767
|
-
for (const table of schema.database) {
|
|
2768
|
-
if (table.notify) {
|
|
2769
|
-
triggers.push(createInsertTriggerSql(table.name, table.notify));
|
|
2770
|
-
triggers.push(createUpdateTriggerSql(table.name, table.notify));
|
|
2771
|
-
triggers.push(createDeleteTriggerSql(table.name, table.notify));
|
|
2772
|
-
}
|
|
2773
|
-
let sql = `CREATE TABLE "${table.name}"
|
|
2774
|
-
( `;
|
|
2775
|
-
const tableColumns = [];
|
|
2776
|
-
for (const column of table.columns) {
|
|
2777
|
-
let columnSql = "";
|
|
2778
|
-
columnSql += ` "${column.name}" ${schemaToPsqlType(column)}`;
|
|
2779
|
-
let value = column.value;
|
|
2780
|
-
if (column.type === "JSON") value = "";
|
|
2781
|
-
if (column.type === "JSONB") value = "";
|
|
2782
|
-
if (column.type === "DECIMAL" && value) {
|
|
2783
|
-
value = value.replace("-", ",").replace(/['"]/g, "");
|
|
2784
|
-
}
|
|
2785
|
-
if (value && column.type !== "ENUM") {
|
|
2786
|
-
columnSql += `(${value})`;
|
|
2787
|
-
} else if (column.length) columnSql += `(${column.length})`;
|
|
2788
|
-
if (column.isPrimary) {
|
|
2789
|
-
columnSql += " PRIMARY KEY ";
|
|
2790
|
-
}
|
|
2791
|
-
if (column.isUnique) {
|
|
2792
|
-
columnSql += ` CONSTRAINT "${table.name}_${column.name}_unique_index" UNIQUE `;
|
|
2793
|
-
}
|
|
2794
|
-
if (column.isNullable) columnSql += " NULL";
|
|
2795
|
-
else columnSql += " NOT NULL";
|
|
2796
|
-
if (column.default) columnSql += ` DEFAULT ${column.default}`;
|
|
2797
|
-
if (value && column.type === "ENUM") {
|
|
2798
|
-
columnSql += ` CHECK ("${column.name}" IN (${value}))`;
|
|
2799
|
-
}
|
|
2800
|
-
tableColumns.push(columnSql);
|
|
2801
|
-
}
|
|
2802
|
-
sql += tableColumns.join(", \n");
|
|
2803
|
-
for (const index of table.indexes) {
|
|
2804
|
-
if (!index.isPrimaryKey) {
|
|
2805
|
-
let unique = " ";
|
|
2806
|
-
if (index.isUnique) unique = "UNIQUE ";
|
|
2807
|
-
let indexSQL = ` CREATE ${unique}INDEX "${index.name}" ON "${table.name}"`;
|
|
2808
|
-
indexSQL += ` (${index.columns.map((item) => `"${item}" ${index.order}`).join(", ")})`;
|
|
2809
|
-
indexSQL += index.where ? ` WHERE ${index.where}` : "";
|
|
2810
|
-
indexSQL += ";";
|
|
2811
|
-
indexes.push(indexSQL);
|
|
2812
|
-
}
|
|
2813
|
-
}
|
|
2814
|
-
sql += "\n);";
|
|
2815
|
-
sqlStatements.push(sql);
|
|
2816
|
-
}
|
|
2817
|
-
for (const table of schema.database) {
|
|
2818
|
-
if (!table.foreignKeys.length) continue;
|
|
2819
|
-
const sql = `ALTER TABLE "${table.name}" `;
|
|
2820
|
-
const constraints = [];
|
|
2821
|
-
for (const foreignKey of table.foreignKeys) {
|
|
2822
|
-
let constraint = ` ADD CONSTRAINT "${foreignKey.name}"
|
|
2823
|
-
FOREIGN KEY ("${foreignKey.column}") REFERENCES "${foreignKey.refTable}" ("${foreignKey.refColumn}")`;
|
|
2824
|
-
constraint += ` ON DELETE ${foreignKey.onDelete}`;
|
|
2825
|
-
constraint += ` ON UPDATE ${foreignKey.onUpdate}`;
|
|
2826
|
-
constraints.push(constraint);
|
|
2827
|
-
}
|
|
2828
|
-
sqlStatements.push(sql + constraints.join(",\n") + ";");
|
|
2829
|
-
}
|
|
2830
|
-
for (const table of schema.database) {
|
|
2831
|
-
if (!table.checkConstraints.length) continue;
|
|
2832
|
-
const sql = `ALTER TABLE "${table.name}" `;
|
|
2833
|
-
const constraints = [];
|
|
2834
|
-
for (const check of table.checkConstraints) {
|
|
2835
|
-
const constraint = `ADD CONSTRAINT "${check.name}" CHECK (${check.check})`;
|
|
2836
|
-
constraints.push(constraint);
|
|
2837
|
-
}
|
|
2838
|
-
sqlStatements.push(sql + constraints.join(",\n") + ";");
|
|
2839
|
-
}
|
|
2840
|
-
sqlStatements.push(indexes.join("\n"));
|
|
2841
|
-
sqlStatements.push(triggers.join("\n"));
|
|
2842
|
-
return sqlStatements.join("\n\n");
|
|
2843
|
-
}
|
|
2844
|
-
async function getNewPublicSchemaAndScratchPool(targetPool, scratchDbName) {
|
|
2845
|
-
const scratchDbExists = await targetPool.runQuery(
|
|
2846
|
-
`SELECT * FROM pg_database WHERE datname = ?;`,
|
|
2847
|
-
[scratchDbName],
|
|
2848
|
-
systemUser
|
|
2849
|
-
);
|
|
2850
|
-
if (scratchDbExists.length === 0) {
|
|
2851
|
-
await targetPool.runQuery(`CREATE DATABASE ${escapeColumnName(scratchDbName)};`, [], systemUser);
|
|
2852
|
-
}
|
|
2853
|
-
const scratchPool = new PsqlPool({
|
|
2854
|
-
host: targetPool.poolConfig.host,
|
|
2855
|
-
port: targetPool.poolConfig.port,
|
|
2856
|
-
user: targetPool.poolConfig.user,
|
|
2857
|
-
database: scratchDbName,
|
|
2858
|
-
password: targetPool.poolConfig.password,
|
|
2859
|
-
max: targetPool.poolConfig.max,
|
|
2860
|
-
idleTimeoutMillis: targetPool.poolConfig.idleTimeoutMillis,
|
|
2861
|
-
connectionTimeoutMillis: targetPool.poolConfig.connectionTimeoutMillis
|
|
2862
|
-
});
|
|
2863
|
-
await scratchPool.runQuery(`DROP SCHEMA public CASCADE;`, [], systemUser);
|
|
2864
|
-
await scratchPool.runQuery(
|
|
2865
|
-
`CREATE SCHEMA public AUTHORIZATION ${escapeColumnName(targetPool.poolConfig.user)};`,
|
|
2866
|
-
[],
|
|
2867
|
-
systemUser
|
|
2868
|
-
);
|
|
2869
|
-
const schemaComment = await targetPool.runQuery(
|
|
2870
|
-
`
|
|
2871
|
-
SELECT pg_description.description
|
|
2872
|
-
FROM pg_description
|
|
2873
|
-
JOIN pg_namespace ON pg_namespace.oid = pg_description.objoid
|
|
2874
|
-
WHERE pg_namespace.nspname = 'public';`,
|
|
2875
|
-
[],
|
|
2876
|
-
systemUser
|
|
2877
|
-
);
|
|
2878
|
-
if (schemaComment[0]?.description) {
|
|
2879
|
-
const escaped = schemaComment[0].description.replace(/'/g, "''");
|
|
2880
|
-
await scratchPool.runQuery(`COMMENT ON SCHEMA public IS '${escaped}';`, [], systemUser);
|
|
2881
|
-
}
|
|
2882
|
-
return scratchPool;
|
|
2883
|
-
}
|
|
2884
|
-
async function diffDatabaseToSchema(schema, targetPool, scratchDbName) {
|
|
2885
|
-
let scratchPool;
|
|
2886
|
-
let originalClient;
|
|
2887
|
-
let scratchClient;
|
|
2888
|
-
try {
|
|
2889
|
-
scratchPool = await getNewPublicSchemaAndScratchPool(targetPool, scratchDbName);
|
|
2890
|
-
const sqlFullStatement = generateDatabaseSchemaFromSchema(schema);
|
|
2891
|
-
await scratchPool.runQuery(sqlFullStatement, [], systemUser);
|
|
2892
|
-
const connectionConfig = {
|
|
2893
|
-
host: targetPool.poolConfig.host,
|
|
2894
|
-
port: targetPool.poolConfig.port,
|
|
2895
|
-
user: targetPool.poolConfig.user,
|
|
2896
|
-
password: targetPool.poolConfig.password,
|
|
2897
|
-
ssl: targetPool.poolConfig.ssl
|
|
2898
|
-
};
|
|
2899
|
-
originalClient = new Client({ ...connectionConfig, database: targetPool.poolConfig.database });
|
|
2900
|
-
scratchClient = new Client({ ...connectionConfig, database: scratchDbName });
|
|
2901
|
-
await Promise.all([originalClient.connect(), scratchClient.connect()]);
|
|
2902
|
-
const [info1, info2] = await Promise.all([
|
|
2903
|
-
pgInfo({ client: originalClient }),
|
|
2904
|
-
pgInfo({ client: scratchClient })
|
|
2905
|
-
]);
|
|
2906
|
-
const diff = getDiff(info1, info2);
|
|
2907
|
-
return diff.join("\n");
|
|
2908
|
-
} finally {
|
|
2909
|
-
const cleanups = [];
|
|
2910
|
-
if (originalClient) cleanups.push(originalClient.end());
|
|
2911
|
-
if (scratchClient) cleanups.push(scratchClient.end());
|
|
2912
|
-
if (scratchPool) cleanups.push(scratchPool.pool.end());
|
|
2913
|
-
await Promise.allSettled(cleanups);
|
|
2914
|
-
}
|
|
2915
|
-
}
|
|
3148
|
+
var fullGrammar = entryGrammar + oldGrammar + newGrammar;
|
|
3149
|
+
var filterPsqlParser = peg.generate(fullGrammar, {
|
|
3150
|
+
format: "commonjs",
|
|
3151
|
+
dependencies: { format: "pg-format" }
|
|
3152
|
+
});
|
|
3153
|
+
var filterPsqlParser_default = filterPsqlParser;
|
|
2916
3154
|
|
|
2917
3155
|
// src/restura/sql/PsqlEngine.ts
|
|
2918
|
-
var { Client:
|
|
3156
|
+
var { Client: Client3, types } = pg4;
|
|
2919
3157
|
var PsqlEngine = class extends SqlEngine {
|
|
2920
|
-
|
|
2921
|
-
constructor(psqlConnectionPool, shouldListenForDbTriggers = false, scratchDatabaseSuffix = "") {
|
|
3158
|
+
constructor(psqlConnectionPool, shouldListenForDbTriggers = false, scratchDatabaseSuffix = "", eventDelivery) {
|
|
2922
3159
|
super();
|
|
2923
3160
|
this.psqlConnectionPool = psqlConnectionPool;
|
|
3161
|
+
this.eventDelivery = eventDelivery || { mode: "direct" };
|
|
2924
3162
|
this.setupPgReturnTypes();
|
|
3163
|
+
if (this.eventDelivery.mode === "outbox") {
|
|
3164
|
+
this.outboxConsumer = new EventOutboxConsumer(psqlConnectionPool, this.eventDelivery.outbox);
|
|
3165
|
+
}
|
|
2925
3166
|
if (shouldListenForDbTriggers) {
|
|
2926
|
-
this.setupTriggerListeners = this.listenForDbTriggers()
|
|
3167
|
+
this.setupTriggerListeners = this.listenForDbTriggers().catch((error) => {
|
|
3168
|
+
logger.error(`Failed to setup trigger listeners: ${error}`);
|
|
3169
|
+
void this.reconnectTriggerClient();
|
|
3170
|
+
});
|
|
3171
|
+
this.outboxConsumer?.start();
|
|
2927
3172
|
}
|
|
2928
3173
|
this.scratchDbName = `${psqlConnectionPool.poolConfig.database}_scratch${scratchDatabaseSuffix ? `_${scratchDatabaseSuffix}` : ""}`;
|
|
2929
3174
|
}
|
|
@@ -2931,12 +3176,34 @@ var PsqlEngine = class extends SqlEngine {
|
|
|
2931
3176
|
triggerClient;
|
|
2932
3177
|
scratchDbName = "";
|
|
2933
3178
|
reconnectAttempts = 0;
|
|
2934
|
-
MAX_RECONNECT_ATTEMPTS = 5;
|
|
2935
3179
|
INITIAL_RECONNECT_DELAY = 5e3;
|
|
3180
|
+
MAX_RECONNECT_DELAY = 6e4;
|
|
3181
|
+
HEARTBEAT_INTERVAL_MS = 3e4;
|
|
3182
|
+
eventDelivery;
|
|
3183
|
+
outboxConsumer;
|
|
3184
|
+
heartbeatTimer;
|
|
3185
|
+
isListenerConnected = false;
|
|
3186
|
+
isReconnecting = false;
|
|
3187
|
+
isClosed = false;
|
|
3188
|
+
lastHeartbeatOn = null;
|
|
2936
3189
|
async close() {
|
|
3190
|
+
this.isClosed = true;
|
|
3191
|
+
if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
|
|
3192
|
+
if (this.outboxConsumer) await this.outboxConsumer.stop();
|
|
2937
3193
|
if (this.triggerClient) {
|
|
2938
3194
|
await this.triggerClient.end();
|
|
2939
3195
|
}
|
|
3196
|
+
this.isListenerConnected = false;
|
|
3197
|
+
}
|
|
3198
|
+
getEventListenerHealth() {
|
|
3199
|
+
return {
|
|
3200
|
+
connected: this.isListenerConnected,
|
|
3201
|
+
lastHeartbeatOn: this.lastHeartbeatOn,
|
|
3202
|
+
reconnectAttempts: this.reconnectAttempts
|
|
3203
|
+
};
|
|
3204
|
+
}
|
|
3205
|
+
getOutboxConsumer() {
|
|
3206
|
+
return this.outboxConsumer;
|
|
2940
3207
|
}
|
|
2941
3208
|
/**
|
|
2942
3209
|
* Setup the return types for the PostgreSQL connection.
|
|
@@ -2959,35 +3226,55 @@ var PsqlEngine = class extends SqlEngine {
|
|
|
2959
3226
|
types.setTypeParser(PG_TYPE_OID.TIMESTAMPTZ, (val) => val === null ? null : new Date(val).toISOString());
|
|
2960
3227
|
}
|
|
2961
3228
|
async reconnectTriggerClient() {
|
|
2962
|
-
if (this.
|
|
2963
|
-
|
|
2964
|
-
|
|
3229
|
+
if (this.isReconnecting || this.isClosed) return;
|
|
3230
|
+
this.isReconnecting = true;
|
|
3231
|
+
this.isListenerConnected = false;
|
|
3232
|
+
if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
|
|
3233
|
+
try {
|
|
3234
|
+
while (!this.isClosed) {
|
|
3235
|
+
if (this.triggerClient) {
|
|
3236
|
+
try {
|
|
3237
|
+
await this.triggerClient.end();
|
|
3238
|
+
} catch (error) {
|
|
3239
|
+
logger.error(`Error closing trigger client: ${error}`);
|
|
3240
|
+
}
|
|
3241
|
+
this.triggerClient = void 0;
|
|
3242
|
+
}
|
|
3243
|
+
const exponentialDelay = this.INITIAL_RECONNECT_DELAY * Math.pow(2, this.reconnectAttempts);
|
|
3244
|
+
const delay = Math.min(exponentialDelay, this.MAX_RECONNECT_DELAY) + Math.floor(Math.random() * 1e3);
|
|
3245
|
+
logger.info(
|
|
3246
|
+
`Attempting to reconnect trigger client in ${Math.round(delay / 1e3)} seconds... (attempt ${this.reconnectAttempts + 1})`
|
|
3247
|
+
);
|
|
3248
|
+
await new Promise((resolve2) => setTimeout(resolve2, delay));
|
|
3249
|
+
if (this.isClosed) return;
|
|
3250
|
+
this.reconnectAttempts++;
|
|
3251
|
+
try {
|
|
3252
|
+
await this.listenForDbTriggers();
|
|
3253
|
+
this.reconnectAttempts = 0;
|
|
3254
|
+
return;
|
|
3255
|
+
} catch (error) {
|
|
3256
|
+
logger.error(`Reconnection attempt ${this.reconnectAttempts} failed: ${error}`);
|
|
3257
|
+
}
|
|
3258
|
+
}
|
|
3259
|
+
} finally {
|
|
3260
|
+
this.isReconnecting = false;
|
|
2965
3261
|
}
|
|
2966
|
-
|
|
3262
|
+
}
|
|
3263
|
+
startHeartbeat() {
|
|
3264
|
+
if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
|
|
3265
|
+
this.heartbeatTimer = setInterval(async () => {
|
|
3266
|
+
if (!this.triggerClient || this.isClosed) return;
|
|
2967
3267
|
try {
|
|
2968
|
-
await this.triggerClient.
|
|
3268
|
+
await this.triggerClient.query("SELECT 1");
|
|
3269
|
+
this.lastHeartbeatOn = (/* @__PURE__ */ new Date()).toISOString();
|
|
2969
3270
|
} catch (error) {
|
|
2970
|
-
logger.error(`
|
|
2971
|
-
|
|
2972
|
-
}
|
|
2973
|
-
const delay = this.INITIAL_RECONNECT_DELAY * Math.pow(2, this.reconnectAttempts);
|
|
2974
|
-
logger.info(
|
|
2975
|
-
`Attempting to reconnect trigger client in ${delay / 1e3} seconds... (Attempt ${this.reconnectAttempts + 1}/${this.MAX_RECONNECT_ATTEMPTS})`
|
|
2976
|
-
);
|
|
2977
|
-
await new Promise((resolve2) => setTimeout(resolve2, delay));
|
|
2978
|
-
this.reconnectAttempts++;
|
|
2979
|
-
try {
|
|
2980
|
-
await this.listenForDbTriggers();
|
|
2981
|
-
this.reconnectAttempts = 0;
|
|
2982
|
-
} catch (error) {
|
|
2983
|
-
logger.error(`Reconnection attempt ${this.reconnectAttempts} failed: ${error}`);
|
|
2984
|
-
if (this.reconnectAttempts < this.MAX_RECONNECT_ATTEMPTS) {
|
|
2985
|
-
await this.reconnectTriggerClient();
|
|
3271
|
+
logger.error(`Trigger client heartbeat failed, reconnecting: ${error}`);
|
|
3272
|
+
void this.reconnectTriggerClient();
|
|
2986
3273
|
}
|
|
2987
|
-
}
|
|
3274
|
+
}, this.HEARTBEAT_INTERVAL_MS);
|
|
2988
3275
|
}
|
|
2989
3276
|
async listenForDbTriggers() {
|
|
2990
|
-
|
|
3277
|
+
const client = new Client3({
|
|
2991
3278
|
user: this.psqlConnectionPool.poolConfig.user,
|
|
2992
3279
|
host: this.psqlConnectionPool.poolConfig.host,
|
|
2993
3280
|
database: this.psqlConnectionPool.poolConfig.database,
|
|
@@ -2995,25 +3282,52 @@ var PsqlEngine = class extends SqlEngine {
|
|
|
2995
3282
|
port: this.psqlConnectionPool.poolConfig.port,
|
|
2996
3283
|
connectionTimeoutMillis: this.psqlConnectionPool.poolConfig.connectionTimeoutMillis
|
|
2997
3284
|
});
|
|
3285
|
+
this.triggerClient = client;
|
|
3286
|
+
client.on("error", (error) => {
|
|
3287
|
+
logger.error(`Trigger client error: ${error}`);
|
|
3288
|
+
void this.reconnectTriggerClient();
|
|
3289
|
+
});
|
|
3290
|
+
client.on("notification", async (msg) => {
|
|
3291
|
+
if (this.outboxConsumer && msg.channel === this.outboxConsumer.channel) {
|
|
3292
|
+
await this.outboxConsumer.drain();
|
|
3293
|
+
} else if (msg.channel === "insert" || msg.channel === "update" || msg.channel === "delete") {
|
|
3294
|
+
const payload = ObjectUtils3.safeParse(msg.payload);
|
|
3295
|
+
await this.handleTrigger(payload, msg.channel.toUpperCase());
|
|
3296
|
+
}
|
|
3297
|
+
});
|
|
3298
|
+
await client.connect();
|
|
3299
|
+
const channels = ["insert", "update", "delete"];
|
|
3300
|
+
if (this.outboxConsumer) channels.push(this.outboxConsumer.channel);
|
|
3301
|
+
for (const channel of channels) {
|
|
3302
|
+
await client.query(`LISTEN ${escapeColumnName(channel)}`);
|
|
3303
|
+
}
|
|
3304
|
+
this.isListenerConnected = true;
|
|
3305
|
+
this.lastHeartbeatOn = (/* @__PURE__ */ new Date()).toISOString();
|
|
3306
|
+
this.startHeartbeat();
|
|
3307
|
+
logger.info("Successfully connected to database triggers");
|
|
3308
|
+
void this.warnIfOutboxBacklogInDirectMode();
|
|
3309
|
+
}
|
|
3310
|
+
async warnIfOutboxBacklogInDirectMode() {
|
|
3311
|
+
if (this.eventDelivery.mode !== "direct") return;
|
|
2998
3312
|
try {
|
|
2999
|
-
await this.
|
|
3000
|
-
|
|
3001
|
-
|
|
3313
|
+
const tableExists = await this.psqlConnectionPool.runQuery(
|
|
3314
|
+
`SELECT to_regclass('public."dbEventOutbox"') AS exists;`,
|
|
3315
|
+
[],
|
|
3316
|
+
systemUser
|
|
3317
|
+
);
|
|
3318
|
+
if (!tableExists[0]?.exists) return;
|
|
3319
|
+
const pending = await this.psqlConnectionPool.runQuery(
|
|
3320
|
+
`SELECT COUNT(*)::int AS count FROM "dbEventOutbox" WHERE "processedOn" IS NULL;`,
|
|
3321
|
+
[],
|
|
3322
|
+
systemUser
|
|
3323
|
+
);
|
|
3324
|
+
if (Number(pending[0]?.count) > 0) {
|
|
3325
|
+
logger.warn(
|
|
3326
|
+
`eventDelivery is 'direct' but "dbEventOutbox" exists with ${pending[0].count} unprocessed rows \u2014 the database appears to have outbox-mode triggers installed. Switch eventDelivery to 'outbox' or regenerate direct-mode triggers.`
|
|
3327
|
+
);
|
|
3002
3328
|
}
|
|
3003
|
-
this.triggerClient.on("error", async (error) => {
|
|
3004
|
-
logger.error(`Trigger client error: ${error}`);
|
|
3005
|
-
await this.reconnectTriggerClient();
|
|
3006
|
-
});
|
|
3007
|
-
this.triggerClient.on("notification", async (msg) => {
|
|
3008
|
-
if (msg.channel === "insert" || msg.channel === "update" || msg.channel === "delete") {
|
|
3009
|
-
const payload = ObjectUtils3.safeParse(msg.payload);
|
|
3010
|
-
await this.handleTrigger(payload, msg.channel.toUpperCase());
|
|
3011
|
-
}
|
|
3012
|
-
});
|
|
3013
|
-
logger.info("Successfully connected to database triggers");
|
|
3014
3329
|
} catch (error) {
|
|
3015
|
-
logger.
|
|
3016
|
-
await this.reconnectTriggerClient();
|
|
3330
|
+
logger.warn(`Could not check dbEventOutbox backlog: ${error}`);
|
|
3017
3331
|
}
|
|
3018
3332
|
}
|
|
3019
3333
|
async handleTrigger(payload, mutationType) {
|
|
@@ -3027,10 +3341,21 @@ var PsqlEngine = class extends SqlEngine {
|
|
|
3027
3341
|
return sqlFullStatement;
|
|
3028
3342
|
}
|
|
3029
3343
|
generateDatabaseSchemaFromSchema(schema) {
|
|
3030
|
-
return generateDatabaseSchemaFromSchema(schema);
|
|
3344
|
+
return generateDatabaseSchemaFromSchema(schema, this.schemaGenerationOptions());
|
|
3031
3345
|
}
|
|
3032
3346
|
async diffDatabaseToSchema(schema) {
|
|
3033
|
-
return diffDatabaseToSchema(
|
|
3347
|
+
return diffDatabaseToSchema(
|
|
3348
|
+
schema,
|
|
3349
|
+
this.psqlConnectionPool,
|
|
3350
|
+
this.scratchDbName,
|
|
3351
|
+
this.schemaGenerationOptions()
|
|
3352
|
+
);
|
|
3353
|
+
}
|
|
3354
|
+
schemaGenerationOptions() {
|
|
3355
|
+
return {
|
|
3356
|
+
eventDelivery: this.eventDelivery.mode,
|
|
3357
|
+
outboxChannel: this.eventDelivery.outbox?.channel
|
|
3358
|
+
};
|
|
3034
3359
|
}
|
|
3035
3360
|
createNestedSelect(req, schema, item, routeData, sqlParams) {
|
|
3036
3361
|
if (!item.subquery) return "";
|
|
@@ -3468,7 +3793,13 @@ var ResturaEngine = class {
|
|
|
3468
3793
|
this.multerCommonUpload = getMulterUpload(this.resturaConfig.fileTempCachePath);
|
|
3469
3794
|
new TempCache(this.resturaConfig.fileTempCachePath);
|
|
3470
3795
|
this.psqlConnectionPool = psqlConnectionPool;
|
|
3471
|
-
|
|
3796
|
+
if (this.resturaConfig.queryMetadataKeys) {
|
|
3797
|
+
PsqlConnection.setQueryMetadataKeys(this.resturaConfig.queryMetadataKeys);
|
|
3798
|
+
}
|
|
3799
|
+
this.psqlEngine = new PsqlEngine(this.psqlConnectionPool, true, this.resturaConfig.scratchDatabaseSuffix, {
|
|
3800
|
+
mode: this.resturaConfig.eventDelivery,
|
|
3801
|
+
outbox: this.resturaConfig.eventOutbox
|
|
3802
|
+
});
|
|
3472
3803
|
await customApiFactory_default.loadApiFiles(this.resturaConfig.customApiFolderPath);
|
|
3473
3804
|
this.authenticationHandler = authenticationHandler;
|
|
3474
3805
|
app.use(compression());
|
|
@@ -3494,6 +3825,7 @@ var ResturaEngine = class {
|
|
|
3494
3825
|
this.expressApp = app;
|
|
3495
3826
|
await this.reloadEndpoints();
|
|
3496
3827
|
await this.initializeGeneratedTypesFolder();
|
|
3828
|
+
eventManager_default.validateHandlersAgainstSchema(this.schema, this.resturaConfig.notifyValidation);
|
|
3497
3829
|
logger.info("Restura Engine Initialized");
|
|
3498
3830
|
}
|
|
3499
3831
|
/**
|
|
@@ -3623,6 +3955,7 @@ var ResturaEngine = class {
|
|
|
3623
3955
|
}
|
|
3624
3956
|
async updateSchema(req, res) {
|
|
3625
3957
|
try {
|
|
3958
|
+
eventManager_default.validateHandlersAgainstSchema(req.data, this.resturaConfig.notifyValidation);
|
|
3626
3959
|
this.schema = sortObjectKeysAlphabetically(req.data);
|
|
3627
3960
|
await this.storeFileSystemSchema();
|
|
3628
3961
|
await this.reloadEndpoints();
|
|
@@ -4431,47 +4764,12 @@ function defaultsMatch(desired, live, column) {
|
|
|
4431
4764
|
const normalizedLive = live.replace(/::[a-z][a-z0-9_ ]*(\[\])?$/gi, "").trim();
|
|
4432
4765
|
return lowercaseOutsideStrings(desired.trim()) === lowercaseOutsideStrings(normalizedLive);
|
|
4433
4766
|
}
|
|
4434
|
-
|
|
4435
|
-
// src/restura/sql/PsqlTransaction.ts
|
|
4436
|
-
import pg4 from "pg";
|
|
4437
|
-
var { Client: Client3 } = pg4;
|
|
4438
|
-
var PsqlTransaction = class extends PsqlConnection {
|
|
4439
|
-
constructor(clientConfig, instanceId) {
|
|
4440
|
-
super(instanceId);
|
|
4441
|
-
this.clientConfig = clientConfig;
|
|
4442
|
-
this.client = new Client3(clientConfig);
|
|
4443
|
-
this.connectPromise = this.client.connect();
|
|
4444
|
-
this.beginTransactionPromise = this.beginTransaction();
|
|
4445
|
-
}
|
|
4446
|
-
client;
|
|
4447
|
-
beginTransactionPromise;
|
|
4448
|
-
connectPromise;
|
|
4449
|
-
async close() {
|
|
4450
|
-
if (this.client) {
|
|
4451
|
-
await this.client.end();
|
|
4452
|
-
}
|
|
4453
|
-
}
|
|
4454
|
-
async beginTransaction() {
|
|
4455
|
-
await this.connectPromise;
|
|
4456
|
-
return this.client.query("BEGIN");
|
|
4457
|
-
}
|
|
4458
|
-
async rollback() {
|
|
4459
|
-
return this.query("ROLLBACK");
|
|
4460
|
-
}
|
|
4461
|
-
async commit() {
|
|
4462
|
-
return this.query("COMMIT");
|
|
4463
|
-
}
|
|
4464
|
-
async release() {
|
|
4465
|
-
return this.client.end();
|
|
4466
|
-
}
|
|
4467
|
-
async query(query, values) {
|
|
4468
|
-
await this.connectPromise;
|
|
4469
|
-
await this.beginTransactionPromise;
|
|
4470
|
-
return this.client.query(query, values);
|
|
4471
|
-
}
|
|
4472
|
-
};
|
|
4473
4767
|
export {
|
|
4768
|
+
DEFAULT_OUTBOX_CHANNEL,
|
|
4769
|
+
EventManager,
|
|
4770
|
+
EventOutboxConsumer,
|
|
4474
4771
|
HtmlStatusCodes,
|
|
4772
|
+
OUTBOX_TABLE_NAME,
|
|
4475
4773
|
PsqlConnection,
|
|
4476
4774
|
PsqlEngine,
|
|
4477
4775
|
PsqlPool,
|
|
@@ -4481,17 +4779,21 @@ export {
|
|
|
4481
4779
|
apiGenerator,
|
|
4482
4780
|
createDeleteTriggerSql,
|
|
4483
4781
|
createInsertTriggerSql,
|
|
4782
|
+
createOutboxTableSql,
|
|
4484
4783
|
createUpdateTriggerSql,
|
|
4784
|
+
defaultEventOutboxOptions,
|
|
4485
4785
|
diffDatabaseToSchema,
|
|
4486
4786
|
diffSchemaToDatabase,
|
|
4487
4787
|
escapeColumnName,
|
|
4488
4788
|
eventManager_default as eventManager,
|
|
4489
4789
|
filterPsqlParser_default as filterPsqlParser,
|
|
4490
4790
|
generateDatabaseSchemaFromSchema,
|
|
4791
|
+
generateNotifyTriggersSql,
|
|
4491
4792
|
getNewPublicSchemaAndScratchPool,
|
|
4492
4793
|
insertObjectQuery,
|
|
4493
4794
|
introspectDatabase,
|
|
4494
4795
|
isSchemaValid,
|
|
4796
|
+
isSensitiveColumnName,
|
|
4495
4797
|
isValueNumber,
|
|
4496
4798
|
logger,
|
|
4497
4799
|
modelGenerator,
|