@happyvertical/smrt-reports 0.36.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +26 -0
- package/CLAUDE.md +1 -0
- package/LICENSE +7 -0
- package/dist/aggregate.d.ts +25 -0
- package/dist/aggregate.js +6 -0
- package/dist/aggregate.js.map +1 -0
- package/dist/compiler.d.ts +78 -0
- package/dist/compiler.js +218 -0
- package/dist/compiler.js.map +1 -0
- package/dist/decorators.d.ts +90 -0
- package/dist/decorators.js +84 -0
- package/dist/decorators.js.map +1 -0
- package/dist/index.d.ts +407 -0
- package/dist/index.js +162 -0
- package/dist/index.js.map +1 -0
- package/dist/manifest.json +1639 -0
- package/dist/refresh.d.ts +106 -0
- package/dist/refresh.js +778 -0
- package/dist/refresh.js.map +1 -0
- package/dist/scheduler.d.ts +112 -0
- package/dist/scheduler.js +449 -0
- package/dist/scheduler.js.map +1 -0
- package/dist/smrt-knowledge.json +584 -0
- package/dist/state.d.ts +108 -0
- package/dist/state.js +320 -0
- package/dist/state.js.map +1 -0
- package/dist/types.d.ts +114 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +87 -0
package/dist/refresh.js
ADDED
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { ObjectRegistry } from "@happyvertical/smrt-core";
|
|
3
|
+
import { toSnakeCase } from "@happyvertical/smrt-core/utils";
|
|
4
|
+
import { withTenant, getTenantId } from "@happyvertical/smrt-tenancy";
|
|
5
|
+
import { tableExists, buildAggregate, validateColumnName, buildWhere } from "@happyvertical/sql";
|
|
6
|
+
import { getReportGroupingColumns, buildReportDefinition, compileReportDefinition } from "./compiler.js";
|
|
7
|
+
import { assertReportTablesReady, REPORT_RUNS_TABLE, REPORT_WATERMARKS_TABLE, REPORT_LOCKS_TABLE, scopeKeyForTenant } from "./state.js";
|
|
8
|
+
function isSqlAdapterType(value) {
|
|
9
|
+
return value === "sqlite" || value === "postgres" || value === "duckdb" || value === "json";
|
|
10
|
+
}
|
|
11
|
+
function adapterTypeFromDb(db, fallback) {
|
|
12
|
+
if (isSqlAdapterType(fallback)) return fallback;
|
|
13
|
+
const url = String(db.url ?? "");
|
|
14
|
+
if (url.startsWith("postgres://") || url.startsWith("postgresql://")) {
|
|
15
|
+
return "postgres";
|
|
16
|
+
}
|
|
17
|
+
if (url.endsWith(".duckdb")) {
|
|
18
|
+
return "duckdb";
|
|
19
|
+
}
|
|
20
|
+
if (url === ":memory:" || url.endsWith(".sqlite") || url.endsWith(".db")) {
|
|
21
|
+
return "sqlite";
|
|
22
|
+
}
|
|
23
|
+
throw new Error(
|
|
24
|
+
"Report refresh requires a database adapter type. Pass { adapterType }."
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
function stableReportId(values) {
|
|
28
|
+
const hash = createHash("sha256").update(JSON.stringify(values)).digest("hex");
|
|
29
|
+
const variant = (Number.parseInt(hash[16], 16) & 3 | 8).toString(16);
|
|
30
|
+
return [
|
|
31
|
+
hash.slice(0, 8),
|
|
32
|
+
hash.slice(8, 12),
|
|
33
|
+
`4${hash.slice(13, 16)}`,
|
|
34
|
+
`${variant}${hash.slice(17, 20)}`,
|
|
35
|
+
hash.slice(20, 32)
|
|
36
|
+
].join("-");
|
|
37
|
+
}
|
|
38
|
+
function getReportTableName(reportCtor) {
|
|
39
|
+
const registered = ObjectRegistry.getClassByConstructor(reportCtor) ?? ObjectRegistry.getClass(reportCtor.name);
|
|
40
|
+
const tableName = registered ? ObjectRegistry.getTableName(registered.qualifiedName ?? registered.name) : ObjectRegistry.getTableName(reportCtor.name);
|
|
41
|
+
if (!tableName) {
|
|
42
|
+
throw new Error(`No report table registered for ${reportCtor.name}`);
|
|
43
|
+
}
|
|
44
|
+
return tableName;
|
|
45
|
+
}
|
|
46
|
+
function getActualGroupingColumns(definition) {
|
|
47
|
+
return definition.fields.filter(
|
|
48
|
+
(field) => field.report?.kind === "group" || field.report?.kind === "bucket"
|
|
49
|
+
).map((field) => field.columnName ?? toSnakeCase(field.fieldName));
|
|
50
|
+
}
|
|
51
|
+
function materializeRows(rows, definition, refreshedAt, scope) {
|
|
52
|
+
const groupingColumns = getActualGroupingColumns(definition);
|
|
53
|
+
const now = refreshedAt.toISOString();
|
|
54
|
+
return rows.map((row, index) => {
|
|
55
|
+
const groupingValues = groupingColumns.length === 0 ? [definition.reportClassName, scope.scopeKey, index] : [
|
|
56
|
+
definition.reportClassName,
|
|
57
|
+
scope.scopeKey,
|
|
58
|
+
...groupingColumns.map((column) => row[column])
|
|
59
|
+
];
|
|
60
|
+
const id = stableReportId(groupingValues);
|
|
61
|
+
const tenantFields = scope.reportTenantColumn ? { [scope.reportTenantColumn]: scope.tenantId } : {};
|
|
62
|
+
return {
|
|
63
|
+
id,
|
|
64
|
+
slug: id,
|
|
65
|
+
context: scope.scopeKey,
|
|
66
|
+
...tenantFields,
|
|
67
|
+
...row,
|
|
68
|
+
refreshed_at: now,
|
|
69
|
+
created_at: now,
|
|
70
|
+
updated_at: now
|
|
71
|
+
};
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
function sourceClassName(source) {
|
|
75
|
+
if (typeof source === "string") return source;
|
|
76
|
+
return source.name;
|
|
77
|
+
}
|
|
78
|
+
async function fieldColumn(className, requested) {
|
|
79
|
+
const fields = await ObjectRegistry.getAllFields(className);
|
|
80
|
+
const requestedColumn = toSnakeCase(requested);
|
|
81
|
+
for (const [fieldName] of fields.entries()) {
|
|
82
|
+
if (fieldName === requested || toSnakeCase(fieldName) === requestedColumn) {
|
|
83
|
+
return requestedColumn;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
async function tenantColumn(className) {
|
|
89
|
+
const registered = ObjectRegistry.getClass(className);
|
|
90
|
+
const configuredField = registered?.tenantScopedConfig?.field;
|
|
91
|
+
if (configuredField) {
|
|
92
|
+
return toSnakeCase(configuredField);
|
|
93
|
+
}
|
|
94
|
+
const fields = await ObjectRegistry.getAllFields(className);
|
|
95
|
+
for (const [fieldName, field] of fields.entries()) {
|
|
96
|
+
if (field?._meta?.__tenancy?.isTenantIdField) {
|
|
97
|
+
return toSnakeCase(fieldName);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const tenantIdField = await fieldColumn(className, "tenantId");
|
|
101
|
+
return tenantIdField;
|
|
102
|
+
}
|
|
103
|
+
async function resolveScope(definition, options) {
|
|
104
|
+
const tenantId = options.tenantId !== void 0 ? options.tenantId : getTenantId() ?? null;
|
|
105
|
+
const sourceTenantColumn = await tenantColumn(definition.sourceClassName);
|
|
106
|
+
const reportTenantColumn = await tenantColumn(definition.reportClassName);
|
|
107
|
+
if (tenantId && !sourceTenantColumn) {
|
|
108
|
+
throw new Error(
|
|
109
|
+
`Tenant-scoped refresh for ${definition.reportClassName} requires source ${definition.sourceClassName} to expose a tenantId field.`
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
if (tenantId && !reportTenantColumn) {
|
|
113
|
+
throw new Error(
|
|
114
|
+
`Tenant-scoped refresh for ${definition.reportClassName} requires the report table to expose a tenantId field.`
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
tenantId,
|
|
119
|
+
scopeKey: scopeKeyForTenant(tenantId),
|
|
120
|
+
sourceTenantColumn,
|
|
121
|
+
reportTenantColumn
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function tenantWhere(scope) {
|
|
125
|
+
if (!scope.sourceTenantColumn) return {};
|
|
126
|
+
return { [scope.sourceTenantColumn]: scope.tenantId };
|
|
127
|
+
}
|
|
128
|
+
function andWhere(base, extra) {
|
|
129
|
+
if (Object.keys(extra).length === 0) return base;
|
|
130
|
+
if (!base) return extra;
|
|
131
|
+
if (Array.isArray(base)) {
|
|
132
|
+
return base.map((andGroup) => [...andGroup, extra]);
|
|
133
|
+
}
|
|
134
|
+
return { ...base, ...extra };
|
|
135
|
+
}
|
|
136
|
+
function andHaving(base, extra) {
|
|
137
|
+
if (Object.keys(extra).length === 0) return base;
|
|
138
|
+
if (!base) return extra;
|
|
139
|
+
if (Array.isArray(base)) {
|
|
140
|
+
return base.map((andGroup) => [...andGroup, extra]);
|
|
141
|
+
}
|
|
142
|
+
return { ...base, ...extra };
|
|
143
|
+
}
|
|
144
|
+
function withRefreshFilters(spec, scope, extraWhere = {}) {
|
|
145
|
+
return {
|
|
146
|
+
...spec,
|
|
147
|
+
where: andWhere(spec.where, {
|
|
148
|
+
...tenantWhere(scope),
|
|
149
|
+
...extraWhere
|
|
150
|
+
})
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
async function resolveWatermarkConfig(definition) {
|
|
154
|
+
const configured = definition.refresh ?? {};
|
|
155
|
+
const watermarkField = configured.watermarkColumn ?? "updatedAt";
|
|
156
|
+
const softDeleteField = configured.softDeleteColumn ?? "deletedAt";
|
|
157
|
+
const watermarkColumn = await fieldColumn(
|
|
158
|
+
definition.sourceClassName,
|
|
159
|
+
watermarkField
|
|
160
|
+
);
|
|
161
|
+
const softDeleteColumn = await fieldColumn(
|
|
162
|
+
definition.sourceClassName,
|
|
163
|
+
softDeleteField
|
|
164
|
+
);
|
|
165
|
+
if (!watermarkColumn) {
|
|
166
|
+
throw new Error(
|
|
167
|
+
`Incremental report ${definition.reportClassName} requires source ${definition.sourceClassName} to define watermark column '${watermarkField}'.`
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
if (!softDeleteColumn) {
|
|
171
|
+
throw new Error(
|
|
172
|
+
`Incremental report ${definition.reportClassName} requires source ${definition.sourceClassName} to define soft-delete column '${softDeleteField}'.`
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
return { watermarkColumn, softDeleteColumn };
|
|
176
|
+
}
|
|
177
|
+
function compileChangedGroupSpec(definition, scope, watermark, watermarkBefore) {
|
|
178
|
+
const spec = compileReportDefinition(definition);
|
|
179
|
+
const select = spec.select.filter(
|
|
180
|
+
(expr) => !("fn" in expr)
|
|
181
|
+
);
|
|
182
|
+
return withRefreshFilters(
|
|
183
|
+
{
|
|
184
|
+
...spec,
|
|
185
|
+
// Ignore the report filter while finding changed groups so rows that
|
|
186
|
+
// just stopped matching `report.where` still trigger a recompute/delete.
|
|
187
|
+
where: void 0,
|
|
188
|
+
select: select.length > 0 ? select : [{ fn: "count", as: "__smrt_changed_count" }],
|
|
189
|
+
groupBy: select.length > 0 ? spec.groupBy : [],
|
|
190
|
+
having: void 0
|
|
191
|
+
},
|
|
192
|
+
scope,
|
|
193
|
+
{ [`${watermark.watermarkColumn} >`]: watermarkBefore }
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
function compileAffectedGroupSpec(definition, scope, watermark, groupRow) {
|
|
197
|
+
const groupingColumns = getActualGroupingColumns(definition);
|
|
198
|
+
const groupHaving = Object.fromEntries(
|
|
199
|
+
groupingColumns.map((column) => [column, groupRow[column]])
|
|
200
|
+
);
|
|
201
|
+
const spec = withRefreshFilters(compileReportDefinition(definition), scope, {
|
|
202
|
+
[watermark.softDeleteColumn]: null
|
|
203
|
+
});
|
|
204
|
+
return {
|
|
205
|
+
...spec,
|
|
206
|
+
having: andHaving(spec.having, groupHaving)
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
function conflictColumns(definition, scope) {
|
|
210
|
+
const groupingColumns = getActualGroupingColumns(definition);
|
|
211
|
+
if (groupingColumns.length === 0) return ["id"];
|
|
212
|
+
return [
|
|
213
|
+
...scope.reportTenantColumn ? [scope.reportTenantColumn] : [],
|
|
214
|
+
...groupingColumns
|
|
215
|
+
];
|
|
216
|
+
}
|
|
217
|
+
function predicateSql(tableName, predicate) {
|
|
218
|
+
validateColumnName(tableName);
|
|
219
|
+
const clauses = [];
|
|
220
|
+
const values = [];
|
|
221
|
+
for (const [column, value] of Object.entries(predicate)) {
|
|
222
|
+
validateColumnName(column);
|
|
223
|
+
if (value === null || value === void 0) {
|
|
224
|
+
clauses.push(`${column} IS NULL`);
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
clauses.push(`${column} = ?`);
|
|
228
|
+
values.push(value);
|
|
229
|
+
}
|
|
230
|
+
return {
|
|
231
|
+
sql: clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "",
|
|
232
|
+
values
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
function dedupeGroupRows(definition, rows) {
|
|
236
|
+
const groupingColumns = getActualGroupingColumns(definition);
|
|
237
|
+
const groups = /* @__PURE__ */ new Map();
|
|
238
|
+
for (const row of rows) {
|
|
239
|
+
if (!row) continue;
|
|
240
|
+
const groupRow = groupingColumns.length === 0 ? {} : Object.fromEntries(
|
|
241
|
+
groupingColumns.map((column) => [column, row[column] ?? null])
|
|
242
|
+
);
|
|
243
|
+
const key = groupingColumns.length === 0 ? "__smrt_global_group__" : JSON.stringify(groupingColumns.map((column) => groupRow[column]));
|
|
244
|
+
if (!groups.has(key)) {
|
|
245
|
+
groups.set(key, groupRow);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return [...groups.values()];
|
|
249
|
+
}
|
|
250
|
+
async function readMaterializedGroups(db, tableName, definition, scope) {
|
|
251
|
+
const groupingColumns = getActualGroupingColumns(definition);
|
|
252
|
+
if (groupingColumns.length === 0) {
|
|
253
|
+
return [{}];
|
|
254
|
+
}
|
|
255
|
+
const where = scope.reportTenantColumn ? predicateSql(tableName, { [scope.reportTenantColumn]: scope.tenantId }) : { sql: "", values: [] };
|
|
256
|
+
const columns = groupingColumns.map((column) => validateColumnName(column));
|
|
257
|
+
const result = await db.query(
|
|
258
|
+
`SELECT ${columns.join(", ")} FROM ${validateColumnName(tableName)} ${where.sql}`,
|
|
259
|
+
...where.values
|
|
260
|
+
);
|
|
261
|
+
return result.rows.map((row) => parseMaybeJson(row)).filter((row) => Boolean(row));
|
|
262
|
+
}
|
|
263
|
+
async function deleteMaterializedGroup(db, tableName, definition, scope, groupRow) {
|
|
264
|
+
const predicate = {};
|
|
265
|
+
if (scope.reportTenantColumn) {
|
|
266
|
+
predicate[scope.reportTenantColumn] = scope.tenantId;
|
|
267
|
+
}
|
|
268
|
+
for (const column of getActualGroupingColumns(definition)) {
|
|
269
|
+
predicate[column] = groupRow[column];
|
|
270
|
+
}
|
|
271
|
+
const where = predicateSql(tableName, predicate);
|
|
272
|
+
await db.query(
|
|
273
|
+
`DELETE FROM ${validateColumnName(tableName)} ${where.sql}`,
|
|
274
|
+
...where.values
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
async function replaceRows(db, tableName, rows, scope) {
|
|
278
|
+
validateColumnName(tableName);
|
|
279
|
+
const run = async (tx) => {
|
|
280
|
+
if (scope.reportTenantColumn) {
|
|
281
|
+
const where = predicateSql(tableName, {
|
|
282
|
+
[scope.reportTenantColumn]: scope.tenantId
|
|
283
|
+
});
|
|
284
|
+
await tx.query(`DELETE FROM ${tableName} ${where.sql}`, ...where.values);
|
|
285
|
+
} else {
|
|
286
|
+
await tx.query(`DELETE FROM ${tableName}`);
|
|
287
|
+
}
|
|
288
|
+
if (rows.length > 0) {
|
|
289
|
+
await tx.insert(tableName, rows);
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
if (db.transaction) {
|
|
293
|
+
await db.transaction(run);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
await run(db);
|
|
297
|
+
}
|
|
298
|
+
function parseMaybeJson(value) {
|
|
299
|
+
if (value == null) return null;
|
|
300
|
+
if (typeof value !== "string") return value;
|
|
301
|
+
try {
|
|
302
|
+
return JSON.parse(value);
|
|
303
|
+
} catch {
|
|
304
|
+
return value;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
function watermarkValue(value) {
|
|
308
|
+
if (value === null || value === void 0) return null;
|
|
309
|
+
if (value instanceof Date) return value.toISOString();
|
|
310
|
+
return String(value);
|
|
311
|
+
}
|
|
312
|
+
async function readWatermark(db, definition, scope, watermarkColumn) {
|
|
313
|
+
const result = await db.query(
|
|
314
|
+
`SELECT watermark_value FROM ${REPORT_WATERMARKS_TABLE}
|
|
315
|
+
WHERE report_class = ?
|
|
316
|
+
AND scope_key = ?
|
|
317
|
+
AND source_class = ?
|
|
318
|
+
AND watermark_column = ?
|
|
319
|
+
LIMIT 1`,
|
|
320
|
+
definition.reportClassName,
|
|
321
|
+
scope.scopeKey,
|
|
322
|
+
definition.sourceClassName,
|
|
323
|
+
watermarkColumn
|
|
324
|
+
);
|
|
325
|
+
const row = result.rows[0];
|
|
326
|
+
return watermarkValue(row?.watermark_value);
|
|
327
|
+
}
|
|
328
|
+
async function writeWatermark(db, definition, scope, watermarkColumn, value, runId) {
|
|
329
|
+
if (value === null) return;
|
|
330
|
+
const id = stableReportId([
|
|
331
|
+
"watermark",
|
|
332
|
+
definition.reportClassName,
|
|
333
|
+
scope.scopeKey,
|
|
334
|
+
definition.sourceClassName,
|
|
335
|
+
watermarkColumn
|
|
336
|
+
]);
|
|
337
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
338
|
+
await db.upsert(
|
|
339
|
+
REPORT_WATERMARKS_TABLE,
|
|
340
|
+
["report_class", "scope_key", "source_class", "watermark_column"],
|
|
341
|
+
{
|
|
342
|
+
id,
|
|
343
|
+
slug: id,
|
|
344
|
+
context: scope.scopeKey,
|
|
345
|
+
tenant_id: scope.tenantId,
|
|
346
|
+
scope_key: scope.scopeKey,
|
|
347
|
+
report_class: definition.reportClassName,
|
|
348
|
+
source_class: definition.sourceClassName,
|
|
349
|
+
watermark_column: watermarkColumn,
|
|
350
|
+
watermark_value: value,
|
|
351
|
+
last_run_id: runId ?? null,
|
|
352
|
+
created_at: now,
|
|
353
|
+
updated_at: now
|
|
354
|
+
}
|
|
355
|
+
);
|
|
356
|
+
}
|
|
357
|
+
async function maxWatermark(db, definition, scope, watermarkColumn, adapterType) {
|
|
358
|
+
const where = andWhere(void 0, tenantWhere(scope));
|
|
359
|
+
const builtWhere = where ? buildWhere(where, 1, adapterType) : { sql: "", values: [] };
|
|
360
|
+
const result = await db.query(
|
|
361
|
+
`SELECT MAX(${validateColumnName(watermarkColumn)}) AS watermark FROM ${validateColumnName(definition.sourceTable)} ${builtWhere.sql}`,
|
|
362
|
+
...builtWhere.values
|
|
363
|
+
);
|
|
364
|
+
const row = result.rows[0];
|
|
365
|
+
return watermarkValue(row?.watermark);
|
|
366
|
+
}
|
|
367
|
+
async function startRun(db, definition, scope, options, mode, watermarkBefore) {
|
|
368
|
+
if (options.trackRuns === false) return void 0;
|
|
369
|
+
const id = randomUUID();
|
|
370
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
371
|
+
await db.insert(REPORT_RUNS_TABLE, {
|
|
372
|
+
id,
|
|
373
|
+
slug: id,
|
|
374
|
+
context: scope.scopeKey,
|
|
375
|
+
tenant_id: scope.tenantId,
|
|
376
|
+
scope_key: scope.scopeKey,
|
|
377
|
+
report_class: definition.reportClassName,
|
|
378
|
+
source_class: definition.sourceClassName,
|
|
379
|
+
mode,
|
|
380
|
+
trigger: options.trigger ?? "manual",
|
|
381
|
+
status: "running",
|
|
382
|
+
started_at: now,
|
|
383
|
+
row_count: 0,
|
|
384
|
+
changed_group_count: 0,
|
|
385
|
+
watermark_before: null,
|
|
386
|
+
watermark_after: null,
|
|
387
|
+
error: null,
|
|
388
|
+
metadata: {
|
|
389
|
+
scheduleId: options.scheduleId
|
|
390
|
+
},
|
|
391
|
+
created_at: now,
|
|
392
|
+
updated_at: now
|
|
393
|
+
});
|
|
394
|
+
return id;
|
|
395
|
+
}
|
|
396
|
+
async function completeRun(db, runId, status, data = {}) {
|
|
397
|
+
if (!runId) return;
|
|
398
|
+
const error = data.error instanceof Error ? data.error.message : data.error ? String(data.error) : null;
|
|
399
|
+
await db.update(
|
|
400
|
+
REPORT_RUNS_TABLE,
|
|
401
|
+
{ id: runId },
|
|
402
|
+
{
|
|
403
|
+
status,
|
|
404
|
+
completed_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
405
|
+
row_count: data.rowCount ?? 0,
|
|
406
|
+
changed_group_count: data.changedGroupCount ?? 0,
|
|
407
|
+
watermark_after: data.watermarkAfter ?? null,
|
|
408
|
+
error,
|
|
409
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
410
|
+
}
|
|
411
|
+
);
|
|
412
|
+
}
|
|
413
|
+
async function acquireLock(db, definition, scope, ttlMs) {
|
|
414
|
+
const now = /* @__PURE__ */ new Date();
|
|
415
|
+
const nowIso = now.toISOString();
|
|
416
|
+
const ownerId = randomUUID();
|
|
417
|
+
const expiresAt = new Date(now.getTime() + ttlMs);
|
|
418
|
+
const expiresAtIso = expiresAt.toISOString();
|
|
419
|
+
const id = stableReportId([
|
|
420
|
+
"lock",
|
|
421
|
+
definition.reportClassName,
|
|
422
|
+
scope.scopeKey
|
|
423
|
+
]);
|
|
424
|
+
await db.query(
|
|
425
|
+
`INSERT INTO ${REPORT_LOCKS_TABLE} (
|
|
426
|
+
id,
|
|
427
|
+
slug,
|
|
428
|
+
context,
|
|
429
|
+
tenant_id,
|
|
430
|
+
scope_key,
|
|
431
|
+
report_class,
|
|
432
|
+
owner_id,
|
|
433
|
+
acquired_at,
|
|
434
|
+
heartbeat_at,
|
|
435
|
+
expires_at,
|
|
436
|
+
created_at,
|
|
437
|
+
updated_at
|
|
438
|
+
)
|
|
439
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
440
|
+
ON CONFLICT(report_class, scope_key) DO UPDATE SET
|
|
441
|
+
tenant_id = excluded.tenant_id,
|
|
442
|
+
owner_id = excluded.owner_id,
|
|
443
|
+
acquired_at = excluded.acquired_at,
|
|
444
|
+
heartbeat_at = excluded.heartbeat_at,
|
|
445
|
+
expires_at = excluded.expires_at,
|
|
446
|
+
updated_at = excluded.updated_at
|
|
447
|
+
WHERE ${REPORT_LOCKS_TABLE}.owner_id IS NULL
|
|
448
|
+
OR ${REPORT_LOCKS_TABLE}.expires_at IS NULL
|
|
449
|
+
OR ${REPORT_LOCKS_TABLE}.expires_at <= ?`,
|
|
450
|
+
id,
|
|
451
|
+
id,
|
|
452
|
+
scope.scopeKey,
|
|
453
|
+
scope.tenantId,
|
|
454
|
+
scope.scopeKey,
|
|
455
|
+
definition.reportClassName,
|
|
456
|
+
ownerId,
|
|
457
|
+
nowIso,
|
|
458
|
+
nowIso,
|
|
459
|
+
expiresAtIso,
|
|
460
|
+
nowIso,
|
|
461
|
+
nowIso,
|
|
462
|
+
nowIso
|
|
463
|
+
);
|
|
464
|
+
const claimed = await db.query(
|
|
465
|
+
`SELECT owner_id FROM ${REPORT_LOCKS_TABLE}
|
|
466
|
+
WHERE report_class = ? AND scope_key = ?
|
|
467
|
+
LIMIT 1`,
|
|
468
|
+
definition.reportClassName,
|
|
469
|
+
scope.scopeKey
|
|
470
|
+
);
|
|
471
|
+
const row = claimed.rows[0];
|
|
472
|
+
if (row?.owner_id !== ownerId) {
|
|
473
|
+
return null;
|
|
474
|
+
}
|
|
475
|
+
return {
|
|
476
|
+
ownerId,
|
|
477
|
+
async release() {
|
|
478
|
+
await db.query(
|
|
479
|
+
`UPDATE ${REPORT_LOCKS_TABLE}
|
|
480
|
+
SET owner_id = NULL,
|
|
481
|
+
heartbeat_at = NULL,
|
|
482
|
+
expires_at = NULL,
|
|
483
|
+
updated_at = ?
|
|
484
|
+
WHERE report_class = ?
|
|
485
|
+
AND scope_key = ?
|
|
486
|
+
AND owner_id = ?`,
|
|
487
|
+
(/* @__PURE__ */ new Date()).toISOString(),
|
|
488
|
+
definition.reportClassName,
|
|
489
|
+
scope.scopeKey,
|
|
490
|
+
ownerId
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
async function rebuildReport(db, definition, reportTable, scope, adapterType, runId, watermark) {
|
|
496
|
+
const explicitWatermarkColumn = definition.refresh?.watermarkColumn && await fieldColumn(
|
|
497
|
+
definition.sourceClassName,
|
|
498
|
+
definition.refresh.watermarkColumn
|
|
499
|
+
);
|
|
500
|
+
const explicitSoftDeleteColumn = definition.refresh?.softDeleteColumn && await fieldColumn(
|
|
501
|
+
definition.sourceClassName,
|
|
502
|
+
definition.refresh.softDeleteColumn
|
|
503
|
+
);
|
|
504
|
+
const watermarkColumn = watermark?.watermarkColumn ?? explicitWatermarkColumn ?? null;
|
|
505
|
+
const softDeleteColumn = watermark?.softDeleteColumn ?? explicitSoftDeleteColumn ?? null;
|
|
506
|
+
const deletedFilter = softDeleteColumn ? { [softDeleteColumn]: null } : {};
|
|
507
|
+
const spec = withRefreshFilters(
|
|
508
|
+
compileReportDefinition(definition),
|
|
509
|
+
scope,
|
|
510
|
+
deletedFilter
|
|
511
|
+
);
|
|
512
|
+
const aggregate = buildAggregate(spec, 1, adapterType);
|
|
513
|
+
const result = await db.query(aggregate.sql, ...aggregate.values);
|
|
514
|
+
const refreshedAt = /* @__PURE__ */ new Date();
|
|
515
|
+
const rows = materializeRows(result.rows, definition, refreshedAt, scope);
|
|
516
|
+
await replaceRows(db, reportTable, rows, scope);
|
|
517
|
+
const watermarkAfter = watermarkColumn ? await maxWatermark(db, definition, scope, watermarkColumn, adapterType) : null;
|
|
518
|
+
if (watermarkColumn) {
|
|
519
|
+
await writeWatermark(
|
|
520
|
+
db,
|
|
521
|
+
definition,
|
|
522
|
+
scope,
|
|
523
|
+
watermarkColumn,
|
|
524
|
+
watermarkAfter,
|
|
525
|
+
runId
|
|
526
|
+
);
|
|
527
|
+
}
|
|
528
|
+
return {
|
|
529
|
+
rowCount: rows.length,
|
|
530
|
+
changedGroupCount: rows.length,
|
|
531
|
+
watermarkAfter,
|
|
532
|
+
mode: "rebuild"
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
async function incrementalReport(db, definition, reportTable, scope, adapterType, runId, changedRows = []) {
|
|
536
|
+
const watermark = await resolveWatermarkConfig(definition);
|
|
537
|
+
const watermarkBefore = await readWatermark(
|
|
538
|
+
db,
|
|
539
|
+
definition,
|
|
540
|
+
scope,
|
|
541
|
+
watermark.watermarkColumn
|
|
542
|
+
);
|
|
543
|
+
if (!watermarkBefore) {
|
|
544
|
+
const seeded = await rebuildReport(
|
|
545
|
+
db,
|
|
546
|
+
definition,
|
|
547
|
+
reportTable,
|
|
548
|
+
scope,
|
|
549
|
+
adapterType,
|
|
550
|
+
runId,
|
|
551
|
+
watermark
|
|
552
|
+
);
|
|
553
|
+
return {
|
|
554
|
+
...seeded,
|
|
555
|
+
watermarkBefore
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
const changedSpec = compileChangedGroupSpec(
|
|
559
|
+
definition,
|
|
560
|
+
scope,
|
|
561
|
+
watermark,
|
|
562
|
+
watermarkBefore
|
|
563
|
+
);
|
|
564
|
+
const changedAggregate = buildAggregate(changedSpec, 1, adapterType);
|
|
565
|
+
const changed = await db.query(
|
|
566
|
+
changedAggregate.sql,
|
|
567
|
+
...changedAggregate.values
|
|
568
|
+
);
|
|
569
|
+
const changedGroups = changed.rows.map(
|
|
570
|
+
(row) => parseMaybeJson(row)
|
|
571
|
+
);
|
|
572
|
+
const materializedGroups = changedGroups.length > 0 || changedRows.length > 0 ? await readMaterializedGroups(db, reportTable, definition, scope) : [];
|
|
573
|
+
const affectedGroups = dedupeGroupRows(definition, [
|
|
574
|
+
...changedGroups,
|
|
575
|
+
...materializedGroups
|
|
576
|
+
]);
|
|
577
|
+
if (affectedGroups.length === 0) {
|
|
578
|
+
return {
|
|
579
|
+
rowCount: 0,
|
|
580
|
+
changedGroupCount: 0,
|
|
581
|
+
watermarkBefore,
|
|
582
|
+
watermarkAfter: watermarkBefore,
|
|
583
|
+
mode: "incremental"
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
let rowCount = 0;
|
|
587
|
+
const conflicts = conflictColumns(definition, scope);
|
|
588
|
+
for (const groupRow of affectedGroups) {
|
|
589
|
+
const affectedSpec = compileAffectedGroupSpec(
|
|
590
|
+
definition,
|
|
591
|
+
scope,
|
|
592
|
+
watermark,
|
|
593
|
+
groupRow
|
|
594
|
+
);
|
|
595
|
+
const aggregate = buildAggregate(affectedSpec, 1, adapterType);
|
|
596
|
+
const result = await db.query(aggregate.sql, ...aggregate.values);
|
|
597
|
+
const refreshedAt = /* @__PURE__ */ new Date();
|
|
598
|
+
const rows = materializeRows(result.rows, definition, refreshedAt, scope);
|
|
599
|
+
if (rows.length === 0) {
|
|
600
|
+
await deleteMaterializedGroup(
|
|
601
|
+
db,
|
|
602
|
+
reportTable,
|
|
603
|
+
definition,
|
|
604
|
+
scope,
|
|
605
|
+
groupRow
|
|
606
|
+
);
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
for (const row of rows) {
|
|
610
|
+
await db.upsert(reportTable, conflicts, row);
|
|
611
|
+
rowCount += 1;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
const watermarkAfter = await maxWatermark(
|
|
615
|
+
db,
|
|
616
|
+
definition,
|
|
617
|
+
scope,
|
|
618
|
+
watermark.watermarkColumn,
|
|
619
|
+
adapterType
|
|
620
|
+
);
|
|
621
|
+
await writeWatermark(
|
|
622
|
+
db,
|
|
623
|
+
definition,
|
|
624
|
+
scope,
|
|
625
|
+
watermark.watermarkColumn,
|
|
626
|
+
watermarkAfter,
|
|
627
|
+
runId
|
|
628
|
+
);
|
|
629
|
+
return {
|
|
630
|
+
rowCount,
|
|
631
|
+
changedGroupCount: affectedGroups.length,
|
|
632
|
+
watermarkBefore,
|
|
633
|
+
watermarkAfter,
|
|
634
|
+
mode: "incremental"
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
function requiredReportRuntimeTables(mode, options) {
|
|
638
|
+
const tables = /* @__PURE__ */ new Set();
|
|
639
|
+
if (options.trackRuns !== false) tables.add(REPORT_RUNS_TABLE);
|
|
640
|
+
if (mode === "incremental") tables.add(REPORT_WATERMARKS_TABLE);
|
|
641
|
+
if (options.lock !== false) tables.add(REPORT_LOCKS_TABLE);
|
|
642
|
+
return [...tables];
|
|
643
|
+
}
|
|
644
|
+
async function refreshReportOnce(reportCtor, options) {
|
|
645
|
+
if (!options.db) {
|
|
646
|
+
throw new Error("refreshReport requires a database handle");
|
|
647
|
+
}
|
|
648
|
+
const definition = await buildReportDefinition(reportCtor);
|
|
649
|
+
const reportTable = getReportTableName(reportCtor);
|
|
650
|
+
const requestedMode = options.mode ?? definition.refresh?.mode ?? "rebuild";
|
|
651
|
+
const exists = await tableExists(options.db, reportTable);
|
|
652
|
+
if (!exists) {
|
|
653
|
+
throw new Error(
|
|
654
|
+
`Report table '${reportTable}' does not exist for ${reportCtor.name}. Run smrt db:migrate before refreshing reports.`
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
const requiredTables = requiredReportRuntimeTables(requestedMode, options);
|
|
658
|
+
if (requiredTables.length > 0) {
|
|
659
|
+
await assertReportTablesReady(options.db, requiredTables);
|
|
660
|
+
}
|
|
661
|
+
const adapterType = adapterTypeFromDb(options.db, options.adapterType);
|
|
662
|
+
const scope = await resolveScope(definition, options);
|
|
663
|
+
const lock = options.lock === false ? null : await acquireLock(
|
|
664
|
+
options.db,
|
|
665
|
+
definition,
|
|
666
|
+
scope,
|
|
667
|
+
options.lockTtlMs ?? 15 * 60 * 1e3
|
|
668
|
+
);
|
|
669
|
+
if (options.lock !== false && !lock) {
|
|
670
|
+
const runId2 = await startRun(
|
|
671
|
+
options.db,
|
|
672
|
+
definition,
|
|
673
|
+
scope,
|
|
674
|
+
options,
|
|
675
|
+
requestedMode
|
|
676
|
+
);
|
|
677
|
+
await completeRun(options.db, runId2, "skipped");
|
|
678
|
+
return {
|
|
679
|
+
rowCount: 0,
|
|
680
|
+
refreshedAt: /* @__PURE__ */ new Date(),
|
|
681
|
+
mode: requestedMode,
|
|
682
|
+
tenantId: scope.tenantId,
|
|
683
|
+
runId: runId2,
|
|
684
|
+
changedGroupCount: 0,
|
|
685
|
+
skipped: true
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
let runId;
|
|
689
|
+
try {
|
|
690
|
+
runId = await startRun(
|
|
691
|
+
options.db,
|
|
692
|
+
definition,
|
|
693
|
+
scope,
|
|
694
|
+
options,
|
|
695
|
+
requestedMode
|
|
696
|
+
);
|
|
697
|
+
const work = requestedMode === "incremental" ? await incrementalReport(
|
|
698
|
+
options.db,
|
|
699
|
+
definition,
|
|
700
|
+
reportTable,
|
|
701
|
+
scope,
|
|
702
|
+
adapterType,
|
|
703
|
+
runId,
|
|
704
|
+
options.changedRows
|
|
705
|
+
) : await rebuildReport(
|
|
706
|
+
options.db,
|
|
707
|
+
definition,
|
|
708
|
+
reportTable,
|
|
709
|
+
scope,
|
|
710
|
+
adapterType,
|
|
711
|
+
runId
|
|
712
|
+
);
|
|
713
|
+
await completeRun(options.db, runId, "success", {
|
|
714
|
+
rowCount: work.rowCount,
|
|
715
|
+
changedGroupCount: work.changedGroupCount,
|
|
716
|
+
watermarkAfter: work.watermarkAfter
|
|
717
|
+
});
|
|
718
|
+
return {
|
|
719
|
+
rowCount: work.rowCount,
|
|
720
|
+
refreshedAt: /* @__PURE__ */ new Date(),
|
|
721
|
+
mode: work.mode,
|
|
722
|
+
tenantId: scope.tenantId,
|
|
723
|
+
runId,
|
|
724
|
+
changedGroupCount: work.changedGroupCount
|
|
725
|
+
};
|
|
726
|
+
} catch (error) {
|
|
727
|
+
await completeRun(options.db, runId, "failed", { error });
|
|
728
|
+
throw error;
|
|
729
|
+
} finally {
|
|
730
|
+
await lock?.release();
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
async function refreshReport(reportCtor, options = {}) {
|
|
734
|
+
if (options.tenantIds && options.tenantIds.length > 0) {
|
|
735
|
+
if (!options.db) {
|
|
736
|
+
throw new Error("refreshReport requires a database handle");
|
|
737
|
+
}
|
|
738
|
+
const results = [];
|
|
739
|
+
for (const tenantId of options.tenantIds) {
|
|
740
|
+
const result = await withTenant(
|
|
741
|
+
{ tenantId },
|
|
742
|
+
() => refreshReportOnce(reportCtor, {
|
|
743
|
+
...options,
|
|
744
|
+
tenantId,
|
|
745
|
+
tenantIds: void 0
|
|
746
|
+
})
|
|
747
|
+
);
|
|
748
|
+
results.push(result);
|
|
749
|
+
}
|
|
750
|
+
return {
|
|
751
|
+
rowCount: results.reduce((sum, result) => sum + result.rowCount, 0),
|
|
752
|
+
refreshedAt: /* @__PURE__ */ new Date(),
|
|
753
|
+
mode: results[0]?.mode ?? options.mode ?? "rebuild",
|
|
754
|
+
tenantId: null,
|
|
755
|
+
changedGroupCount: results.reduce(
|
|
756
|
+
(sum, result) => sum + (result.changedGroupCount ?? 0),
|
|
757
|
+
0
|
|
758
|
+
),
|
|
759
|
+
tenantResults: results
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
return refreshReportOnce(reportCtor, options);
|
|
763
|
+
}
|
|
764
|
+
function reportRowIdentity(row, definition) {
|
|
765
|
+
const groupingColumns = getReportGroupingColumns(definition);
|
|
766
|
+
return stableReportId(
|
|
767
|
+
groupingColumns.map((column) => row[toSnakeCase(column)])
|
|
768
|
+
);
|
|
769
|
+
}
|
|
770
|
+
function reportSourceClassName(source) {
|
|
771
|
+
return sourceClassName(source);
|
|
772
|
+
}
|
|
773
|
+
export {
|
|
774
|
+
refreshReport,
|
|
775
|
+
reportRowIdentity,
|
|
776
|
+
reportSourceClassName
|
|
777
|
+
};
|
|
778
|
+
//# sourceMappingURL=refresh.js.map
|