@appweaver/data 2.4.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/config.d.ts +3 -0
- package/config.js +3 -0
- package/database/database-config.d.ts +9 -0
- package/database/database-config.js +56 -1
- package/database/database-error.d.ts +8 -0
- package/database/database-error.js +18 -0
- package/database/prisma-database.js +18 -1
- package/database/setup-database.d.ts +23 -0
- package/database/setup-database.js +59 -0
- package/database/sqlserver-extension.d.ts +18 -0
- package/database/sqlserver-extension.js +183 -0
- package/database/telemetry-extension.d.ts +10 -0
- package/database/telemetry-extension.js +52 -0
- package/export/export-service.js +4 -0
- package/factory/create-model.d.ts +1 -1
- package/factory/create-model.js +135 -5
- package/factory/create-service.js +8 -0
- package/index.d.ts +3 -0
- package/index.js +6 -0
- package/internal.d.ts +2 -0
- package/internal.js +3 -1
- package/module.js +23 -3
- package/package.json +3 -3
- package/prisma/client/browser.d.ts +10 -0
- package/prisma/client/client.d.ts +10 -0
- package/prisma/client/commonInputTypes.d.ts +81 -0
- package/prisma/client/enums.d.ts +6 -1
- package/prisma/client/enums.js +6 -0
- package/prisma/client/internal/class.d.ts +22 -0
- package/prisma/client/internal/class.js +4 -4
- package/prisma/client/internal/prismaNamespace.d.ts +175 -1
- package/prisma/client/internal/prismaNamespace.js +19 -1
- package/prisma/client/internal/prismaNamespaceBrowser.d.ts +20 -0
- package/prisma/client/internal/prismaNamespaceBrowser.js +19 -1
- package/prisma/client/models/TwoFactorMethod.d.ts +1140 -0
- package/prisma/client/models/TwoFactorMethod.js +2 -0
- package/prisma/client/models/TwoFactorRecoveryCode.d.ts +1005 -0
- package/prisma/client/models/TwoFactorRecoveryCode.js +2 -0
- package/prisma/client/models.d.ts +2 -0
- package/resource/resource-service.d.ts +32 -7
- package/resource/resource-service.js +248 -35
- package/resource/schemas/resource-filter-schema.d.ts +8 -0
- package/resource/schemas/resource-filter-schema.js +40 -1
- package/resource/schemas/resource-sort-schema.js +10 -0
- package/resource/utils/cursor-util.d.ts +13 -0
- package/resource/utils/cursor-util.js +21 -0
- package/resource/utils/filter-util.d.ts +9 -0
- package/resource/utils/filter-util.js +17 -0
- package/resource/utils/prune-util.js +3 -1
- package/resource/utils/sort-util.d.ts +14 -0
- package/resource/utils/sort-util.js +1 -0
- package/resource/utils/where-util.d.ts +20 -0
- package/resource/utils/where-util.js +210 -0
- package/resources.d.ts +4 -0
- package/resources.js +17 -9
- package/security/auth-config.d.ts +3 -0
- package/security/auth-config.js +6 -1
- package/security/create-auth-resources.d.ts +3 -3
- package/security/create-auth-resources.js +44 -7
- package/security/helper.d.ts +6 -0
- package/security/helper.js +11 -0
- package/security/resources/two-factor-method/model.d.ts +2 -0
- package/security/resources/two-factor-method/model.js +64 -0
- package/security/resources/two-factor-method/service.d.ts +2 -0
- package/security/resources/two-factor-method/service.js +7 -0
- package/security/resources/two-factor-recovery-code/model.d.ts +2 -0
- package/security/resources/two-factor-recovery-code/model.js +49 -0
- package/security/resources/two-factor-recovery-code/service.d.ts +2 -0
- package/security/resources/two-factor-recovery-code/service.js +7 -0
- package/security/store/database-security-store.d.ts +2 -2
- package/security/store/database-security-store.js +11 -1
- package/seeder/create-seeder.d.ts +3 -0
- package/seeder/create-seeder.js +6 -2
- package/seeder/seeder.d.ts +4 -2
- package/seeder/seeder.js +56 -23
- package/spatial/geo-schema.d.ts +22 -0
- package/spatial/geo-schema.js +60 -0
- package/spatial/spatial-config.d.ts +21 -0
- package/spatial/spatial-config.js +52 -0
- package/spatial/spatial-engine.d.ts +78 -0
- package/spatial/spatial-engine.js +232 -0
- package/spatial/spatial-extension.d.ts +10 -0
- package/spatial/spatial-extension.js +100 -0
- package/spatial/spatial-service.d.ts +54 -0
- package/spatial/spatial-service.js +213 -0
- package/spatial/utils/box-util.d.ts +31 -0
- package/spatial/utils/box-util.js +62 -0
- package/timeseries/database-series-config.d.ts +5 -0
- package/timeseries/database-series-config.js +15 -0
- package/timeseries/database-time-series.d.ts +36 -0
- package/timeseries/database-time-series.js +147 -0
- package/timeseries/index.d.ts +2 -0
- package/timeseries/index.js +18 -0
- package/timeseries/timescale-config.d.ts +6 -0
- package/timeseries/timescale-config.js +19 -0
- package/timeseries/timescale-time-series.d.ts +21 -0
- package/timeseries/timescale-time-series.js +158 -0
- package/timeseries/utils/series-sql-util.d.ts +90 -0
- package/timeseries/utils/series-sql-util.js +313 -0
- package/types/generated.d.ts +92 -0
- package/utils/schema-util.d.ts +8 -0
- package/utils/schema-util.js +22 -0
- package/utils/write-data-util.d.ts +22 -0
- package/utils/write-data-util.js +144 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.boxWithinFilter = boxWithinFilter;
|
|
4
|
+
exports.boxOverlapFilter = boxOverlapFilter;
|
|
5
|
+
exports.existsFilter = existsFilter;
|
|
6
|
+
const common_1 = require("@appweaver/common");
|
|
7
|
+
/**
|
|
8
|
+
* The condition of the bounding box columns of the geometries covered by one of the boxes, which is exact: a
|
|
9
|
+
* geometry lies in a box when its own box does. The upper bounds of the minimums repeat the ones of the maximums,
|
|
10
|
+
* for the index on the minimums.
|
|
11
|
+
*
|
|
12
|
+
* @param {string} field - The geo field.
|
|
13
|
+
* @param {GeoBox[]} boxes - The boxes.
|
|
14
|
+
* @return {Object} The Prisma condition.
|
|
15
|
+
*/
|
|
16
|
+
function boxWithinFilter(field, boxes) {
|
|
17
|
+
const columns = (0, common_1.geoColumns)(field);
|
|
18
|
+
return anyOf(boxes.map(([minLng, minLat, maxLng, maxLat]) => ({
|
|
19
|
+
[columns.minLng]: { gte: minLng, lte: maxLng },
|
|
20
|
+
[columns.minLat]: { gte: minLat, lte: maxLat },
|
|
21
|
+
[columns.maxLng]: { lte: maxLng },
|
|
22
|
+
[columns.maxLat]: { lte: maxLat }
|
|
23
|
+
})));
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* The condition of the bounding box columns of the geometries whose box overlaps one of the boxes, the only ones
|
|
27
|
+
* that can share a point with it.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} field - The geo field.
|
|
30
|
+
* @param {GeoBox[]} boxes - The boxes.
|
|
31
|
+
* @param {boolean} [point=false] - Whether the field holds points, whose minimums equal their maximums, so the
|
|
32
|
+
* index on the minimums bounds both sides.
|
|
33
|
+
* @return {Object} The Prisma condition.
|
|
34
|
+
*/
|
|
35
|
+
function boxOverlapFilter(field, boxes, point = false) {
|
|
36
|
+
const columns = (0, common_1.geoColumns)(field);
|
|
37
|
+
return anyOf(boxes.map(([minLng, minLat, maxLng, maxLat]) => point
|
|
38
|
+
? {
|
|
39
|
+
[columns.minLng]: { gte: minLng, lte: maxLng },
|
|
40
|
+
[columns.minLat]: { gte: minLat, lte: maxLat }
|
|
41
|
+
}
|
|
42
|
+
: {
|
|
43
|
+
[columns.minLng]: { lte: maxLng },
|
|
44
|
+
[columns.minLat]: { lte: maxLat },
|
|
45
|
+
[columns.maxLng]: { gte: minLng },
|
|
46
|
+
[columns.maxLat]: { gte: minLat }
|
|
47
|
+
}));
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The condition of the records with a value of a geo field, or without one. The bounding box columns are null
|
|
51
|
+
* exactly when the geometry is, whichever null its JSON column holds.
|
|
52
|
+
*
|
|
53
|
+
* @param {string} field - The geo field.
|
|
54
|
+
* @param {boolean} exists - Whether the records have a value.
|
|
55
|
+
* @return {Object} The Prisma condition.
|
|
56
|
+
*/
|
|
57
|
+
function existsFilter(field, exists) {
|
|
58
|
+
return { [(0, common_1.geoColumns)(field).minLng]: exists ? { not: null } : null };
|
|
59
|
+
}
|
|
60
|
+
function anyOf(conditions) {
|
|
61
|
+
return conditions.length === 1 ? conditions[0] : { OR: conditions };
|
|
62
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.databaseSeriesConfig = void 0;
|
|
4
|
+
const typebox_1 = require("@sinclair/typebox");
|
|
5
|
+
const common_1 = require("@appweaver/common");
|
|
6
|
+
exports.databaseSeriesConfig = (0, common_1.defineConfig)('timeseries.database', {
|
|
7
|
+
/** How long the `database` provider keeps the points, a duration (`365d`), overridden by the
|
|
8
|
+
* `seriesOptions.retention` of a model, empty to keep every point. Default: `''`. */
|
|
9
|
+
RETENTION: typebox_1.Type.String({ default: '' }),
|
|
10
|
+
/** How often the `database` provider deletes the points older than their retention, a duration. Default: `'1h'`. */
|
|
11
|
+
RETENTION_INTERVAL: typebox_1.Type.String({ default: '1h' }),
|
|
12
|
+
/** Delete the expired points in this instance, `false` for the other instances of an application whose deletes run
|
|
13
|
+
* in one instance (or in a task calling `purge`). Default: `true`. */
|
|
14
|
+
RETENTION_WORKER: typebox_1.Type.Boolean({ default: true })
|
|
15
|
+
});
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { HEALTH_CHECK, HealthCheckResult, SeriesDeleteQuery, SeriesModel, SeriesPoint, SeriesQuery, TimeSeries } from '@appweaver/common';
|
|
2
|
+
/**
|
|
3
|
+
* The `database` driver of `TimeSeries`: the points of a model in the `<Model>Series` table `weaver generate` adds to
|
|
4
|
+
* the Prisma schema, keyed by the resource and the time, written and queried with SQL over the connection of the
|
|
5
|
+
* database, on SQLite, PostgreSQL, MySQL and SQL Server. The points of a resource are deleted with it by the foreign
|
|
6
|
+
* key of the table, and the points older than the retention of their model (`seriesOptions.retention`, or the
|
|
7
|
+
* `timeseries.database.retention` config) once the application starts, then every `timeseries.database.retentionInterval`,
|
|
8
|
+
* in the instances with `timeseries.database.retentionWorker`.
|
|
9
|
+
*/
|
|
10
|
+
export declare class DatabaseTimeSeries extends TimeSeries {
|
|
11
|
+
static [HEALTH_CHECK]: boolean;
|
|
12
|
+
readonly cascadesDeletes: boolean;
|
|
13
|
+
readonly transactional: boolean;
|
|
14
|
+
onInit(): Promise<void>;
|
|
15
|
+
onDestroy(): Promise<void>;
|
|
16
|
+
/**
|
|
17
|
+
* Starts deleting the points older than the retention of their model, at once and every
|
|
18
|
+
* `timeseries.database.retentionInterval`.
|
|
19
|
+
*
|
|
20
|
+
* @throws {ConfigurationError} `CONFIGURATION_INVALID` for a retention or an interval that is not a duration.
|
|
21
|
+
*/
|
|
22
|
+
register(models: SeriesModel[]): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Deletes the points older than the retention of their model, of the models registered with a retention.
|
|
25
|
+
*
|
|
26
|
+
* @param {Date} [now] - The time the retentions count back from.
|
|
27
|
+
* @return {Promise<number>} The number of points deleted.
|
|
28
|
+
*/
|
|
29
|
+
purge(now?: Date): Promise<number>;
|
|
30
|
+
write(model: SeriesModel, points: SeriesPoint[]): Promise<number>;
|
|
31
|
+
query(model: SeriesModel, query: SeriesQuery): Promise<SeriesPoint[]>;
|
|
32
|
+
delete(model: SeriesModel, query: SeriesDeleteQuery): Promise<number>;
|
|
33
|
+
checkHealth(): Promise<HealthCheckResult>;
|
|
34
|
+
/** The start of the bucket of a time column, overridden by a store with its own bucket function. */
|
|
35
|
+
protected bucket(column: string, width: number): string;
|
|
36
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DatabaseTimeSeries = void 0;
|
|
4
|
+
const common_1 = require("@appweaver/common");
|
|
5
|
+
const core_1 = require("@appweaver/core");
|
|
6
|
+
const database_config_1 = require("../database/database-config");
|
|
7
|
+
const database_error_1 = require("../database/database-error");
|
|
8
|
+
const schema_util_1 = require("../utils/schema-util");
|
|
9
|
+
const database_series_config_1 = require("./database-series-config");
|
|
10
|
+
const series_sql_util_1 = require("./utils/series-sql-util");
|
|
11
|
+
const logger = (0, common_1.createLogger)('timeseries');
|
|
12
|
+
/**
|
|
13
|
+
* The `database` driver of `TimeSeries`: the points of a model in the `<Model>Series` table `weaver generate` adds to
|
|
14
|
+
* the Prisma schema, keyed by the resource and the time, written and queried with SQL over the connection of the
|
|
15
|
+
* database, on SQLite, PostgreSQL, MySQL and SQL Server. The points of a resource are deleted with it by the foreign
|
|
16
|
+
* key of the table, and the points older than the retention of their model (`seriesOptions.retention`, or the
|
|
17
|
+
* `timeseries.database.retention` config) once the application starts, then every `timeseries.database.retentionInterval`,
|
|
18
|
+
* in the instances with `timeseries.database.retentionWorker`.
|
|
19
|
+
*/
|
|
20
|
+
class DatabaseTimeSeries extends common_1.TimeSeries {
|
|
21
|
+
// The database has its own health check
|
|
22
|
+
static [common_1.HEALTH_CHECK] = false;
|
|
23
|
+
cascadesDeletes = true;
|
|
24
|
+
transactional = true;
|
|
25
|
+
/** @internal */
|
|
26
|
+
_db = (0, core_1.inject)(common_1.Database);
|
|
27
|
+
/** @internal */
|
|
28
|
+
_type = (0, common_1.resolveDatabaseType)(database_config_1.databaseConfig.TYPE, database_config_1.databaseConfig.URL);
|
|
29
|
+
/** The PostgreSQL schema of the connection, `public` without one. @internal */
|
|
30
|
+
_schema = (0, schema_util_1.postgresSchema)(this._type);
|
|
31
|
+
/** @internal */
|
|
32
|
+
_dialect = (0, series_sql_util_1.seriesDialect)(this._type, this._schema);
|
|
33
|
+
/** @internal */
|
|
34
|
+
_retentions = [];
|
|
35
|
+
/** @internal */
|
|
36
|
+
_purgeTimer;
|
|
37
|
+
async onInit() { }
|
|
38
|
+
async onDestroy() {
|
|
39
|
+
clearInterval(this._purgeTimer);
|
|
40
|
+
this._purgeTimer = undefined;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Starts deleting the points older than the retention of their model, at once and every
|
|
44
|
+
* `timeseries.database.retentionInterval`.
|
|
45
|
+
*
|
|
46
|
+
* @throws {ConfigurationError} `CONFIGURATION_INVALID` for a retention or an interval that is not a duration.
|
|
47
|
+
*/
|
|
48
|
+
async register(models) {
|
|
49
|
+
this._retentions = models.flatMap((model) => {
|
|
50
|
+
const retention = model.options.retention ?? database_series_config_1.databaseSeriesConfig.RETENTION;
|
|
51
|
+
return retention
|
|
52
|
+
? [
|
|
53
|
+
[model, seconds(retention, `retention of ${model.name}`)]
|
|
54
|
+
]
|
|
55
|
+
: [];
|
|
56
|
+
});
|
|
57
|
+
clearInterval(this._purgeTimer);
|
|
58
|
+
this._purgeTimer = undefined;
|
|
59
|
+
const interval = seconds(database_series_config_1.databaseSeriesConfig.RETENTION_INTERVAL, 'retention interval');
|
|
60
|
+
if (this._retentions.length === 0 ||
|
|
61
|
+
!database_series_config_1.databaseSeriesConfig.RETENTION_WORKER) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const run = () => void this.purge().catch((error) => logger.error(error, 'Cannot delete the expired time series points'));
|
|
65
|
+
this._purgeTimer = setInterval(run, interval * 1000);
|
|
66
|
+
this._purgeTimer.unref();
|
|
67
|
+
run();
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Deletes the points older than the retention of their model, of the models registered with a retention.
|
|
71
|
+
*
|
|
72
|
+
* @param {Date} [now] - The time the retentions count back from.
|
|
73
|
+
* @return {Promise<number>} The number of points deleted.
|
|
74
|
+
*/
|
|
75
|
+
async purge(now = new Date()) {
|
|
76
|
+
let count = 0;
|
|
77
|
+
for (const [model, retention] of this._retentions) {
|
|
78
|
+
try {
|
|
79
|
+
const deleted = await this.client().$executeRaw((0, series_sql_util_1.expiredStatement)(this._dialect, model, new Date(now.getTime() - retention * 1000)));
|
|
80
|
+
count += deleted;
|
|
81
|
+
if (deleted > 0) {
|
|
82
|
+
logger.debug({ model: model.name, points: deleted }, 'Expired time series points deleted');
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
throw (0, database_error_1.toDatabaseError)(error, model.name);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return count;
|
|
90
|
+
}
|
|
91
|
+
async write(model, points) {
|
|
92
|
+
const statements = (0, series_sql_util_1.upsertStatements)(this._dialect, model, points);
|
|
93
|
+
try {
|
|
94
|
+
if (statements.length === 1) {
|
|
95
|
+
await this.client().$executeRaw(statements[0]);
|
|
96
|
+
}
|
|
97
|
+
else if (statements.length > 1) {
|
|
98
|
+
await this._db.transaction(async (tx) => {
|
|
99
|
+
for (const statement of statements) {
|
|
100
|
+
await tx.$executeRaw(statement);
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
throw (0, database_error_1.toDatabaseError)(error, model.name);
|
|
107
|
+
}
|
|
108
|
+
return points.length;
|
|
109
|
+
}
|
|
110
|
+
async query(model, query) {
|
|
111
|
+
try {
|
|
112
|
+
const rows = await this.client().$queryRaw((0, series_sql_util_1.selectStatement)(this._dialect, model, query, this.bucket.bind(this)));
|
|
113
|
+
return (0, series_sql_util_1.toSeriesPoints)(model, query, rows);
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
throw (0, database_error_1.toDatabaseError)(error, model.name);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
async delete(model, query) {
|
|
120
|
+
try {
|
|
121
|
+
return await this.client().$executeRaw((0, series_sql_util_1.deleteStatement)(this._dialect, model, query));
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
throw (0, database_error_1.toDatabaseError)(error, model.name);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
async checkHealth() {
|
|
128
|
+
return this._db.checkHealth();
|
|
129
|
+
}
|
|
130
|
+
/** The start of the bucket of a time column, overridden by a store with its own bucket function. */
|
|
131
|
+
bucket(column, width) {
|
|
132
|
+
return this._dialect.bucket(column, width);
|
|
133
|
+
}
|
|
134
|
+
/** @internal */
|
|
135
|
+
client() {
|
|
136
|
+
return this._db.client();
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
exports.DatabaseTimeSeries = DatabaseTimeSeries;
|
|
140
|
+
/** The seconds of a duration, checked to be one of whole seconds. */
|
|
141
|
+
function seconds(duration, option) {
|
|
142
|
+
const value = (0, common_1.parseDuration)(duration);
|
|
143
|
+
if (!value || !Number.isInteger(value)) {
|
|
144
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, `The time series ${option} '${duration}' is not a duration of whole seconds, i.e. 365d`);
|
|
145
|
+
}
|
|
146
|
+
return value;
|
|
147
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./database-time-series"), exports);
|
|
18
|
+
__exportStar(require("./timescale-time-series"), exports);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.timescaleConfig = void 0;
|
|
4
|
+
const typebox_1 = require("@sinclair/typebox");
|
|
5
|
+
const common_1 = require("@appweaver/common");
|
|
6
|
+
exports.timescaleConfig = (0, common_1.defineConfig)('timeseries.timescale', {
|
|
7
|
+
/** Time range of a chunk of the hypertables, a duration (`7d`), overridden by the `seriesOptions.chunkInterval` of
|
|
8
|
+
* a model. Default: `'7d'`. */
|
|
9
|
+
CHUNK_INTERVAL: typebox_1.Type.String({ default: '7d' }),
|
|
10
|
+
/** Age from which the chunks are compressed, a duration (`30d`), overridden by the `seriesOptions.compressAfter` of
|
|
11
|
+
* a model, empty to compress none. Default: `''`. */
|
|
12
|
+
COMPRESS_AFTER: typebox_1.Type.String({ default: '' }),
|
|
13
|
+
/** Age from which the chunks are dropped, a duration (`365d`), overridden by the `seriesOptions.retention` of a
|
|
14
|
+
* model, empty to keep every point. Default: `''`. */
|
|
15
|
+
RETENTION: typebox_1.Type.String({ default: '' }),
|
|
16
|
+
/** Maximum time in milliseconds the hypertables and their policies take to set up at startup. Default: `60000`
|
|
17
|
+
* (1 min). */
|
|
18
|
+
SETUP_TIMEOUT: typebox_1.Type.Integer({ default: 60000, minimum: 1 })
|
|
19
|
+
});
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { HEALTH_CHECK, SeriesModel } from '@appweaver/common';
|
|
2
|
+
import { DatabaseTimeSeries } from './database-time-series';
|
|
3
|
+
/**
|
|
4
|
+
* The `timescale` driver of `TimeSeries`: the `database` driver on a PostgreSQL database with the TimescaleDB
|
|
5
|
+
* extension, which turns the series tables into hypertables at startup, partitioned by time, aggregates with
|
|
6
|
+
* `time_bucket`, and keeps the compression and retention policies of each model in line with its config.
|
|
7
|
+
*/
|
|
8
|
+
export declare class TimescaleTimeSeries extends DatabaseTimeSeries {
|
|
9
|
+
static [HEALTH_CHECK]: boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Creates the extension when missing, turns the series tables into hypertables and applies their chunk interval and
|
|
12
|
+
* policies, under an advisory lock, so the instances starting together set them up once. Only what differs from the
|
|
13
|
+
* models and the config is changed, so a database set up by `weaver migrate` runs no DDL, and the user of the
|
|
14
|
+
* application needs no privilege beyond reading and writing the data.
|
|
15
|
+
*
|
|
16
|
+
* @throws {ConfigurationError} `CONFIGURATION_INVALID` on a database other than PostgreSQL, one without the
|
|
17
|
+
* TimescaleDB extension that cannot be created, or one the user of the connection lacks the privileges to set up.
|
|
18
|
+
*/
|
|
19
|
+
register(models: SeriesModel[]): Promise<void>;
|
|
20
|
+
protected bucket(column: string, width: number): string;
|
|
21
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TimescaleTimeSeries = void 0;
|
|
4
|
+
const common_1 = require("@appweaver/common");
|
|
5
|
+
const client_1 = require("@prisma/client/runtime/client");
|
|
6
|
+
const database_config_1 = require("../database/database-config");
|
|
7
|
+
const database_error_1 = require("../database/database-error");
|
|
8
|
+
const database_time_series_1 = require("./database-time-series");
|
|
9
|
+
const timescale_config_1 = require("./timescale-config");
|
|
10
|
+
const logger = (0, common_1.createLogger)('timeseries');
|
|
11
|
+
const COMPRESSION = {
|
|
12
|
+
proc: 'policy_compression',
|
|
13
|
+
key: 'compress_after',
|
|
14
|
+
add: (table, interval) => `SELECT add_compression_policy('${table}', INTERVAL '${interval}')`,
|
|
15
|
+
remove: (table) => `SELECT remove_compression_policy('${table}', if_exists => TRUE)`
|
|
16
|
+
};
|
|
17
|
+
const RETENTION = {
|
|
18
|
+
proc: 'policy_retention',
|
|
19
|
+
key: 'drop_after',
|
|
20
|
+
add: (table, interval) => `SELECT add_retention_policy('${table}', INTERVAL '${interval}')`,
|
|
21
|
+
remove: (table) => `SELECT remove_retention_policy('${table}', if_exists => TRUE)`
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* The `timescale` driver of `TimeSeries`: the `database` driver on a PostgreSQL database with the TimescaleDB
|
|
25
|
+
* extension, which turns the series tables into hypertables at startup, partitioned by time, aggregates with
|
|
26
|
+
* `time_bucket`, and keeps the compression and retention policies of each model in line with its config.
|
|
27
|
+
*/
|
|
28
|
+
class TimescaleTimeSeries extends database_time_series_1.DatabaseTimeSeries {
|
|
29
|
+
static [common_1.HEALTH_CHECK] = false;
|
|
30
|
+
/**
|
|
31
|
+
* Creates the extension when missing, turns the series tables into hypertables and applies their chunk interval and
|
|
32
|
+
* policies, under an advisory lock, so the instances starting together set them up once. Only what differs from the
|
|
33
|
+
* models and the config is changed, so a database set up by `weaver migrate` runs no DDL, and the user of the
|
|
34
|
+
* application needs no privilege beyond reading and writing the data.
|
|
35
|
+
*
|
|
36
|
+
* @throws {ConfigurationError} `CONFIGURATION_INVALID` on a database other than PostgreSQL, one without the
|
|
37
|
+
* TimescaleDB extension that cannot be created, or one the user of the connection lacks the privileges to set up.
|
|
38
|
+
*/
|
|
39
|
+
async register(models) {
|
|
40
|
+
const type = (0, common_1.resolveDatabaseType)(database_config_1.databaseConfig.TYPE, database_config_1.databaseConfig.URL);
|
|
41
|
+
if (type !== common_1.DatabaseType.PostgreSQL) {
|
|
42
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, `The timescale time series provider needs a PostgreSQL database, not ${type}`);
|
|
43
|
+
}
|
|
44
|
+
if (models.length === 0) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
await this.ensureExtension();
|
|
48
|
+
try {
|
|
49
|
+
await this._db.transaction(async (tx) => {
|
|
50
|
+
await tx.$executeRawUnsafe(`SELECT pg_advisory_xact_lock(hashtext('appweaver:timeseries'))`);
|
|
51
|
+
for (const model of models) {
|
|
52
|
+
await this.setupHypertable(tx, model);
|
|
53
|
+
}
|
|
54
|
+
}, { timeout: timescale_config_1.timescaleConfig.SETUP_TIMEOUT });
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
if ((0, database_error_1.isPrivilegeError)(error)) {
|
|
58
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, 'The time series hypertables are not set up, and the database user lacks the privileges to set them up, ' +
|
|
59
|
+
'run `weaver migrate`, which sets them up with the user of database.migrationUrl', undefined, { cause: error });
|
|
60
|
+
}
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
bucket(column, width) {
|
|
65
|
+
return `time_bucket(INTERVAL '${width} seconds', ${column}, TIMESTAMP '1970-01-01 00:00:00')`;
|
|
66
|
+
}
|
|
67
|
+
/** @internal */
|
|
68
|
+
async ensureExtension() {
|
|
69
|
+
const installed = async (db) => (await db.$queryRaw((0, client_1.sqltag) `SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'`)).length > 0;
|
|
70
|
+
if (await installed(this.client())) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
try {
|
|
74
|
+
// Under the lock of the setup, since a CREATE EXTENSION IF NOT EXISTS
|
|
75
|
+
// racing another instance starting fails with a unique violation
|
|
76
|
+
await this._db.transaction(async (tx) => {
|
|
77
|
+
await tx.$executeRawUnsafe(`SELECT pg_advisory_xact_lock(hashtext('appweaver:timeseries'))`);
|
|
78
|
+
if (!(await installed(tx))) {
|
|
79
|
+
await tx.$executeRawUnsafe('CREATE EXTENSION IF NOT EXISTS timescaledb');
|
|
80
|
+
logger.info('Created the timescaledb extension');
|
|
81
|
+
}
|
|
82
|
+
}, { timeout: timescale_config_1.timescaleConfig.SETUP_TIMEOUT });
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, 'The timescaledb extension is not installed in the database and cannot be created, ' +
|
|
86
|
+
'create it with `CREATE EXTENSION timescaledb`, run `weaver migrate`, which creates it with the user of ' +
|
|
87
|
+
'database.migrationUrl, or use a TimescaleDB image', undefined, { cause: error });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
/** @internal */
|
|
91
|
+
async setupHypertable(tx, model) {
|
|
92
|
+
const table = this.regclass(model.table);
|
|
93
|
+
const chunk = interval(model.options.chunkInterval ?? timescale_config_1.timescaleConfig.CHUNK_INTERVAL, 'chunkInterval');
|
|
94
|
+
const [dimension] = await tx.$queryRaw((0, client_1.sqltag) `SELECT time_interval = ${(0, client_1.raw)(`INTERVAL '${chunk}'`)} AS same FROM timescaledb_information.dimensions
|
|
95
|
+
WHERE hypertable_schema = ${this._schema ?? (0, client_1.raw)('current_schema()')} AND hypertable_name = ${model.table}
|
|
96
|
+
AND column_name = 'time'`);
|
|
97
|
+
if (!dimension) {
|
|
98
|
+
// The primary key holds the time, and no other index is created, so the
|
|
99
|
+
// schema stays the one of the Prisma migrations
|
|
100
|
+
await tx.$executeRawUnsafe(`SELECT create_hypertable('${table}', by_range('time', INTERVAL '${chunk}'), ` +
|
|
101
|
+
'if_not_exists => TRUE, create_default_indexes => FALSE, migrate_data => TRUE)');
|
|
102
|
+
}
|
|
103
|
+
else if (!dimension.same) {
|
|
104
|
+
await tx.$executeRawUnsafe(`SELECT set_chunk_time_interval('${table}', INTERVAL '${chunk}')`);
|
|
105
|
+
}
|
|
106
|
+
const compressAfter = model.options.compressAfter ?? timescale_config_1.timescaleConfig.COMPRESS_AFTER;
|
|
107
|
+
if (compressAfter) {
|
|
108
|
+
const [settings] = await tx.$queryRaw((0, client_1.sqltag) `SELECT compression_enabled FROM timescaledb_information.hypertables
|
|
109
|
+
WHERE hypertable_schema = ${this._schema ?? (0, client_1.raw)('current_schema()')} AND hypertable_name = ${model.table}`);
|
|
110
|
+
if (!settings?.compression_enabled) {
|
|
111
|
+
await tx.$executeRawUnsafe(`ALTER TABLE ${this._dialect.table(model.table)} SET (timescaledb.compress, ` +
|
|
112
|
+
`timescaledb.compress_segmentby = '"resourceId"', timescaledb.compress_orderby = '"time" DESC')`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
await this.syncPolicy(tx, model, COMPRESSION, compressAfter, 'compressAfter');
|
|
116
|
+
await this.syncPolicy(tx, model, RETENTION, model.options.retention ?? timescale_config_1.timescaleConfig.RETENTION, 'retention');
|
|
117
|
+
logger.debug({ model: model.name, table: model.table }, 'Time series hypertable set up');
|
|
118
|
+
}
|
|
119
|
+
/** The table of a model as a regclass literal, qualified by the schema of the connection. @internal */
|
|
120
|
+
regclass(table) {
|
|
121
|
+
return this._dialect.table(table).replace(/'/g, "''");
|
|
122
|
+
}
|
|
123
|
+
/** Adds, replaces or removes a policy of a hypertable, so it follows its config. @internal */
|
|
124
|
+
async syncPolicy(tx, model, policy, duration, option) {
|
|
125
|
+
const table = this.regclass(model.table);
|
|
126
|
+
const wanted = duration ? interval(duration, option) : undefined;
|
|
127
|
+
const [job] = await tx.$queryRaw((0, client_1.sqltag) `SELECT (config->>${policy.key})::interval = ${(0, client_1.raw)(wanted ? `INTERVAL '${wanted}'` : 'NULL')} AS same FROM timescaledb_information.jobs
|
|
128
|
+
WHERE proc_name = ${policy.proc} AND hypertable_schema = ${this._schema ?? (0, client_1.raw)('current_schema()')} AND hypertable_name = ${model.table}`);
|
|
129
|
+
if (job?.same) {
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
if (job) {
|
|
133
|
+
await tx.$executeRawUnsafe(policy.remove(table));
|
|
134
|
+
}
|
|
135
|
+
if (wanted) {
|
|
136
|
+
await tx.$executeRawUnsafe(policy.add(table, wanted));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
exports.TimescaleTimeSeries = TimescaleTimeSeries;
|
|
141
|
+
/** The interval of a duration in seconds, checked to be one. */
|
|
142
|
+
function interval(duration, option) {
|
|
143
|
+
const seconds = (0, common_1.parseDuration)(duration);
|
|
144
|
+
if (!seconds || !Number.isInteger(seconds)) {
|
|
145
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, `The time series ${option} '${duration}' is not a duration of whole seconds, i.e. 7d`);
|
|
146
|
+
}
|
|
147
|
+
for (const [unit, size] of INTERVAL_UNITS) {
|
|
148
|
+
if (seconds % size === 0) {
|
|
149
|
+
return `${seconds / size} ${unit}`;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return `${seconds} seconds`;
|
|
153
|
+
}
|
|
154
|
+
const INTERVAL_UNITS = [
|
|
155
|
+
['days', 86_400],
|
|
156
|
+
['hours', 3600],
|
|
157
|
+
['minutes', 60]
|
|
158
|
+
];
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { Sql } from '@prisma/client/runtime/client';
|
|
2
|
+
import { DatabaseType, SeriesField, SeriesModel, SeriesPoint, SeriesQuery } from '@appweaver/common';
|
|
3
|
+
/** The SQL of a database the time series queries differ in. */
|
|
4
|
+
export type SeriesDialect = {
|
|
5
|
+
/** The most parameters a statement binds */
|
|
6
|
+
maxParams: number;
|
|
7
|
+
/** Quotes an identifier */
|
|
8
|
+
quote: (name: string) => string;
|
|
9
|
+
/** Quotes a table name, qualified by the schema of the connection when it has one */
|
|
10
|
+
table: (name: string) => string;
|
|
11
|
+
/** The start of the bucket of `width` seconds holding the time column, aligned to the Unix epoch */
|
|
12
|
+
bucket: (column: string, width: number) => string;
|
|
13
|
+
/** A boolean or numeric column as a number */
|
|
14
|
+
numeric: (column: string, field: SeriesField) => string;
|
|
15
|
+
/** The argument of `AVG`, computed in floating point */
|
|
16
|
+
average: (column: string) => string;
|
|
17
|
+
/** The argument of `SUM`, so a total of an integer field fits */
|
|
18
|
+
sum: (column: string, field: SeriesField) => string;
|
|
19
|
+
/** Limits the rows of an ordered query */
|
|
20
|
+
limit: (count: number) => string;
|
|
21
|
+
/** Writes the rows of the columns into the quoted table, merging a row into the stored one of the same resource
|
|
22
|
+
* and time */
|
|
23
|
+
upsert: (table: string, columns: string[], fields: string[], rows: Sql) => Sql;
|
|
24
|
+
};
|
|
25
|
+
export declare const postgresDialect: SeriesDialect;
|
|
26
|
+
export declare const sqliteDialect: SeriesDialect;
|
|
27
|
+
export declare const mysqlDialect: SeriesDialect;
|
|
28
|
+
export declare const mssqlDialect: SeriesDialect;
|
|
29
|
+
/**
|
|
30
|
+
* Returns the SQL dialect of the time series of a database, its tables qualified by the PostgreSQL schema of the
|
|
31
|
+
* connection, since the raw queries are not qualified by the Prisma client.
|
|
32
|
+
*
|
|
33
|
+
* @param {DatabaseType} type - The database type.
|
|
34
|
+
* @param {string} [schema] - The PostgreSQL schema of the connection (the `schema` parameter of its URL).
|
|
35
|
+
* @return {SeriesDialect} The dialect.
|
|
36
|
+
*/
|
|
37
|
+
export declare function seriesDialect(type: DatabaseType, schema?: string): SeriesDialect;
|
|
38
|
+
/**
|
|
39
|
+
* Builds the statements writing the points of a model, each binding up to the parameters the database allows.
|
|
40
|
+
*
|
|
41
|
+
* @param {SeriesDialect} dialect - The SQL dialect.
|
|
42
|
+
* @param {SeriesModel} model - The series model.
|
|
43
|
+
* @param {SeriesPoint[]} points - The points, at most one per resource and time.
|
|
44
|
+
* @return {Sql[]} The statements.
|
|
45
|
+
*/
|
|
46
|
+
export declare function upsertStatements(dialect: SeriesDialect, model: SeriesModel, points: SeriesPoint[]): Sql[];
|
|
47
|
+
/**
|
|
48
|
+
* Builds the query of the points of a model: the stored points, or the points aggregated by bucket, or over the whole
|
|
49
|
+
* range of each resource without one. The points without a value of the selected fields are skipped.
|
|
50
|
+
*
|
|
51
|
+
* @param {SeriesDialect} dialect - The SQL dialect.
|
|
52
|
+
* @param {SeriesModel} model - The series model.
|
|
53
|
+
* @param {SeriesQuery} query - The normalized query, its fields selected.
|
|
54
|
+
* @param {Function} [bucket] - Builds the bucket start of a time column, the one of the dialect by default.
|
|
55
|
+
* @return {Sql} The query, its columns `rid`, `t` and the fields.
|
|
56
|
+
*/
|
|
57
|
+
export declare function selectStatement(dialect: SeriesDialect, model: SeriesModel, query: SeriesQuery, bucket?: (column: string, width: number) => string): Sql;
|
|
58
|
+
/**
|
|
59
|
+
* Builds the statement deleting the points of a model within a time range.
|
|
60
|
+
*
|
|
61
|
+
* @param {SeriesDialect} dialect - The SQL dialect.
|
|
62
|
+
* @param {SeriesModel} model - The series model.
|
|
63
|
+
* @param {Object} query - The resources and the time range.
|
|
64
|
+
* @return {Sql} The statement.
|
|
65
|
+
*/
|
|
66
|
+
export declare function deleteStatement(dialect: SeriesDialect, model: SeriesModel, query: Pick<SeriesQuery, 'resourceIds' | 'from' | 'to'>): Sql;
|
|
67
|
+
/**
|
|
68
|
+
* Builds the statement deleting the points of a model older than a time, of every resource.
|
|
69
|
+
*
|
|
70
|
+
* @param {SeriesDialect} dialect - The SQL dialect.
|
|
71
|
+
* @param {SeriesModel} model - The series model.
|
|
72
|
+
* @param {Date} before - The time the points are older than.
|
|
73
|
+
* @return {Sql} The statement.
|
|
74
|
+
*/
|
|
75
|
+
export declare function expiredStatement(dialect: SeriesDialect, model: SeriesModel, before: Date): Sql;
|
|
76
|
+
/** The row of a time series query. */
|
|
77
|
+
export type SeriesRow = Record<string, unknown>;
|
|
78
|
+
/**
|
|
79
|
+
* Converts the rows of a time series query to points, the values to the types of their fields: the numbers of a
|
|
80
|
+
* database returning them as text, decimals or big integers, the booleans of one storing them as numbers, and the
|
|
81
|
+
* times of one returning them as text in UTC.
|
|
82
|
+
*
|
|
83
|
+
* @param {SeriesModel} model - The series model.
|
|
84
|
+
* @param {SeriesQuery} query - The query.
|
|
85
|
+
* @param {SeriesRow[]} rows - The rows.
|
|
86
|
+
* @return {SeriesPoint[]} The points.
|
|
87
|
+
*/
|
|
88
|
+
export declare function toSeriesPoints(model: SeriesModel, query: SeriesQuery, rows: SeriesRow[]): SeriesPoint[];
|
|
89
|
+
/** Converts a time read from a database, a text without an offset being in UTC. */
|
|
90
|
+
export declare function toDate(value: unknown): Date;
|