@balena/pinejs 15.0.0-true-boolean-911aca4062d3132ad3c34712014739b6849fa13a → 15.0.0
Sign up to get free protection for your applications and to get access to all the features.
- package/.dockerignore +4 -0
- package/.github/workflows/flowzone.yml +21 -0
- package/.husky/pre-commit +4 -0
- package/.pinejs-cache.json +1 -0
- package/.resinci.yml +1 -0
- package/.versionbot/CHANGELOG.yml +9678 -2001
- package/CHANGELOG.md +2975 -2
- package/Dockerfile +14 -0
- package/Gruntfile.ts +3 -6
- package/README.md +10 -1
- package/VERSION +1 -0
- package/build/browser.ts +1 -1
- package/build/config.ts +0 -1
- package/docker-compose.npm-test.yml +11 -0
- package/docs/AdvancedUsage.md +77 -63
- package/docs/GettingStarted.md +90 -41
- package/docs/Migrations.md +102 -1
- package/docs/ProjectConfig.md +12 -21
- package/docs/Testing.md +7 -0
- package/out/bin/abstract-sql-compiler.js +17 -17
- package/out/bin/abstract-sql-compiler.js.map +1 -1
- package/out/bin/odata-compiler.js +23 -20
- package/out/bin/odata-compiler.js.map +1 -1
- package/out/bin/sbvr-compiler.js +22 -22
- package/out/bin/sbvr-compiler.js.map +1 -1
- package/out/bin/utils.d.ts +2 -2
- package/out/bin/utils.js +3 -3
- package/out/bin/utils.js.map +1 -1
- package/out/config-loader/config-loader.d.ts +9 -8
- package/out/config-loader/config-loader.js +135 -78
- package/out/config-loader/config-loader.js.map +1 -1
- package/out/config-loader/env.d.ts +41 -16
- package/out/config-loader/env.js +46 -2
- package/out/config-loader/env.js.map +1 -1
- package/out/data-server/sbvr-server.d.ts +2 -19
- package/out/data-server/sbvr-server.js +44 -38
- package/out/data-server/sbvr-server.js.map +1 -1
- package/out/database-layer/db.d.ts +32 -14
- package/out/database-layer/db.js +120 -41
- package/out/database-layer/db.js.map +1 -1
- package/out/express-emulator/express.js +10 -11
- package/out/express-emulator/express.js.map +1 -1
- package/out/http-transactions/transactions.d.ts +2 -18
- package/out/http-transactions/transactions.js +29 -21
- package/out/http-transactions/transactions.js.map +1 -1
- package/out/migrator/async.d.ts +7 -0
- package/out/migrator/async.js +168 -0
- package/out/migrator/async.js.map +1 -0
- package/out/migrator/migrations.sbvr +43 -0
- package/out/migrator/sync.d.ts +9 -0
- package/out/migrator/sync.js +106 -0
- package/out/migrator/sync.js.map +1 -0
- package/out/migrator/utils.d.ts +78 -0
- package/out/migrator/utils.js +283 -0
- package/out/migrator/utils.js.map +1 -0
- package/out/odata-metadata/odata-metadata-generator.js +10 -13
- package/out/odata-metadata/odata-metadata-generator.js.map +1 -1
- package/out/passport-pinejs/passport-pinejs.d.ts +1 -1
- package/out/passport-pinejs/passport-pinejs.js +8 -7
- package/out/passport-pinejs/passport-pinejs.js.map +1 -1
- package/out/pinejs-session-store/pinejs-session-store.d.ts +1 -1
- package/out/pinejs-session-store/pinejs-session-store.js +20 -6
- package/out/pinejs-session-store/pinejs-session-store.js.map +1 -1
- package/out/sbvr-api/abstract-sql.d.ts +3 -2
- package/out/sbvr-api/abstract-sql.js +9 -9
- package/out/sbvr-api/abstract-sql.js.map +1 -1
- package/out/sbvr-api/cached-compile.js +1 -1
- package/out/sbvr-api/cached-compile.js.map +1 -1
- package/out/sbvr-api/common-types.d.ts +6 -5
- package/out/sbvr-api/control-flow.d.ts +8 -1
- package/out/sbvr-api/control-flow.js +36 -9
- package/out/sbvr-api/control-flow.js.map +1 -1
- package/out/sbvr-api/errors.d.ts +47 -40
- package/out/sbvr-api/errors.js +78 -77
- package/out/sbvr-api/errors.js.map +1 -1
- package/out/sbvr-api/express-extension.d.ts +4 -0
- package/out/sbvr-api/hooks.d.ts +16 -15
- package/out/sbvr-api/hooks.js +74 -48
- package/out/sbvr-api/hooks.js.map +1 -1
- package/out/sbvr-api/odata-response.d.ts +2 -2
- package/out/sbvr-api/odata-response.js +28 -30
- package/out/sbvr-api/odata-response.js.map +1 -1
- package/out/sbvr-api/permissions.d.ts +17 -16
- package/out/sbvr-api/permissions.js +369 -304
- package/out/sbvr-api/permissions.js.map +1 -1
- package/out/sbvr-api/sbvr-utils.d.ts +33 -15
- package/out/sbvr-api/sbvr-utils.js +397 -235
- package/out/sbvr-api/sbvr-utils.js.map +1 -1
- package/out/sbvr-api/translations.d.ts +6 -0
- package/out/sbvr-api/translations.js +150 -0
- package/out/sbvr-api/translations.js.map +1 -0
- package/out/sbvr-api/uri-parser.d.ts +23 -17
- package/out/sbvr-api/uri-parser.js +33 -27
- package/out/sbvr-api/uri-parser.js.map +1 -1
- package/out/sbvr-api/user.sbvr +2 -0
- package/out/server-glue/module.d.ts +6 -6
- package/out/server-glue/module.js +4 -2
- package/out/server-glue/module.js.map +1 -1
- package/out/server-glue/server.js +5 -5
- package/out/server-glue/server.js.map +1 -1
- package/package.json +89 -73
- package/pinejs.png +0 -0
- package/repo.yml +9 -9
- package/src/bin/abstract-sql-compiler.ts +5 -7
- package/src/bin/odata-compiler.ts +11 -13
- package/src/bin/sbvr-compiler.ts +11 -17
- package/src/bin/utils.ts +3 -5
- package/src/config-loader/config-loader.ts +167 -53
- package/src/config-loader/env.ts +106 -6
- package/src/data-server/sbvr-server.js +44 -38
- package/src/database-layer/db.ts +205 -64
- package/src/express-emulator/express.js +10 -11
- package/src/http-transactions/transactions.js +29 -21
- package/src/migrator/async.ts +323 -0
- package/src/migrator/migrations.sbvr +43 -0
- package/src/migrator/sync.ts +152 -0
- package/src/migrator/utils.ts +458 -0
- package/src/odata-metadata/odata-metadata-generator.ts +12 -15
- package/src/passport-pinejs/passport-pinejs.ts +9 -7
- package/src/pinejs-session-store/pinejs-session-store.ts +15 -1
- package/src/sbvr-api/abstract-sql.ts +17 -14
- package/src/sbvr-api/common-types.ts +2 -1
- package/src/sbvr-api/control-flow.ts +45 -11
- package/src/sbvr-api/errors.ts +82 -77
- package/src/sbvr-api/express-extension.ts +6 -1
- package/src/sbvr-api/hooks.ts +123 -50
- package/src/sbvr-api/odata-response.ts +23 -28
- package/src/sbvr-api/permissions.ts +548 -415
- package/src/sbvr-api/sbvr-utils.ts +581 -259
- package/src/sbvr-api/translations.ts +248 -0
- package/src/sbvr-api/uri-parser.ts +63 -49
- package/src/sbvr-api/user.sbvr +2 -0
- package/src/server-glue/module.ts +16 -10
- package/src/server-glue/server.ts +5 -5
- package/tsconfig.dev.json +1 -0
- package/tsconfig.json +1 -2
- package/typings/lf-to-abstract-sql.d.ts +6 -9
- package/typings/memoizee.d.ts +1 -1
- package/.github/CODEOWNERS +0 -1
- package/circle.yml +0 -37
- package/docs/todo.txt +0 -22
- package/out/migrator/migrator.d.ts +0 -20
- package/out/migrator/migrator.js +0 -188
- package/out/migrator/migrator.js.map +0 -1
- package/src/migrator/migrator.ts +0 -286
@@ -0,0 +1,458 @@
|
|
1
|
+
import type { Result, Tx } from '../database-layer/db';
|
2
|
+
import type { Resolvable } from '../sbvr-api/common-types';
|
3
|
+
|
4
|
+
import { createHash } from 'crypto';
|
5
|
+
import { Engines } from '@balena/abstract-sql-compiler';
|
6
|
+
import * as _ from 'lodash';
|
7
|
+
import { TypedError } from 'typed-error';
|
8
|
+
import { migrator as migratorEnv } from '../config-loader/env';
|
9
|
+
export { migrator as migratorEnv } from '../config-loader/env';
|
10
|
+
import { PINEJS_ADVISORY_LOCK } from '../config-loader/env';
|
11
|
+
import { delay } from '../sbvr-api/control-flow';
|
12
|
+
|
13
|
+
// tslint:disable-next-line:no-var-requires
|
14
|
+
export const modelText = require('./migrations.sbvr');
|
15
|
+
export const migrations: Migrations = {
|
16
|
+
'15.0.0-data-types': async (tx, { db }) => {
|
17
|
+
switch (db.engine) {
|
18
|
+
case 'mysql':
|
19
|
+
await tx.executeSql(`\
|
20
|
+
ALTER TABLE "migration"
|
21
|
+
MODIFY "executed migrations" JSON NOT NULL;`);
|
22
|
+
await tx.executeSql(`\
|
23
|
+
ALTER TABLE "migration status"
|
24
|
+
MODIFY "is backing off" BOOLEAN NOT NULL;`);
|
25
|
+
break;
|
26
|
+
case 'postgres':
|
27
|
+
await tx.executeSql(`\
|
28
|
+
ALTER TABLE "migration"
|
29
|
+
ALTER COLUMN "executed migrations" SET DATA TYPE JSONB USING "executed migrations"::JSONB;`);
|
30
|
+
await tx.executeSql(`\
|
31
|
+
ALTER TABLE "migration status"
|
32
|
+
ALTER COLUMN "is backing off" DROP DEFAULT,
|
33
|
+
ALTER COLUMN "is backing off" SET DATA TYPE BOOLEAN USING "is backing off"::BOOLEAN,
|
34
|
+
ALTER COLUMN "is backing off" SET DEFAULT FALSE;`);
|
35
|
+
break;
|
36
|
+
// No need to migrate for websql
|
37
|
+
}
|
38
|
+
},
|
39
|
+
};
|
40
|
+
|
41
|
+
import * as sbvrUtils from '../sbvr-api/sbvr-utils';
|
42
|
+
export enum MigrationCategories {
|
43
|
+
'sync' = 'sync',
|
44
|
+
'async' = 'async',
|
45
|
+
}
|
46
|
+
export const defaultMigrationCategory = MigrationCategories.sync;
|
47
|
+
export type CategorizedMigrations = {
|
48
|
+
[key in MigrationCategories]: RunnableMigrations;
|
49
|
+
};
|
50
|
+
|
51
|
+
type SbvrUtils = typeof sbvrUtils;
|
52
|
+
export type MigrationTuple = [string, Migration];
|
53
|
+
export type MigrationFn = (tx: Tx, sbvrUtils: SbvrUtils) => Resolvable<void>;
|
54
|
+
export type RunnableMigrations = { [key: string]: Migration };
|
55
|
+
export type RunnableAsyncMigrations = { [key: string]: AsyncMigration };
|
56
|
+
export type Migrations = CategorizedMigrations | RunnableMigrations;
|
57
|
+
export type AsyncMigrationFn = (
|
58
|
+
tx: Tx,
|
59
|
+
options: { batchSize: number },
|
60
|
+
sbvrUtils: SbvrUtils,
|
61
|
+
) => Resolvable<number>;
|
62
|
+
|
63
|
+
type AddFn<T extends {}, x extends 'sync' | 'async'> = T & {
|
64
|
+
[key in `${x}Fn`]: key extends 'syncFn' ? MigrationFn : AsyncMigrationFn;
|
65
|
+
} & {
|
66
|
+
[key in `${x}Sql`]?: undefined;
|
67
|
+
};
|
68
|
+
type AddSql<T extends {}, x extends 'sync' | 'async'> = T & {
|
69
|
+
[key in `${x}Fn`]?: undefined;
|
70
|
+
} & {
|
71
|
+
[key in `${x}Sql`]: string;
|
72
|
+
};
|
73
|
+
|
74
|
+
export type BaseAsyncMigration = {
|
75
|
+
type?: MigrationCategories.async;
|
76
|
+
delayMS?: number | undefined;
|
77
|
+
backoffDelayMS?: number | undefined;
|
78
|
+
errorThreshold?: number | undefined;
|
79
|
+
asyncBatchSize?: number | undefined;
|
80
|
+
finalize?: boolean | undefined;
|
81
|
+
};
|
82
|
+
export type AsyncMigration =
|
83
|
+
| AddFn<BaseAsyncMigration, 'async' | 'sync'>
|
84
|
+
| AddSql<BaseAsyncMigration, 'async' | 'sync'>
|
85
|
+
| AddFn<AddSql<BaseAsyncMigration, 'async'>, 'sync'>
|
86
|
+
| AddFn<AddSql<BaseAsyncMigration, 'sync'>, 'async'>;
|
87
|
+
|
88
|
+
export function isAsyncMigration(
|
89
|
+
migration: string | MigrationFn | AsyncMigration | RunnableMigrations,
|
90
|
+
): migration is AsyncMigration {
|
91
|
+
return (
|
92
|
+
((typeof (migration as AsyncMigration).asyncFn === 'function' ||
|
93
|
+
typeof (migration as AsyncMigration).asyncSql === 'string') &&
|
94
|
+
(typeof (migration as AsyncMigration).syncFn === 'function' ||
|
95
|
+
typeof (migration as AsyncMigration).syncSql === 'string')) ||
|
96
|
+
(migration as AsyncMigration).type === MigrationCategories.async
|
97
|
+
);
|
98
|
+
}
|
99
|
+
|
100
|
+
export function isSyncMigration(
|
101
|
+
migration: string | MigrationFn | RunnableMigrations | AsyncMigration,
|
102
|
+
): migration is MigrationFn {
|
103
|
+
return typeof migration === 'function' || typeof migration === 'string';
|
104
|
+
}
|
105
|
+
export function areCategorizedMigrations(
|
106
|
+
$migrations: Migrations,
|
107
|
+
): $migrations is CategorizedMigrations {
|
108
|
+
const containsCategories = Object.keys(MigrationCategories).some(
|
109
|
+
(key) => key in $migrations,
|
110
|
+
);
|
111
|
+
if (
|
112
|
+
containsCategories &&
|
113
|
+
Object.keys($migrations).some((key) => !(key in MigrationCategories))
|
114
|
+
) {
|
115
|
+
throw new Error(
|
116
|
+
'Mixing categorized and uncategorized migrations is not supported',
|
117
|
+
);
|
118
|
+
}
|
119
|
+
return containsCategories;
|
120
|
+
}
|
121
|
+
|
122
|
+
export type Migration = string | MigrationFn | AsyncMigration;
|
123
|
+
export class MigrationError extends TypedError {}
|
124
|
+
|
125
|
+
export type MigrationStatus = {
|
126
|
+
migration_key: string;
|
127
|
+
start_time: Date;
|
128
|
+
last_run_time: Date | null;
|
129
|
+
run_count: number;
|
130
|
+
migrated_row_count: number;
|
131
|
+
error_count: number;
|
132
|
+
converged_time: Date | undefined;
|
133
|
+
is_backing_off: boolean;
|
134
|
+
};
|
135
|
+
|
136
|
+
export const getRunnableAsyncMigrations = (
|
137
|
+
$migrations: Migrations,
|
138
|
+
): RunnableAsyncMigrations | undefined => {
|
139
|
+
if ($migrations[MigrationCategories.async]) {
|
140
|
+
if (
|
141
|
+
Object.values($migrations[MigrationCategories.async]).some(
|
142
|
+
(migration) => !isAsyncMigration(migration),
|
143
|
+
) ||
|
144
|
+
typeof $migrations[MigrationCategories.async] !== 'object'
|
145
|
+
) {
|
146
|
+
throw new Error(
|
147
|
+
`All loaded async migrations need to be of type: ${MigrationCategories.async}`,
|
148
|
+
);
|
149
|
+
}
|
150
|
+
return $migrations[MigrationCategories.async] as RunnableAsyncMigrations;
|
151
|
+
}
|
152
|
+
};
|
153
|
+
|
154
|
+
// migration loader should either get migrations from model
|
155
|
+
// or from the filepath
|
156
|
+
export const getRunnableSyncMigrations = (
|
157
|
+
$migrations: Migrations,
|
158
|
+
): RunnableMigrations => {
|
159
|
+
if (areCategorizedMigrations($migrations)) {
|
160
|
+
const runnableMigrations: RunnableMigrations = {};
|
161
|
+
for (const [category, categoryMigrations] of Object.entries($migrations)) {
|
162
|
+
if (category in MigrationCategories) {
|
163
|
+
for (const [key, migration] of Object.entries(
|
164
|
+
categoryMigrations as Migrations,
|
165
|
+
)) {
|
166
|
+
if (isAsyncMigration(migration)) {
|
167
|
+
if (migration.finalize) {
|
168
|
+
runnableMigrations[key] = migration.syncFn ?? migration.syncSql;
|
169
|
+
}
|
170
|
+
} else if (isSyncMigration(migration)) {
|
171
|
+
runnableMigrations[key] = migration;
|
172
|
+
}
|
173
|
+
}
|
174
|
+
}
|
175
|
+
}
|
176
|
+
return runnableMigrations;
|
177
|
+
}
|
178
|
+
return $migrations;
|
179
|
+
};
|
180
|
+
|
181
|
+
// turns {"key1": migration, "key3": migration, "key2": migration}
|
182
|
+
// into [["key1", migration], ["key2", migration], ["key3", migration]]
|
183
|
+
export const filterAndSortPendingMigrations = (
|
184
|
+
$migrations: NonNullable<RunnableMigrations | RunnableAsyncMigrations>,
|
185
|
+
executedMigrations: string[],
|
186
|
+
): MigrationTuple[] =>
|
187
|
+
(_($migrations).omit(executedMigrations) as _.Object<typeof $migrations>)
|
188
|
+
.toPairs()
|
189
|
+
.sortBy(([migrationKey]) => migrationKey)
|
190
|
+
.value();
|
191
|
+
|
192
|
+
// Tagged template to convert binds from `?` format to the necessary output format,
|
193
|
+
// eg `$1`/`$2`/etc for postgres
|
194
|
+
export const binds = (strings: TemplateStringsArray, ...bindNums: number[]) =>
|
195
|
+
strings
|
196
|
+
.map((str, i) => {
|
197
|
+
if (i === bindNums.length) {
|
198
|
+
return str;
|
199
|
+
}
|
200
|
+
if (i + 1 !== bindNums[i]) {
|
201
|
+
throw new SyntaxError('Migration sql binds must be sequential');
|
202
|
+
}
|
203
|
+
if (sbvrUtils.db.engine === Engines.postgres) {
|
204
|
+
return str + `$${bindNums[i]}`;
|
205
|
+
}
|
206
|
+
return str + `?`;
|
207
|
+
})
|
208
|
+
.join('');
|
209
|
+
|
210
|
+
/**
|
211
|
+
* Lock mechanism that tries to write model name to the migration lock table
|
212
|
+
* This creates an index write lock on this row. This lock is never persisted
|
213
|
+
* as the lock is hold only in the transaction and is delete at the end of the
|
214
|
+
* transaction.
|
215
|
+
*
|
216
|
+
* Disadvantage is that no blocking-wait queue can be generated on this lock mechanism
|
217
|
+
* It's database engine agnostic and works also for webSQL
|
218
|
+
*/
|
219
|
+
const $lockMigrations = async <T>(
|
220
|
+
tx: Tx,
|
221
|
+
modelName: string,
|
222
|
+
fn: () => Promise<T>,
|
223
|
+
): Promise<T | undefined> => {
|
224
|
+
try {
|
225
|
+
await tx.executeSql(
|
226
|
+
binds`
|
227
|
+
DELETE FROM "migration lock"
|
228
|
+
WHERE "model name" = ${1}
|
229
|
+
AND "created at" < ${2}`,
|
230
|
+
[modelName, new Date(Date.now() - migratorEnv.lockTimeout)],
|
231
|
+
);
|
232
|
+
await tx.executeSql(
|
233
|
+
binds`
|
234
|
+
INSERT INTO "migration lock" ("model name")
|
235
|
+
VALUES (${1})`,
|
236
|
+
[modelName],
|
237
|
+
);
|
238
|
+
} catch (err: any) {
|
239
|
+
await delay(migratorEnv.lockFailDelay);
|
240
|
+
throw err;
|
241
|
+
}
|
242
|
+
try {
|
243
|
+
return await fn();
|
244
|
+
} finally {
|
245
|
+
try {
|
246
|
+
await tx.executeSql(
|
247
|
+
binds`
|
248
|
+
DELETE FROM "migration lock"
|
249
|
+
WHERE "model name" = ${1}`,
|
250
|
+
[modelName],
|
251
|
+
);
|
252
|
+
} catch {
|
253
|
+
// We ignore errors here as it's mostly likely caused by the migration failing and
|
254
|
+
// rolling back the transaction, and if we rethrow here we'll overwrite the real error
|
255
|
+
// making it much harder for users to see what went wrong and fix it
|
256
|
+
}
|
257
|
+
}
|
258
|
+
};
|
259
|
+
|
260
|
+
export const lockMigrations = async <T>(
|
261
|
+
options: { tx: Tx; modelName: string; blocking: boolean },
|
262
|
+
fn: () => Promise<T>,
|
263
|
+
): Promise<T | undefined> => {
|
264
|
+
if (!(await migrationTablesExist(options.tx))) {
|
265
|
+
return;
|
266
|
+
}
|
267
|
+
|
268
|
+
if (sbvrUtils.db.engine === Engines.websql) {
|
269
|
+
return $lockMigrations(options.tx, options.modelName, fn);
|
270
|
+
} else if (sbvrUtils.db.engine === Engines.mysql) {
|
271
|
+
// right now the mysql locks are not testable
|
272
|
+
// pinejs generates models that are not executable on mysql databases
|
273
|
+
return $lockMigrations(options.tx, options.modelName, fn);
|
274
|
+
} else if (sbvrUtils.db.engine === Engines.postgres) {
|
275
|
+
// getTxLevelLock expects a 4 byte integer as the lock key.
|
276
|
+
// Therefore the model name is hashed and the first 4 bytes are taken as the Integer representation.
|
277
|
+
const modelKey: number = createHash('shake128', { outputLength: 4 })
|
278
|
+
.update('resin')
|
279
|
+
.digest()
|
280
|
+
.readInt32BE();
|
281
|
+
const lockStatus = await options.tx.getTxLevelLock(
|
282
|
+
PINEJS_ADVISORY_LOCK.namespaceKey,
|
283
|
+
modelKey,
|
284
|
+
options.blocking,
|
285
|
+
);
|
286
|
+
|
287
|
+
if (lockStatus) {
|
288
|
+
return await fn();
|
289
|
+
}
|
290
|
+
} else {
|
291
|
+
// we report any error here, as no error should happen at all
|
292
|
+
throw new Error(`unknown database engine for getting migration locks`);
|
293
|
+
}
|
294
|
+
};
|
295
|
+
|
296
|
+
export const setExecutedMigrations = async (
|
297
|
+
tx: Tx,
|
298
|
+
modelName: string,
|
299
|
+
executedMigrations: string[],
|
300
|
+
): Promise<void> => {
|
301
|
+
if (!(await migrationTablesExist(tx))) {
|
302
|
+
return;
|
303
|
+
}
|
304
|
+
|
305
|
+
const stringifiedMigrations = await sbvrUtils.sbvrTypes.JSON.validate(
|
306
|
+
executedMigrations,
|
307
|
+
true,
|
308
|
+
);
|
309
|
+
|
310
|
+
const { rowsAffected } = await tx.executeSql(
|
311
|
+
binds`
|
312
|
+
UPDATE "migration"
|
313
|
+
SET "model name" = ${1},
|
314
|
+
"executed migrations" = ${2}
|
315
|
+
WHERE "migration"."model name" = ${3}`,
|
316
|
+
[modelName, stringifiedMigrations, modelName],
|
317
|
+
);
|
318
|
+
|
319
|
+
if (rowsAffected === 0) {
|
320
|
+
await tx.executeSql(
|
321
|
+
binds`
|
322
|
+
INSERT INTO "migration" ("model name", "executed migrations")
|
323
|
+
VALUES (${1}, ${2})`,
|
324
|
+
[modelName, stringifiedMigrations],
|
325
|
+
);
|
326
|
+
}
|
327
|
+
};
|
328
|
+
|
329
|
+
export const getExecutedMigrations = async (
|
330
|
+
tx: Tx,
|
331
|
+
modelName: string,
|
332
|
+
): Promise<string[]> => {
|
333
|
+
if (!(await migrationTablesExist(tx))) {
|
334
|
+
return [];
|
335
|
+
}
|
336
|
+
|
337
|
+
const { rows } = await tx.executeSql(
|
338
|
+
binds`
|
339
|
+
SELECT "migration"."executed migrations" AS "executed_migrations"
|
340
|
+
FROM "migration"
|
341
|
+
WHERE "migration"."model name" = ${1}`,
|
342
|
+
[modelName],
|
343
|
+
);
|
344
|
+
|
345
|
+
const data = rows[0];
|
346
|
+
if (data == null) {
|
347
|
+
return [];
|
348
|
+
}
|
349
|
+
return sbvrUtils.sbvrTypes.JSON.fetchProcessing(data.executed_migrations);
|
350
|
+
};
|
351
|
+
|
352
|
+
export const migrationTablesExist = async (tx: Tx) => {
|
353
|
+
const tables = ['migration', 'migration lock', 'migration status'];
|
354
|
+
const where = tables.map((tableName) => `name = '${tableName}'`).join(' OR ');
|
355
|
+
const result = await tx.tableList(where);
|
356
|
+
return result.rows.length === tables.length;
|
357
|
+
};
|
358
|
+
|
359
|
+
export const initMigrationStatus = async (
|
360
|
+
tx: Tx,
|
361
|
+
migrationStatus: MigrationStatus,
|
362
|
+
): Promise<Result | undefined> => {
|
363
|
+
try {
|
364
|
+
return await tx.executeSql(
|
365
|
+
binds`
|
366
|
+
INSERT INTO "migration status" ("migration key", "start time", "is backing off", "run count")
|
367
|
+
SELECT ${1}, ${2}, ${3}, ${4}
|
368
|
+
WHERE NOT EXISTS (SELECT 1 FROM "migration status" WHERE "migration key" = ${5})
|
369
|
+
`,
|
370
|
+
[
|
371
|
+
migrationStatus['migration_key'],
|
372
|
+
migrationStatus['start_time'],
|
373
|
+
migrationStatus['is_backing_off'],
|
374
|
+
migrationStatus['run_count'],
|
375
|
+
migrationStatus['migration_key'],
|
376
|
+
],
|
377
|
+
);
|
378
|
+
} catch (err: any) {
|
379
|
+
// we report any error here, as no error should happen at all
|
380
|
+
throw new Error(`unknown error in init migration status: ${err}`);
|
381
|
+
}
|
382
|
+
};
|
383
|
+
|
384
|
+
// Update all fields of migration status for cross-instance sync
|
385
|
+
export const updateMigrationStatus = async (
|
386
|
+
tx: Tx,
|
387
|
+
migrationStatus: MigrationStatus,
|
388
|
+
): Promise<Result | undefined> => {
|
389
|
+
try {
|
390
|
+
return await tx.executeSql(
|
391
|
+
binds`
|
392
|
+
UPDATE "migration status"
|
393
|
+
SET
|
394
|
+
"run count" = ${1},
|
395
|
+
"last run time" = ${2},
|
396
|
+
"migrated row count" = ${3},
|
397
|
+
"error count" = ${4},
|
398
|
+
"converged time" = ${5},
|
399
|
+
"is backing off" = ${6}
|
400
|
+
WHERE "migration status"."migration key" = ${7};`,
|
401
|
+
[
|
402
|
+
migrationStatus['run_count'],
|
403
|
+
migrationStatus['last_run_time'],
|
404
|
+
migrationStatus['migrated_row_count'],
|
405
|
+
migrationStatus['error_count'],
|
406
|
+
migrationStatus['converged_time'],
|
407
|
+
migrationStatus['is_backing_off'],
|
408
|
+
migrationStatus['migration_key'],
|
409
|
+
],
|
410
|
+
);
|
411
|
+
} catch (err: any) {
|
412
|
+
// we report any error here, as no error should happen at all
|
413
|
+
throw new Error(`unknown error in update migration status: ${err}`);
|
414
|
+
}
|
415
|
+
};
|
416
|
+
|
417
|
+
export const readMigrationStatus = async (
|
418
|
+
tx: Tx,
|
419
|
+
migrationKey: string,
|
420
|
+
): Promise<MigrationStatus | undefined> => {
|
421
|
+
try {
|
422
|
+
const { rows } = await tx.executeSql(
|
423
|
+
binds`
|
424
|
+
SELECT *
|
425
|
+
FROM "migration status"
|
426
|
+
WHERE "migration status"."migration key" = ${1}
|
427
|
+
LIMIT 1;`,
|
428
|
+
[migrationKey],
|
429
|
+
);
|
430
|
+
|
431
|
+
const data = rows[0];
|
432
|
+
if (data == null) {
|
433
|
+
return;
|
434
|
+
}
|
435
|
+
|
436
|
+
return {
|
437
|
+
migration_key: data['migration key'],
|
438
|
+
start_time: sbvrUtils.sbvrTypes['Date Time'].fetchProcessing(
|
439
|
+
data['start time'],
|
440
|
+
),
|
441
|
+
last_run_time: sbvrUtils.sbvrTypes['Date Time'].fetchProcessing(
|
442
|
+
data['last run time'],
|
443
|
+
),
|
444
|
+
run_count: data['run count'],
|
445
|
+
migrated_row_count: data['migrated row count'],
|
446
|
+
error_count: data['error count'],
|
447
|
+
converged_time: sbvrUtils.sbvrTypes['Date Time'].fetchProcessing(
|
448
|
+
data['converged time'],
|
449
|
+
),
|
450
|
+
is_backing_off: sbvrUtils.sbvrTypes.Boolean.fetchProcessing(
|
451
|
+
data['is backing off'],
|
452
|
+
),
|
453
|
+
};
|
454
|
+
} catch (err: any) {
|
455
|
+
// we report any error here, as no error should happen at all
|
456
|
+
throw new Error(`unknown error in read migration status: ${err}`);
|
457
|
+
}
|
458
|
+
};
|
@@ -3,7 +3,7 @@ import type {
|
|
3
3
|
AbstractSqlTable,
|
4
4
|
} from '@balena/abstract-sql-compiler';
|
5
5
|
|
6
|
-
import
|
6
|
+
import sbvrTypes, { SbvrType } from '@balena/sbvr-types';
|
7
7
|
|
8
8
|
// tslint:disable-next-line:no-var-requires
|
9
9
|
const { version }: { version: string } = require('../../package.json');
|
@@ -21,17 +21,14 @@ const forEachUniqueTable = <T>(
|
|
21
21
|
const usedTableNames: { [tableName: string]: true } = {};
|
22
22
|
|
23
23
|
const result = [];
|
24
|
-
for (const key
|
25
|
-
if (
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
)
|
32
|
-
usedTableNames[table.name] = true;
|
33
|
-
result.push(callback(key, table));
|
34
|
-
}
|
24
|
+
for (const [key, table] of Object.entries(model)) {
|
25
|
+
if (
|
26
|
+
typeof table !== 'string' &&
|
27
|
+
!table.primitive &&
|
28
|
+
!usedTableNames[table.name]
|
29
|
+
) {
|
30
|
+
usedTableNames[table.name] = true;
|
31
|
+
result.push(callback(key, table));
|
35
32
|
}
|
36
33
|
}
|
37
34
|
return result;
|
@@ -42,12 +39,12 @@ export const generateODataMetadata = (
|
|
42
39
|
abstractSqlModel: AbstractSqlModel,
|
43
40
|
) => {
|
44
41
|
const complexTypes: { [fieldType: string]: string } = {};
|
45
|
-
const resolveDataType = (fieldType:
|
42
|
+
const resolveDataType = (fieldType: keyof typeof sbvrTypes): string => {
|
46
43
|
if (sbvrTypes[fieldType] == null) {
|
47
44
|
console.error('Could not resolve type', fieldType);
|
48
45
|
throw new Error('Could not resolve type' + fieldType);
|
49
46
|
}
|
50
|
-
const { complexType } = sbvrTypes[fieldType].types.odata;
|
47
|
+
const { complexType } = (sbvrTypes[fieldType] as SbvrType).types.odata;
|
51
48
|
if (complexType != null) {
|
52
49
|
complexTypes[fieldType] = complexType;
|
53
50
|
}
|
@@ -104,7 +101,7 @@ export const generateODataMetadata = (
|
|
104
101
|
fields
|
105
102
|
.filter(({ dataType }) => dataType !== 'ForeignKey')
|
106
103
|
.map(({ dataType, fieldName, required }) => {
|
107
|
-
dataType = resolveDataType(dataType);
|
104
|
+
dataType = resolveDataType(dataType as keyof typeof sbvrTypes);
|
108
105
|
fieldName = getResourceName(fieldName);
|
109
106
|
return `<Property Name="${fieldName}" Type="${dataType}" Nullable="${!required}" />`;
|
110
107
|
})
|
@@ -2,6 +2,7 @@ import type * as Express from 'express';
|
|
2
2
|
import type * as Passport from 'passport';
|
3
3
|
import type * as PassportLocal from 'passport-local';
|
4
4
|
import type * as ConfigLoader from '../config-loader/config-loader';
|
5
|
+
import type { User } from '../sbvr-api/sbvr-utils';
|
5
6
|
|
6
7
|
import * as permissions from '../sbvr-api/permissions';
|
7
8
|
|
@@ -9,7 +10,7 @@ import * as permissions from '../sbvr-api/permissions';
|
|
9
10
|
export let login: (
|
10
11
|
fn: (
|
11
12
|
err: any,
|
12
|
-
user: {} | undefined,
|
13
|
+
user: {} | null | false | undefined,
|
13
14
|
req: Express.Request,
|
14
15
|
res: Express.Response,
|
15
16
|
next: Express.NextFunction,
|
@@ -46,26 +47,27 @@ const setup: ConfigLoader.SetupFunction = async (app: Express.Application) => {
|
|
46
47
|
done(null, user);
|
47
48
|
});
|
48
49
|
|
49
|
-
passport.deserializeUser((user, done) => {
|
50
|
+
passport.deserializeUser<User>((user, done) => {
|
50
51
|
done(null, user);
|
51
52
|
});
|
52
53
|
|
53
54
|
passport.use(new LocalStrategy(checkPassword));
|
54
55
|
|
55
56
|
login = (fn) => (req, res, next) =>
|
56
|
-
passport.authenticate('local', (err
|
57
|
-
if (err || user == null) {
|
57
|
+
passport.authenticate('local', ((err, user) => {
|
58
|
+
if (err || user == null || user === false) {
|
58
59
|
fn(err, user, req, res, next);
|
59
60
|
return;
|
60
61
|
}
|
61
62
|
req.login(user, (error) => {
|
62
63
|
fn(error, user, req, res, next);
|
63
64
|
});
|
64
|
-
})(req, res, next);
|
65
|
+
}) as Passport.AuthenticateCallback)(req, res, next);
|
65
66
|
|
66
67
|
logout = (req, _res, next) => {
|
67
|
-
req.logout()
|
68
|
-
|
68
|
+
req.logout((error) => {
|
69
|
+
error ? next(error) : next();
|
70
|
+
});
|
69
71
|
};
|
70
72
|
} else {
|
71
73
|
let loggedIn = false;
|
@@ -146,7 +146,6 @@ export class PinejsSessionStore extends Store {
|
|
146
146
|
},
|
147
147
|
options: {
|
148
148
|
$count: {
|
149
|
-
$select: 'session_id',
|
150
149
|
$filter: {
|
151
150
|
expiry_time: {
|
152
151
|
$ge: Date.now(),
|
@@ -173,6 +172,21 @@ export class PinejsSessionStore extends Store {
|
|
173
172
|
ALTER TABLE "session"
|
174
173
|
ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
|
175
174
|
`,
|
175
|
+
'15.0.0-data-types': async (tx, sbvrUtils) => {
|
176
|
+
switch (sbvrUtils.db.engine) {
|
177
|
+
case 'mysql':
|
178
|
+
await tx.executeSql(`\
|
179
|
+
ALTER TABLE "session"
|
180
|
+
MODIFY "data" JSON NOT NULL;`);
|
181
|
+
break;
|
182
|
+
case 'postgres':
|
183
|
+
await tx.executeSql(`\
|
184
|
+
ALTER TABLE "session"
|
185
|
+
ALTER COLUMN "data" SET DATA TYPE JSONB USING "data"::JSONB;`);
|
186
|
+
break;
|
187
|
+
// No need to migrate for websql
|
188
|
+
}
|
189
|
+
},
|
176
190
|
},
|
177
191
|
},
|
178
192
|
],
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import * as _ from 'lodash';
|
2
2
|
|
3
3
|
import * as AbstractSQLCompiler from '@balena/abstract-sql-compiler';
|
4
|
+
import type { BindKey } from '@balena/odata-parser';
|
4
5
|
import {
|
5
6
|
ODataBinds,
|
6
7
|
odataNameToSqlName,
|
@@ -8,7 +9,6 @@ import {
|
|
8
9
|
} from '@balena/odata-to-abstract-sql';
|
9
10
|
import deepFreeze = require('deep-freeze');
|
10
11
|
import * as memoize from 'memoizee';
|
11
|
-
import memoizeWeak = require('memoizee/weak');
|
12
12
|
import * as env from '../config-loader/env';
|
13
13
|
import { BadRequestError, SqlCompilationError } from './errors';
|
14
14
|
import * as sbvrUtils from './sbvr-utils';
|
@@ -16,14 +16,13 @@ import { ODataRequest } from './uri-parser';
|
|
16
16
|
|
17
17
|
const getMemoizedCompileRule = memoize(
|
18
18
|
(engine: AbstractSQLCompiler.Engines) =>
|
19
|
-
|
19
|
+
env.createCache(
|
20
|
+
'abstractSqlCompiler',
|
20
21
|
(abstractSqlQuery: AbstractSQLCompiler.AbstractSqlQuery) => {
|
21
|
-
const sqlQuery =
|
22
|
-
abstractSqlQuery
|
23
|
-
|
24
|
-
|
25
|
-
abstractSqlQuery,
|
26
|
-
);
|
22
|
+
const sqlQuery =
|
23
|
+
AbstractSQLCompiler[engine].compileRule(abstractSqlQuery);
|
24
|
+
const modifiedFields =
|
25
|
+
AbstractSQLCompiler[engine].getModifiedFields(abstractSqlQuery);
|
27
26
|
if (modifiedFields != null) {
|
28
27
|
deepFreeze(modifiedFields);
|
29
28
|
}
|
@@ -32,7 +31,7 @@ const getMemoizedCompileRule = memoize(
|
|
32
31
|
modifiedFields,
|
33
32
|
};
|
34
33
|
},
|
35
|
-
{
|
34
|
+
{ weak: true },
|
36
35
|
),
|
37
36
|
{ primitive: true },
|
38
37
|
);
|
@@ -49,7 +48,7 @@ export const compileRequest = (request: ODataRequest) => {
|
|
49
48
|
);
|
50
49
|
request.sqlQuery = sqlQuery;
|
51
50
|
request.modifiedFields = modifiedFields;
|
52
|
-
} catch (err) {
|
51
|
+
} catch (err: any) {
|
53
52
|
sbvrUtils.api[request.vocabulary].logger.error(
|
54
53
|
'Failed to compile abstract sql: ',
|
55
54
|
request.abstractSqlQuery,
|
@@ -97,7 +96,8 @@ export const getAndCheckBindValues = async (
|
|
97
96
|
|
98
97
|
const sqlTableName = odataNameToSqlName(tableName);
|
99
98
|
const sqlFieldName = odataNameToSqlName(fieldName);
|
100
|
-
const
|
99
|
+
const table = sqlModelTables[sqlTableName];
|
100
|
+
const maybeField = (table.modifyFields ?? table.fields).find(
|
101
101
|
(f) => f.fieldName === sqlFieldName,
|
102
102
|
);
|
103
103
|
if (maybeField == null) {
|
@@ -124,7 +124,7 @@ export const getAndCheckBindValues = async (
|
|
124
124
|
throw new Error('Invalid binding');
|
125
125
|
}
|
126
126
|
let dataType;
|
127
|
-
[dataType, value] = odataBinds[bindValue];
|
127
|
+
[dataType, value] = odataBinds[bindValue as BindKey];
|
128
128
|
field = { dataType };
|
129
129
|
} else {
|
130
130
|
throw new Error(`Unknown binding: ${binding}`);
|
@@ -141,7 +141,7 @@ export const getAndCheckBindValues = async (
|
|
141
141
|
|
142
142
|
try {
|
143
143
|
return await AbstractSQLCompiler[engine].dataTypeValidate(value, field);
|
144
|
-
} catch (err) {
|
144
|
+
} catch (err: any) {
|
145
145
|
throw new BadRequestError(`"${fieldName}" ${err.message}`);
|
146
146
|
}
|
147
147
|
}),
|
@@ -175,7 +175,10 @@ const checkModifiedFields = (
|
|
175
175
|
};
|
176
176
|
export const isRuleAffected = (
|
177
177
|
rule: AbstractSQLCompiler.SqlRule,
|
178
|
-
request?:
|
178
|
+
request?: Pick<
|
179
|
+
ODataRequest,
|
180
|
+
'abstractSqlQuery' | 'modifiedFields' | 'method' | 'vocabulary'
|
181
|
+
>,
|
179
182
|
) => {
|
180
183
|
// If there is no abstract sql query then nothing was modified
|
181
184
|
if (request?.abstractSqlQuery == null) {
|
@@ -1,4 +1,4 @@
|
|
1
|
-
export { AnyObject } from 'pinejs-client-core';
|
1
|
+
export { AnyObject, Dictionary } from 'pinejs-client-core';
|
2
2
|
|
3
3
|
type Overwrite<T, U> = Pick<T, Exclude<keyof T, keyof U>> & U;
|
4
4
|
export type RequiredField<T, F extends keyof T> = Overwrite<
|
@@ -10,3 +10,4 @@ export type OptionalField<T, F extends keyof T> = Overwrite<
|
|
10
10
|
Partial<Pick<T, F>>
|
11
11
|
>;
|
12
12
|
export type Resolvable<R> = R | PromiseLike<R>;
|
13
|
+
export type Tail<T extends any[]> = T extends [any, ...infer U] ? U : never;
|