@proteinjs/db 1.30.0 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/generated/index.js +1 -1
- package/dist/generated/index.js.map +1 -1
- package/dist/generated/test/index.d.ts.map +1 -1
- package/dist/generated/test/index.js +3 -1
- package/dist/generated/test/index.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/src/CursorWindowPager.d.ts +74 -0
- package/dist/src/CursorWindowPager.d.ts.map +1 -0
- package/dist/src/CursorWindowPager.js +178 -0
- package/dist/src/CursorWindowPager.js.map +1 -0
- package/dist/src/Db.d.ts +10 -0
- package/dist/src/Db.d.ts.map +1 -1
- package/dist/src/Db.js +10 -0
- package/dist/src/Db.js.map +1 -1
- package/dist/src/MigrationRunner.d.ts +58 -1
- package/dist/src/MigrationRunner.d.ts.map +1 -1
- package/dist/src/MigrationRunner.js +108 -2
- package/dist/src/MigrationRunner.js.map +1 -1
- package/dist/src/RecordIterator.d.ts +16 -6
- package/dist/src/RecordIterator.d.ts.map +1 -1
- package/dist/src/RecordIterator.js +50 -37
- package/dist/src/RecordIterator.js.map +1 -1
- package/dist/src/tables/MigrationTable.d.ts +10 -0
- package/dist/src/tables/MigrationTable.d.ts.map +1 -1
- package/dist/src/tables/MigrationTable.js +1 -0
- package/dist/src/tables/MigrationTable.js.map +1 -1
- package/dist/test/CursorWindowPager.test.d.ts +2 -0
- package/dist/test/CursorWindowPager.test.d.ts.map +1 -0
- package/dist/test/CursorWindowPager.test.js +213 -0
- package/dist/test/CursorWindowPager.test.js.map +1 -0
- package/dist/test/index.d.ts +2 -0
- package/dist/test/index.d.ts.map +1 -1
- package/dist/test/index.js +2 -0
- package/dist/test/index.js.map +1 -1
- package/dist/test/reusable/RecordIteratorTests.d.ts +13 -0
- package/dist/test/reusable/RecordIteratorTests.d.ts.map +1 -0
- package/dist/test/reusable/RecordIteratorTests.js +448 -0
- package/dist/test/reusable/RecordIteratorTests.js.map +1 -0
- package/dist/test/util/DbTestEnvironment.d.ts.map +1 -1
- package/dist/test/util/DbTestEnvironment.js +2 -1
- package/dist/test/util/DbTestEnvironment.js.map +1 -1
- package/dist/test/util/tables/recordIteratorTestTables.d.ts +25 -0
- package/dist/test/util/tables/recordIteratorTestTables.d.ts.map +1 -0
- package/dist/test/util/tables/recordIteratorTestTables.js +38 -0
- package/dist/test/util/tables/recordIteratorTestTables.js.map +1 -0
- package/generated/index.ts +1 -1
- package/generated/test/index.ts +3 -1
- package/index.ts +1 -0
- package/package.json +3 -3
- package/src/CursorWindowPager.ts +153 -0
- package/src/Db.ts +10 -0
- package/src/MigrationRunner.ts +114 -2
- package/src/RecordIterator.ts +34 -24
- package/src/tables/MigrationTable.ts +11 -1
- package/test/CursorWindowPager.test.ts +185 -0
- package/test/index.ts +2 -0
- package/test/reusable/RecordIteratorTests.ts +190 -0
- package/test/util/DbTestEnvironment.ts +2 -0
- package/test/util/tables/recordIteratorTestTables.ts +23 -0
package/index.ts
CHANGED
|
@@ -10,6 +10,7 @@ export * from './src/reference/Reference';
|
|
|
10
10
|
export * from './src/UpdatePreserving';
|
|
11
11
|
export * from './src/reference/ReferenceCache';
|
|
12
12
|
export * from './src/RecordIterator';
|
|
13
|
+
export * from './src/CursorWindowPager';
|
|
13
14
|
export * from './src/source/SourceRecord';
|
|
14
15
|
export * from './src/source/SourceRecordRepo';
|
|
15
16
|
export * from './src/MigrationRunner';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@proteinjs/db",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.31.0",
|
|
4
4
|
"main": "./dist/generated/index.js",
|
|
5
5
|
"types": "./dist/generated/index.d.ts",
|
|
6
6
|
"exports": {
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"test": "jest --passWithNoTests"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@proteinjs/db-query": "^1.
|
|
44
|
+
"@proteinjs/db-query": "^1.7.0",
|
|
45
45
|
"@proteinjs/logger": "^1.0.21",
|
|
46
46
|
"@proteinjs/reflection": "^1.1.12",
|
|
47
47
|
"@proteinjs/serializer": "^1.1.10",
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
"ts-jest": "29.1.1",
|
|
67
67
|
"typescript": "5.2.2"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "d84e548336f96b2541c1fbbe6a9ad6074df610c7"
|
|
70
70
|
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { Serializer } from '@proteinjs/serializer';
|
|
2
|
+
import { Condition, LogicalGroup, QueryBuilder, SortCriteria } from '@proteinjs/db-query';
|
|
3
|
+
import { Db, getDb } from './Db';
|
|
4
|
+
import { QueryOptions } from './services/DbService';
|
|
5
|
+
import { Table } from './Table';
|
|
6
|
+
import { Record } from './Record';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A cursor anchor names the last row a window served: the row's value for each effective sort
|
|
10
|
+
* criterion, in criteria order. The final criterion is always the unique `id` axis, so an
|
|
11
|
+
* anchor always identifies exactly one row — ties on the other axes can never blur where the
|
|
12
|
+
* next window starts.
|
|
13
|
+
*/
|
|
14
|
+
export type CursorAnchor = {
|
|
15
|
+
values: unknown[];
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export type AnchoredWindow<T> = {
|
|
19
|
+
rows: T[];
|
|
20
|
+
/** Anchor for the window after this one; null = the data set is exhausted. */
|
|
21
|
+
nextAnchor: CursorAnchor | null;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* THE owner of cursor-window paging over a table query — every windowed consumer (the
|
|
26
|
+
* server-side `RecordIterator`, the UI's `QueryCursorLoader`) frames its windows here.
|
|
27
|
+
*
|
|
28
|
+
* Each window builds a FRESH query (builders are mutable) carrying a lexicographic
|
|
29
|
+
* continuation past the anchor row, the effective sort, and `paginate(0, windowSize)` — so
|
|
30
|
+
* rows inserted or deleted while paging can never shift the window frame, the offset-paging
|
|
31
|
+
* drift class (`OFFSET n` frames windows by position; concurrent writes slide unvisited rows
|
|
32
|
+
* into the consumed range or consumed rows back into the next window).
|
|
33
|
+
*
|
|
34
|
+
* The effective sort is the consumer's criteria with a unique `id` tiebreak appended (an
|
|
35
|
+
* explicit `id` criterion ends the list — axes after it can never influence order). Ties on
|
|
36
|
+
* consumer axes therefore continue across window boundaries instead of being skipped by a
|
|
37
|
+
* bare `field < cursor`. Sort axes should be non-null columns: a window whose tail carries a
|
|
38
|
+
* NULL sort value ends paging (a null can't anchor a lexicographic continuation — under a
|
|
39
|
+
* descending sort, null rows collect at the tail). `byValues` sorts cannot be paged by cursor
|
|
40
|
+
* and are rejected.
|
|
41
|
+
*
|
|
42
|
+
* Anchors are plain in-memory values; `encodeAnchor`/`decodeAnchor` (house `Serializer`, so
|
|
43
|
+
* dates/moments survive) turn them into strings for consumers whose cursors must serialize —
|
|
44
|
+
* e.g. react-query page params.
|
|
45
|
+
*/
|
|
46
|
+
export class CursorWindowPager<T extends Record> {
|
|
47
|
+
private effectiveSort: SortCriteria<T>[];
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* @param table the table to load windows from
|
|
51
|
+
* @param createQuery factory producing a FRESH conditions-only QueryBuilder per call —
|
|
52
|
+
* every window builds its own to hang the cursor conditions, sort, and pagination on
|
|
53
|
+
* @param sort the consumer's sort criteria; the effective sort appends the `id` tiebreak.
|
|
54
|
+
* Omitted/empty = pure `id` order (the complete-iteration default).
|
|
55
|
+
* @param options `db` to ride a specific Db (scoped/system/transactional); defaults to
|
|
56
|
+
* `getDb()` per window. `queryOptions` pass through to every window query.
|
|
57
|
+
*/
|
|
58
|
+
constructor(
|
|
59
|
+
private table: Table<T>,
|
|
60
|
+
private createQuery: () => QueryBuilder<T>,
|
|
61
|
+
sort?: SortCriteria<T>[],
|
|
62
|
+
private options?: { db?: Db; queryOptions?: QueryOptions<T> }
|
|
63
|
+
) {
|
|
64
|
+
this.effectiveSort = this.normalizeSort(sort ?? []);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Load one window: the first when `anchor` is null, otherwise the rows past the anchor. */
|
|
68
|
+
async loadWindow(anchor: CursorAnchor | null, windowSize: number): Promise<AnchoredWindow<T>> {
|
|
69
|
+
if (windowSize < 1) {
|
|
70
|
+
throw new Error(`Cursor window size must be at least 1 (got ${windowSize})`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const qb = this.buildWindowQuery(anchor, windowSize);
|
|
74
|
+
const db = this.options?.db ?? getDb();
|
|
75
|
+
const rows = await db.query(this.table, qb, this.options?.queryOptions);
|
|
76
|
+
return { rows, nextAnchor: this.deriveNextAnchor(rows, windowSize) };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
static encodeAnchor(anchor: CursorAnchor): string {
|
|
80
|
+
return Serializer.serialize(anchor);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
static decodeAnchor(encoded: string): CursorAnchor {
|
|
84
|
+
return Serializer.deserialize(encoded);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private buildWindowQuery(anchor: CursorAnchor | null, windowSize: number): QueryBuilder<T> {
|
|
88
|
+
const qb = this.createQuery();
|
|
89
|
+
if (anchor) {
|
|
90
|
+
qb.or(this.anchorConditions(anchor));
|
|
91
|
+
}
|
|
92
|
+
return qb.sort(this.effectiveSort).paginate({ start: 0, end: windowSize });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Lexicographic continuation past the anchor row: one disjunct per sort criterion — equal on
|
|
97
|
+
* every earlier criterion, strictly past on this one (`<` descending, `>` ascending). The
|
|
98
|
+
* final `id` disjunct is what walks through ties on the consumer's axes.
|
|
99
|
+
*/
|
|
100
|
+
private anchorConditions(anchor: CursorAnchor): Array<Condition<T> | LogicalGroup<T>> {
|
|
101
|
+
return this.effectiveSort.map((criterion, index) => {
|
|
102
|
+
const past: Condition<T> = {
|
|
103
|
+
field: criterion.field,
|
|
104
|
+
operator: criterion.desc ? '<' : '>',
|
|
105
|
+
value: anchor.values[index] as T[keyof T],
|
|
106
|
+
};
|
|
107
|
+
if (index === 0) {
|
|
108
|
+
return past;
|
|
109
|
+
}
|
|
110
|
+
const equals: Condition<T>[] = this.effectiveSort.slice(0, index).map((earlier, i) => ({
|
|
111
|
+
field: earlier.field,
|
|
112
|
+
operator: '=',
|
|
113
|
+
value: anchor.values[i] as T[keyof T],
|
|
114
|
+
}));
|
|
115
|
+
return { operator: 'AND', children: [...equals, past] };
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
private deriveNextAnchor(rows: T[], windowSize: number): CursorAnchor | null {
|
|
120
|
+
// A short window means the data set is exhausted; a full one implies more may exist —
|
|
121
|
+
// the next (short or empty) window settles it.
|
|
122
|
+
if (rows.length < windowSize) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const tail = rows[rows.length - 1] as unknown as { [field: string]: unknown };
|
|
127
|
+
const values = this.effectiveSort.map((criterion) => tail[criterion.field as string]);
|
|
128
|
+
// A null sort value can't anchor a lexicographic continuation; under a descending sort
|
|
129
|
+
// null rows collect at the tail, so the data set's sortable rows are exhausted here.
|
|
130
|
+
if (values.some((value) => value == null)) {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return { values };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
private normalizeSort(sort: SortCriteria<T>[]): SortCriteria<T>[] {
|
|
138
|
+
for (const criterion of sort) {
|
|
139
|
+
if (criterion.byValues && criterion.byValues.length > 0) {
|
|
140
|
+
throw new Error(
|
|
141
|
+
`Cursor windows cannot page a byValues sort (field '${String(criterion.field)}'): a CASE ordering has no comparable cursor axis`
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const idIndex = sort.findIndex((criterion) => criterion.field === 'id');
|
|
147
|
+
if (idIndex >= 0) {
|
|
148
|
+
return sort.slice(0, idIndex + 1);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return [...sort, { field: 'id' as keyof T, desc: false }];
|
|
152
|
+
}
|
|
153
|
+
}
|
package/src/Db.ts
CHANGED
|
@@ -517,6 +517,16 @@ export class Db<R extends Record = Record> implements DbService<R> {
|
|
|
517
517
|
}
|
|
518
518
|
}
|
|
519
519
|
|
|
520
|
+
/**
|
|
521
|
+
* Run a query. Column queries (scope guards etc.) are always applied.
|
|
522
|
+
*
|
|
523
|
+
* PAGING: this method is also the cursor-window surface — `CursorWindowPager` composes each
|
|
524
|
+
* window as a fresh QueryBuilder (cursor conditions + sort + `paginate(0, windowSize)`)
|
|
525
|
+
* through here, so every window rides the caller's driver, ambient transaction, and column
|
|
526
|
+
* queries. Iterating consumers go through it (`RecordIterator` server-side,
|
|
527
|
+
* `QueryCursorLoader` in the UI) instead of positional `paginate(start, end)` offsets,
|
|
528
|
+
* which drift under concurrent writes (rows slide across window frames).
|
|
529
|
+
*/
|
|
520
530
|
async query<T extends R>(table: Table<T>, query: Query<T>, options?: QueryOptions<T>): Promise<T[]> {
|
|
521
531
|
const qb = new QueryBuilderFactory().getQueryBuilder(table, query);
|
|
522
532
|
|
package/src/MigrationRunner.ts
CHANGED
|
@@ -4,12 +4,33 @@ import { Table } from './Table';
|
|
|
4
4
|
import { SourceRecordRepo } from './source/SourceRecordRepo';
|
|
5
5
|
import { MigrationRunnerService, getMigrationRunnerService } from './services/MigrationRunnerService';
|
|
6
6
|
import { Migration, MigrationTable } from './tables/MigrationTable';
|
|
7
|
+
import { QueryBuilderFactory } from './QueryBuilderFactory';
|
|
7
8
|
import { Service } from '@proteinjs/service';
|
|
8
9
|
import { Logger } from '@proteinjs/logger';
|
|
9
10
|
|
|
10
11
|
export const getMigrationRunner = () =>
|
|
11
12
|
typeof self === 'undefined' ? new MigrationRunner() : (getMigrationRunnerService() as MigrationRunner);
|
|
12
13
|
|
|
14
|
+
/**
|
|
15
|
+
* What one deploy-gated series run did (plans/POST_RELEASE_QUEUE.md 27f). Ids appear in ledger
|
|
16
|
+
* order (oldest-first). The caller's gate is `failed`: present → the series stopped there, the
|
|
17
|
+
* deploy Job must exit non-zero, the rollout must not advance.
|
|
18
|
+
*/
|
|
19
|
+
export interface MigrationSeriesSummary {
|
|
20
|
+
/** Ran to success in this series. */
|
|
21
|
+
applied: string[];
|
|
22
|
+
/** Excluded by the explicit `manual` flag (they keep the Migrations-page flow). */
|
|
23
|
+
skippedManual: string[];
|
|
24
|
+
/** Already in 'success' status — skipped (ensureMigrationRun's idempotence). */
|
|
25
|
+
alreadyApplied: string[];
|
|
26
|
+
/** Ledger rows with no source record (a loader deleted after its run; the table keeps history). */
|
|
27
|
+
unresolved: string[];
|
|
28
|
+
/** The failure that stopped the series, if any. */
|
|
29
|
+
failed?: { id: string; description: string; failureMessage?: string };
|
|
30
|
+
/** Ordered after the failure — never started (later migrations may build on earlier ones). */
|
|
31
|
+
notAttempted: string[];
|
|
32
|
+
}
|
|
33
|
+
|
|
13
34
|
export class MigrationRunner implements MigrationRunnerService {
|
|
14
35
|
private logger = new Logger({ name: this.constructor.name });
|
|
15
36
|
public serviceMetadata: Service['serviceMetadata'] = {
|
|
@@ -44,18 +65,109 @@ export class MigrationRunner implements MigrationRunnerService {
|
|
|
44
65
|
*
|
|
45
66
|
* 'ensure' = idempotent: a row already in 'success' status is logged and skipped. A prior
|
|
46
67
|
* 'failure' row is retried by design — a fixed migration should run on the next boot.
|
|
68
|
+
*
|
|
69
|
+
* Returns the migration with its final run state (the skipped row, or the run's outcome) —
|
|
70
|
+
* the series runner ({@link runPendingMigrations}) gates on it.
|
|
47
71
|
*/
|
|
48
|
-
async ensureMigrationRun(id: string): Promise<
|
|
72
|
+
async ensureMigrationRun(id: string): Promise<Migration> {
|
|
49
73
|
const migrationTable: Table<Migration> = new MigrationTable();
|
|
50
74
|
const db = getDbAsSystem();
|
|
51
75
|
const migrationRow = await db.get(migrationTable, { id });
|
|
52
76
|
if (migrationRow?.status === 'success') {
|
|
53
77
|
this.logger.info({ message: `Migration (${id}) already applied, skipping` });
|
|
54
|
-
return;
|
|
78
|
+
return migrationRow;
|
|
55
79
|
}
|
|
56
80
|
|
|
57
81
|
const migration = this.resolveMigration(migrationTable, id);
|
|
58
82
|
await this.runAndRecord(migrationTable, migration, () => db);
|
|
83
|
+
return migration;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Deploy-path API (plans/POST_RELEASE_QUEUE.md 27f): the deploy pipeline's migration Job calls
|
|
88
|
+
* this AFTER `new Db().init()` (schema sync + source-record sync — every source-declared
|
|
89
|
+
* migration has a ledger row by then) and BEFORE the rollout advances. Pod boot never calls it:
|
|
90
|
+
* boot stays migration-free (the startupProbe invariant — DbInitStartupTask is schema sync and
|
|
91
|
+
* source records only).
|
|
92
|
+
*
|
|
93
|
+
* Discovery + order: the migration LEDGER (the migration table) is the authority. Rows run in
|
|
94
|
+
* SERIES, oldest-first by the row's `created` (id tiebreak) — a migration that shipped in an
|
|
95
|
+
* earlier release always runs before a later one. Never parallel.
|
|
96
|
+
*
|
|
97
|
+
* Policy:
|
|
98
|
+
* - `manual: true` (the source record's declaration) is EXCLUDED — the explicit non-automatable
|
|
99
|
+
* class keeps the Migrations-page flow ({@link Migration.manual}).
|
|
100
|
+
* - rows already in 'success' are skipped; any other status is pending — including 'running'
|
|
101
|
+
* (a crashed earlier Job): {@link ensureMigrationRun} re-runs it.
|
|
102
|
+
* - rows with no source record are history (a loader deleted after its run) — skipped,
|
|
103
|
+
* reported as `unresolved`.
|
|
104
|
+
* - the FIRST failure STOPS the series (later migrations may build on earlier ones). The
|
|
105
|
+
* caller exits non-zero, the Job fails, the rollout does not advance.
|
|
106
|
+
*
|
|
107
|
+
* EXPAND-CONTRACT INVARIANT (documented at this seam on purpose): every automated migration —
|
|
108
|
+
* and the schema sync that fronts it — must be backward-compatible with the STILL-RUNNING old
|
|
109
|
+
* release, because the old pods keep serving while this runs and keep serving indefinitely if
|
|
110
|
+
* it fails. "Roll back" = do not advance the code; DDL and applied migrations are never
|
|
111
|
+
* un-applied (Spanner DDL is not transactional-reversible). Contractions (drops, rewrites that
|
|
112
|
+
* break the old reader) belong in the `manual` class, run only after every consumer of the old
|
|
113
|
+
* shape is gone.
|
|
114
|
+
*/
|
|
115
|
+
async runPendingMigrations(): Promise<MigrationSeriesSummary> {
|
|
116
|
+
const migrationTable: Table<Migration> = new MigrationTable();
|
|
117
|
+
const db = getDbAsSystem();
|
|
118
|
+
const qb = new QueryBuilderFactory().createQueryBuilder(migrationTable).sort([
|
|
119
|
+
{ field: 'created', desc: false },
|
|
120
|
+
{ field: 'id', desc: false },
|
|
121
|
+
]);
|
|
122
|
+
const ledger = await db.query(migrationTable, qb);
|
|
123
|
+
|
|
124
|
+
const summary: MigrationSeriesSummary = {
|
|
125
|
+
applied: [],
|
|
126
|
+
skippedManual: [],
|
|
127
|
+
alreadyApplied: [],
|
|
128
|
+
unresolved: [],
|
|
129
|
+
notAttempted: [],
|
|
130
|
+
};
|
|
131
|
+
const sourceRecordRepo = new SourceRecordRepo();
|
|
132
|
+
const pending: Migration[] = [];
|
|
133
|
+
for (const row of ledger) {
|
|
134
|
+
const source = sourceRecordRepo.getSourceRecord<Migration>(migrationTable.name, row.id);
|
|
135
|
+
if (!source) {
|
|
136
|
+
summary.unresolved.push(row.id);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (source.manual) {
|
|
140
|
+
summary.skippedManual.push(row.id);
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (row.status === 'success') {
|
|
144
|
+
summary.alreadyApplied.push(row.id);
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
pending.push(row);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
this.logger.info({
|
|
151
|
+
message: `Running ${pending.length} pending migration${pending.length === 1 ? '' : 's'} in series, oldest-first`,
|
|
152
|
+
obj: { pending: pending.map((row) => row.id), skippedManual: summary.skippedManual },
|
|
153
|
+
});
|
|
154
|
+
for (let i = 0; i < pending.length; i++) {
|
|
155
|
+
const outcome = await this.ensureMigrationRun(pending[i].id);
|
|
156
|
+
if (outcome.status === 'success') {
|
|
157
|
+
summary.applied.push(outcome.id);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
summary.failed = {
|
|
161
|
+
id: outcome.id,
|
|
162
|
+
description: outcome.description,
|
|
163
|
+
failureMessage: outcome.failureMessage,
|
|
164
|
+
};
|
|
165
|
+
summary.notAttempted = pending.slice(i + 1).map((row) => row.id);
|
|
166
|
+
break;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
this.logger.info({ message: `Migration series finished`, obj: summary as any });
|
|
170
|
+
return summary;
|
|
59
171
|
}
|
|
60
172
|
|
|
61
173
|
// The db is taken as a provider, resolved inside this async body: on the service path,
|
package/src/RecordIterator.ts
CHANGED
|
@@ -1,44 +1,54 @@
|
|
|
1
|
-
import { Db
|
|
1
|
+
import { Db } from './Db';
|
|
2
2
|
import { Table } from './Table';
|
|
3
3
|
import { Record } from './Record';
|
|
4
4
|
import { Query } from './services/DbService';
|
|
5
5
|
import { QueryBuilderFactory } from './QueryBuilderFactory';
|
|
6
6
|
import { QueryBuilder } from '@proteinjs/db-query';
|
|
7
|
+
import { CursorAnchor, CursorWindowPager } from './CursorWindowPager';
|
|
7
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Iterate a table query in cursor-anchored windows (`CursorWindowPager`), `windowSize` rows
|
|
11
|
+
* per query. Cursor windows frame on the last row served instead of a positional offset, so
|
|
12
|
+
* rows inserted or deleted while iterating can never skip or duplicate rows — every row that
|
|
13
|
+
* exists throughout the iteration is yielded exactly once.
|
|
14
|
+
*
|
|
15
|
+
* Ordering: the sort criteria on the consumer's query builder become the cursor axes, with a
|
|
16
|
+
* unique `id` tiebreak appended (ties can't skip across window boundaries); with no sort the
|
|
17
|
+
* iteration runs in plain `id` order. Sort axes should be non-null columns — a window whose
|
|
18
|
+
* tail carries a NULL sort value ends the iteration (a null can't anchor a continuation).
|
|
19
|
+
*
|
|
20
|
+
* The caller's query builder is never mutated: every window builds on a fresh copy.
|
|
21
|
+
*/
|
|
8
22
|
export class RecordIterator<T extends Record> implements AsyncIterable<T> {
|
|
9
|
-
private
|
|
10
|
-
private table: Table<T>;
|
|
11
|
-
private query: QueryBuilder<T>;
|
|
12
|
-
private pageSize: number;
|
|
13
|
-
private currentPage: number;
|
|
23
|
+
private pager: CursorWindowPager<T>;
|
|
14
24
|
|
|
15
|
-
constructor(
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
25
|
+
constructor(
|
|
26
|
+
table: Table<T>,
|
|
27
|
+
query: Query<T>,
|
|
28
|
+
private windowSize: number = 10,
|
|
29
|
+
db?: Db
|
|
30
|
+
) {
|
|
31
|
+
// A fresh copy of the consumer's query; its sort criteria lift off the base (the pager
|
|
32
|
+
// owns ordering — cursor axes and ORDER BY have to agree) and the conditions-only base
|
|
33
|
+
// seeds every window's fresh build.
|
|
34
|
+
const base = new QueryBuilderFactory().createQueryBuilder(table, query);
|
|
35
|
+
const sort = base.removeSortCriteria();
|
|
36
|
+
this.pager = new CursorWindowPager<T>(table, () => QueryBuilder.fromQueryBuilder(base, table.name), sort, { db });
|
|
21
37
|
}
|
|
22
38
|
|
|
23
39
|
async *[Symbol.asyncIterator](): AsyncIterator<T> {
|
|
40
|
+
let anchor: CursorAnchor | null = null;
|
|
24
41
|
while (true) {
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if (results.length === 0) {
|
|
30
|
-
return;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
for (const item of results) {
|
|
34
|
-
yield item;
|
|
42
|
+
const window = await this.pager.loadWindow(anchor, this.windowSize);
|
|
43
|
+
for (const row of window.rows) {
|
|
44
|
+
yield row;
|
|
35
45
|
}
|
|
36
46
|
|
|
37
|
-
if (
|
|
47
|
+
if (!window.nextAnchor) {
|
|
38
48
|
return;
|
|
39
49
|
}
|
|
40
50
|
|
|
41
|
-
|
|
51
|
+
anchor = window.nextAnchor;
|
|
42
52
|
}
|
|
43
53
|
}
|
|
44
54
|
}
|
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import { Moment } from '../opt/moment';
|
|
2
|
-
import { DateTimeColumn, ObjectColumn, StringColumn } from '../Columns';
|
|
2
|
+
import { BooleanColumn, DateTimeColumn, ObjectColumn, StringColumn } from '../Columns';
|
|
3
3
|
import { Table } from '../Table';
|
|
4
4
|
import { SourceRecord, withSourceRecordColumns } from '../source/SourceRecord';
|
|
5
5
|
|
|
6
6
|
export interface Migration extends SourceRecord {
|
|
7
7
|
description: string;
|
|
8
|
+
/**
|
|
9
|
+
* The explicit non-automatable class (plans/POST_RELEASE_QUEUE.md 27f): destructive
|
|
10
|
+
* contractions (column/table drops), long backfills over big tables, data-dependent one-offs.
|
|
11
|
+
* `manual: true` EXCLUDES the migration from the deploy-gated auto-run
|
|
12
|
+
* ({@link MigrationRunner.runPendingMigrations}); it keeps the Migrations-page flow
|
|
13
|
+
* (`runMigration`). Absence of the flag = automated — the invariant lives here, in the
|
|
14
|
+
* schema, not in deploy-pipeline prose.
|
|
15
|
+
*/
|
|
16
|
+
manual?: boolean;
|
|
8
17
|
status?: 'proposed' | 'running' | 'success' | 'failure';
|
|
9
18
|
failureMessage?: string;
|
|
10
19
|
failureStack?: string;
|
|
@@ -29,6 +38,7 @@ export class MigrationTable extends Table<Migration> {
|
|
|
29
38
|
};
|
|
30
39
|
public columns = withSourceRecordColumns<Migration>({
|
|
31
40
|
description: new StringColumn('description', {}, 4000),
|
|
41
|
+
manual: new BooleanColumn('manual'),
|
|
32
42
|
status: new StringColumn('status', { defaultValue: async () => 'proposed' }),
|
|
33
43
|
failureMessage: new StringColumn('failure_message', {}, 4000),
|
|
34
44
|
failureStack: new StringColumn('failure_stack', {}, 'MAX'),
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CursorWindowPager (src/CursorWindowPager.ts) at the pure level: effective-sort derivation
|
|
3
|
+
* (id tiebreak appended, explicit id ends the list, byValues rejected), lexicographic anchor
|
|
4
|
+
* conditions, next-anchor derivation (short-window and null-tail exhaustion), anchor
|
|
5
|
+
* encode/decode fidelity, and fresh-query-per-window construction. The db round trip is a
|
|
6
|
+
* constructor-injected stub — no module mocking, the pager's `options.db` seam IS the
|
|
7
|
+
* injection point.
|
|
8
|
+
*/
|
|
9
|
+
import moment from 'moment';
|
|
10
|
+
import { QueryBuilder } from '@proteinjs/db-query';
|
|
11
|
+
import { AnchoredWindow, CursorAnchor, CursorWindowPager, Db, SortCriteria, Table } from '@proteinjs/db';
|
|
12
|
+
// Load the reflection source graph so the house Serializer's custom serializers (moments)
|
|
13
|
+
// are registered — anchor encode/decode fidelity rides them.
|
|
14
|
+
import '../generated/index';
|
|
15
|
+
|
|
16
|
+
type Row = {
|
|
17
|
+
id: string;
|
|
18
|
+
name: string;
|
|
19
|
+
seq?: number | null;
|
|
20
|
+
stampedAt?: moment.Moment | null;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const table = { name: 'pager_test' } as Table<Row & { created: moment.Moment; updated: moment.Moment }>;
|
|
24
|
+
type PagerRow = Row & { created: moment.Moment; updated: moment.Moment };
|
|
25
|
+
|
|
26
|
+
const staged: { rows: Row[] } = { rows: [] };
|
|
27
|
+
const executed: QueryBuilder<PagerRow>[] = [];
|
|
28
|
+
const stubDb = {
|
|
29
|
+
query: async (_table: unknown, qb: QueryBuilder<PagerRow>) => {
|
|
30
|
+
executed.push(qb);
|
|
31
|
+
return staged.rows;
|
|
32
|
+
},
|
|
33
|
+
} as unknown as Db;
|
|
34
|
+
|
|
35
|
+
const createQuery = () => new QueryBuilder<PagerRow>(table.name);
|
|
36
|
+
|
|
37
|
+
const pager = (sort?: SortCriteria<PagerRow>[]) =>
|
|
38
|
+
new CursorWindowPager<PagerRow>(table, createQuery, sort, { db: stubDb });
|
|
39
|
+
|
|
40
|
+
/** House pattern: private internals reachable via a typed cast on the instance. */
|
|
41
|
+
type PagerInternals = {
|
|
42
|
+
effectiveSort: SortCriteria<PagerRow>[];
|
|
43
|
+
buildWindowQuery(anchor: CursorAnchor | null, windowSize: number): QueryBuilder<PagerRow>;
|
|
44
|
+
deriveNextAnchor(rows: Row[], windowSize: number): CursorAnchor | null;
|
|
45
|
+
};
|
|
46
|
+
const internals = (instance: CursorWindowPager<PagerRow>) => instance as unknown as PagerInternals;
|
|
47
|
+
|
|
48
|
+
const graphNodes = (qb: QueryBuilder<PagerRow>): any[] => qb.graph.nodes().map((id: string) => qb.graph.node(id));
|
|
49
|
+
const conditionNodes = (qb: QueryBuilder<PagerRow>) => graphNodes(qb).filter((node) => node?.type === 'CONDITION');
|
|
50
|
+
const logicalNodes = (qb: QueryBuilder<PagerRow>) => graphNodes(qb).filter((node) => node?.type === 'LOGICAL');
|
|
51
|
+
const paginationNode = (qb: QueryBuilder<PagerRow>) => graphNodes(qb).find((node) => node?.type === 'PAGINATION');
|
|
52
|
+
const sortNodes = (qb: QueryBuilder<PagerRow>) => graphNodes(qb).filter((node) => node?.type === 'SORT');
|
|
53
|
+
|
|
54
|
+
const row = (id: string, seq: number | null, at?: string | null): Row => ({
|
|
55
|
+
id,
|
|
56
|
+
name: id,
|
|
57
|
+
seq,
|
|
58
|
+
stampedAt: at ? moment(at) : at === null ? null : undefined,
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
beforeEach(() => {
|
|
62
|
+
staged.rows = [];
|
|
63
|
+
executed.length = 0;
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
describe('effective sort', () => {
|
|
67
|
+
it('appends the id tiebreak to the consumer sort', () => {
|
|
68
|
+
const instance = pager([{ field: 'seq', desc: true }]);
|
|
69
|
+
expect(internals(instance).effectiveSort).toEqual([
|
|
70
|
+
{ field: 'seq', desc: true },
|
|
71
|
+
{ field: 'id', desc: false },
|
|
72
|
+
]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it('no sort = pure id order', () => {
|
|
76
|
+
expect(internals(pager()).effectiveSort).toEqual([{ field: 'id', desc: false }]);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('an explicit id criterion ends the list (axes after it can never influence order)', () => {
|
|
80
|
+
const instance = pager([
|
|
81
|
+
{ field: 'id', desc: true },
|
|
82
|
+
{ field: 'seq', desc: false },
|
|
83
|
+
]);
|
|
84
|
+
expect(internals(instance).effectiveSort).toEqual([{ field: 'id', desc: true }]);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('rejects byValues sorts — a CASE ordering has no comparable cursor axis', () => {
|
|
88
|
+
expect(() => pager([{ field: 'name', byValues: ['a', 'b'] }])).toThrow(/byValues/);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
describe('buildWindowQuery', () => {
|
|
93
|
+
it('the first window carries no cursor conditions — just the effective sort and the window pagination', () => {
|
|
94
|
+
const qb = internals(pager([{ field: 'seq', desc: false }])).buildWindowQuery(null, 30);
|
|
95
|
+
expect(conditionNodes(qb)).toHaveLength(0);
|
|
96
|
+
expect(paginationNode(qb)).toMatchObject({ start: 0, end: 30 });
|
|
97
|
+
expect(sortNodes(qb).map((node) => node.criteria)).toEqual([
|
|
98
|
+
{ field: 'seq', desc: false },
|
|
99
|
+
{ field: 'id', desc: false },
|
|
100
|
+
]);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it('an anchored window builds the lexicographic continuation: past-on-axis OR (equal-on-axis AND past-on-id)', () => {
|
|
104
|
+
const qb = internals(pager([{ field: 'seq', desc: true }])).buildWindowQuery({ values: [40, 'row-4'] }, 10);
|
|
105
|
+
// Disjunct 1: seq < 40. Disjunct 2: seq = 40 AND id > 'row-4'.
|
|
106
|
+
expect(conditionNodes(qb)).toEqual(
|
|
107
|
+
expect.arrayContaining([
|
|
108
|
+
expect.objectContaining({ field: 'seq', operator: '<', value: 40 }),
|
|
109
|
+
expect.objectContaining({ field: 'seq', operator: '=', value: 40 }),
|
|
110
|
+
expect.objectContaining({ field: 'id', operator: '>', value: 'row-4' }),
|
|
111
|
+
])
|
|
112
|
+
);
|
|
113
|
+
expect(conditionNodes(qb)).toHaveLength(3);
|
|
114
|
+
const operators = logicalNodes(qb).map((node) => node.operator);
|
|
115
|
+
expect(operators).toContain('OR');
|
|
116
|
+
expect(operators).toContain('AND');
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('an ascending axis flips the past operator to `>`', () => {
|
|
120
|
+
const qb = internals(pager([{ field: 'seq', desc: false }])).buildWindowQuery({ values: [40, 'row-4'] }, 10);
|
|
121
|
+
expect(conditionNodes(qb)).toEqual(
|
|
122
|
+
expect.arrayContaining([expect.objectContaining({ field: 'seq', operator: '>', value: 40 })])
|
|
123
|
+
);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('every window builds on a FRESH query — cursor conditions never accumulate', () => {
|
|
127
|
+
const instance = pager([{ field: 'seq', desc: true }]);
|
|
128
|
+
internals(instance).buildWindowQuery({ values: [40, 'row-4'] }, 10);
|
|
129
|
+
const second = internals(instance).buildWindowQuery({ values: [20, 'row-2'] }, 10);
|
|
130
|
+
expect(conditionNodes(second)).toHaveLength(3);
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
describe('deriveNextAnchor', () => {
|
|
135
|
+
const instance = pager([{ field: 'seq', desc: false }]);
|
|
136
|
+
|
|
137
|
+
it('a short window is exhausted', () => {
|
|
138
|
+
expect(internals(instance).deriveNextAnchor([row('a', 10)], 3)).toBeNull();
|
|
139
|
+
expect(internals(instance).deriveNextAnchor([], 3)).toBeNull();
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it('a full window anchors at its tail row, one value per effective criterion', () => {
|
|
143
|
+
const anchor = internals(instance).deriveNextAnchor([row('a', 10), row('b', 20), row('c', 30)], 3);
|
|
144
|
+
expect(anchor).toEqual({ values: [30, 'c'] });
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it('a full window with a NULL sort-value tail is exhausted (null rows collect at the tail; no anchor)', () => {
|
|
148
|
+
expect(internals(instance).deriveNextAnchor([row('a', 10), row('b', 20), row('c', null)], 3)).toBeNull();
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
describe('anchor encode/decode', () => {
|
|
153
|
+
it('round-trips values with type fidelity (moments survive)', () => {
|
|
154
|
+
const at = moment('2026-08-10T12:00:00.000Z');
|
|
155
|
+
const encoded = CursorWindowPager.encodeAnchor({ values: [at, 'row-1'] });
|
|
156
|
+
expect(typeof encoded).toBe('string');
|
|
157
|
+
const decoded = CursorWindowPager.decodeAnchor(encoded);
|
|
158
|
+
expect(moment.isMoment(decoded.values[0])).toBe(true);
|
|
159
|
+
expect((decoded.values[0] as moment.Moment).toISOString()).toBe(at.toISOString());
|
|
160
|
+
expect(decoded.values[1]).toBe('row-1');
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
describe('loadWindow', () => {
|
|
165
|
+
it('returns the queried rows with the derived next anchor, querying through the injected db', async () => {
|
|
166
|
+
const instance = pager([{ field: 'seq', desc: false }]);
|
|
167
|
+
staged.rows = [row('a', 10), row('b', 20), row('c', 30)];
|
|
168
|
+
const window: AnchoredWindow<PagerRow> = await instance.loadWindow(null, 3);
|
|
169
|
+
expect(window.rows).toBe(staged.rows);
|
|
170
|
+
expect(window.nextAnchor).toEqual({ values: [30, 'c'] });
|
|
171
|
+
expect(executed).toHaveLength(1);
|
|
172
|
+
expect(paginationNode(executed[0])).toMatchObject({ start: 0, end: 3 });
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('returns an exhausted window (null anchor) when the query comes back short', async () => {
|
|
176
|
+
const instance = pager([{ field: 'seq', desc: false }]);
|
|
177
|
+
staged.rows = [row('a', 10)];
|
|
178
|
+
const window = await instance.loadWindow({ values: [5, 'z'] }, 3);
|
|
179
|
+
expect(window.nextAnchor).toBeNull();
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it('rejects a nonsensical window size', async () => {
|
|
183
|
+
await expect(pager().loadWindow(null, 0)).rejects.toThrow(/window size/i);
|
|
184
|
+
});
|
|
185
|
+
});
|
package/test/index.ts
CHANGED
|
@@ -5,6 +5,7 @@ export * from './reusable/DynamicReferenceColumnTests';
|
|
|
5
5
|
export * from './reusable/TransactionTests';
|
|
6
6
|
export * from './reusable/CascadeDeleteTests';
|
|
7
7
|
export * from './reusable/PreloadReferencesTests';
|
|
8
|
+
export * from './reusable/RecordIteratorTests';
|
|
8
9
|
|
|
9
10
|
export * from './util/testUser';
|
|
10
11
|
|
|
@@ -12,5 +13,6 @@ export * from './util/tables/cascadeDeleteTestTables';
|
|
|
12
13
|
export * from './util/tables/columnTypesTestTables';
|
|
13
14
|
export * from './util/tables/crudTestTables';
|
|
14
15
|
export * from './util/tables/dynamicReferenceColumnTestTables';
|
|
16
|
+
export * from './util/tables/recordIteratorTestTables';
|
|
15
17
|
export * from './util/tables/tableManagerTestTables';
|
|
16
18
|
export * from './util/tables/transactionTestTables';
|