@powersync/service-module-mssql 0.8.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +113 -0
- package/dist/api/MSSQLRouteAPIAdapter.d.ts +2 -2
- package/dist/api/MSSQLRouteAPIAdapter.js +33 -39
- package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/MSSQLSourceTable.d.ts +12 -2
- package/dist/common/MSSQLSourceTable.js +30 -4
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/module/MSSQLModule.js +2 -1
- package/dist/module/MSSQLModule.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +13 -36
- package/dist/replication/CDCPoller.js +173 -129
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +11 -14
- package/dist/replication/CDCStream.js +109 -103
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/CaptureReconciler.d.ts +47 -0
- package/dist/replication/CaptureReconciler.js +155 -0
- package/dist/replication/CaptureReconciler.js.map +1 -0
- package/dist/replication/MSSQLConnectionManager.d.ts +1 -0
- package/dist/replication/MSSQLConnectionManager.js +19 -14
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/replication/MSSQLTableReconciliationContext.d.ts +21 -0
- package/dist/replication/MSSQLTableReconciliationContext.js +7 -0
- package/dist/replication/MSSQLTableReconciliationContext.js.map +1 -0
- package/dist/replication/SchemaChange.d.ts +67 -0
- package/dist/replication/SchemaChange.js +31 -0
- package/dist/replication/SchemaChange.js.map +1 -0
- package/dist/types/types.d.ts +13 -0
- package/dist/types/types.js +28 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/mssql.js +1 -1
- package/dist/utils/schema.d.ts +14 -1
- package/dist/utils/schema.js +43 -45
- package/dist/utils/schema.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MSSQLRouteAPIAdapter.ts +36 -46
- package/src/common/MSSQLSourceTable.ts +32 -4
- package/src/module/MSSQLModule.ts +2 -1
- package/src/replication/CDCPoller.ts +206 -162
- package/src/replication/CDCStream.ts +142 -151
- package/src/replication/CaptureReconciler.ts +186 -0
- package/src/replication/MSSQLConnectionManager.ts +23 -14
- package/src/replication/MSSQLTableReconciliationContext.ts +29 -0
- package/src/replication/SchemaChange.ts +79 -0
- package/src/types/types.ts +47 -2
- package/src/utils/mssql.ts +1 -1
- package/src/utils/schema.ts +60 -54
- package/test/src/CDCPoller.test.ts +382 -0
- package/test/src/CDCStream.test.ts +6 -2
- package/test/src/CDCStreamTestContext.ts +9 -2
- package/test/src/CaptureReconciler.test.ts +171 -0
- package/test/src/MSSQLSourceTable.test.ts +30 -0
- package/test/src/config.test.ts +71 -0
- package/test/src/schema-changes.test.ts +513 -249
- package/test/src/schema.test.ts +17 -0
- package/test/src/util.ts +70 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
api,
|
|
3
|
-
ParseSyncConfigOptions,
|
|
4
|
-
PatternResult,
|
|
5
|
-
ReplicationHeadCallback,
|
|
6
|
-
ReplicationLagOptions
|
|
7
|
-
} from '@powersync/service-core';
|
|
1
|
+
import { api, ParseSyncConfigOptions, PatternResult, ReplicationLagOptions } from '@powersync/service-core';
|
|
8
2
|
import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
|
|
9
3
|
import * as service_types from '@powersync/service-types';
|
|
10
4
|
import sql from 'mssql';
|
|
@@ -18,7 +12,7 @@ import {
|
|
|
18
12
|
getLatestLSN,
|
|
19
13
|
POWERSYNC_CHECKPOINTS_TABLE
|
|
20
14
|
} from '../utils/mssql.js';
|
|
21
|
-
import { getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
|
|
15
|
+
import { getTablesFromPattern, ResolvedTable, unsupportedTablePatternMessage } from '../utils/schema.js';
|
|
22
16
|
|
|
23
17
|
export class MSSQLRouteAPIAdapter implements api.RouteAPI {
|
|
24
18
|
protected connectionManager: MSSQLConnectionManager;
|
|
@@ -27,14 +21,16 @@ export class MSSQLRouteAPIAdapter implements api.RouteAPI {
|
|
|
27
21
|
this.connectionManager = new MSSQLConnectionManager(config, {});
|
|
28
22
|
}
|
|
29
23
|
|
|
30
|
-
async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
|
|
31
|
-
const currentLSN = await getLatestLSN(this.connectionManager);
|
|
32
|
-
const
|
|
24
|
+
async createReplicationHead<T>(callback: api.ReplicationHeadCallback<T>): Promise<T> {
|
|
25
|
+
const currentLSN = (await getLatestLSN(this.connectionManager)).toString();
|
|
26
|
+
const { response, shouldAdvance } = await callback(currentLSN);
|
|
33
27
|
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
if (shouldAdvance) {
|
|
29
|
+
// Updates the powersync checkpoints table on the source database, ensuring that an update with a newer LSN will be captured by the CDC.
|
|
30
|
+
await createCheckpoint(this.connectionManager);
|
|
31
|
+
}
|
|
36
32
|
|
|
37
|
-
return
|
|
33
|
+
return response;
|
|
38
34
|
}
|
|
39
35
|
|
|
40
36
|
async executeQuery(query: string, params: any[]): Promise<service_types.internal_routes.ExecuteSqlResponse> {
|
|
@@ -186,41 +182,35 @@ export class MSSQLRouteAPIAdapter implements api.RouteAPI {
|
|
|
186
182
|
};
|
|
187
183
|
result.push(patternResult);
|
|
188
184
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
// This should technically never happen, but we'll handle it anyway.
|
|
204
|
-
const resolvedTable: ResolvedTable = {
|
|
205
|
-
objectId: 0,
|
|
206
|
-
schema: schema,
|
|
207
|
-
name: tablePattern.name
|
|
208
|
-
};
|
|
209
|
-
patternResult.table = await getDebugTableInfo({
|
|
210
|
-
connectionManager: this.connectionManager,
|
|
211
|
-
tablePattern,
|
|
212
|
-
table: resolvedTable,
|
|
213
|
-
syncRules: sqlSyncRules
|
|
214
|
-
});
|
|
185
|
+
// Unsupported patterns are reported per pattern here, rather than failing the whole request.
|
|
186
|
+
const unsupported = unsupportedTablePatternMessage(tablePattern);
|
|
187
|
+
if (unsupported != null) {
|
|
188
|
+
const unsupportedTable: service_types.TableInfo = {
|
|
189
|
+
schema,
|
|
190
|
+
name: tablePattern.tablePattern,
|
|
191
|
+
pattern: tablePattern.tablePattern,
|
|
192
|
+
replication_id: [],
|
|
193
|
+
data_queries: false,
|
|
194
|
+
parameter_queries: false,
|
|
195
|
+
errors: [{ level: 'fatal', message: unsupported }]
|
|
196
|
+
};
|
|
197
|
+
if (patternResult.wildcard) {
|
|
198
|
+
patternResult.tables = [unsupportedTable];
|
|
215
199
|
} else {
|
|
216
|
-
patternResult.table =
|
|
217
|
-
connectionManager: this.connectionManager,
|
|
218
|
-
tablePattern,
|
|
219
|
-
table: tables[0],
|
|
220
|
-
syncRules: sqlSyncRules
|
|
221
|
-
});
|
|
200
|
+
patternResult.table = unsupportedTable;
|
|
222
201
|
}
|
|
202
|
+
continue;
|
|
223
203
|
}
|
|
204
|
+
|
|
205
|
+
const tables = await getTablesFromPattern(this.connectionManager, tablePattern);
|
|
206
|
+
// A missing table is still reported, so that its errors are surfaced.
|
|
207
|
+
const resolvedTable: ResolvedTable = tables[0] ?? { objectId: 0, schema: schema, name: tablePattern.name };
|
|
208
|
+
patternResult.table = await getDebugTableInfo({
|
|
209
|
+
connectionManager: this.connectionManager,
|
|
210
|
+
tablePattern,
|
|
211
|
+
table: resolvedTable,
|
|
212
|
+
syncRules: sqlSyncRules
|
|
213
|
+
});
|
|
224
214
|
}
|
|
225
215
|
|
|
226
216
|
return result;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { SourceEntityDescriptor, SourceTable } from '@powersync/service-core';
|
|
2
2
|
import { ServiceAssertionError } from '@powersync/service-errors';
|
|
3
|
+
import { readCaptureMetadata } from '../replication/CaptureReconciler.js';
|
|
3
4
|
import { toQualifiedTableName } from '../utils/mssql.js';
|
|
4
5
|
import { CaptureInstance } from './CaptureInstance.js';
|
|
5
6
|
|
|
@@ -26,9 +27,15 @@ export class MSSQLSourceTable {
|
|
|
26
27
|
|
|
27
28
|
public readonly ref: SourceEntityDescriptor;
|
|
28
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Decoded once, since the persisted metadata only changes when a SourceTable is replaced.
|
|
32
|
+
*/
|
|
33
|
+
private captureObjectId: number | null;
|
|
34
|
+
|
|
29
35
|
constructor(ref: SourceEntityDescriptor, sourceTables: SourceTable[]) {
|
|
30
36
|
this.sourceTables = sourceTables;
|
|
31
37
|
this.ref = ref;
|
|
38
|
+
this.captureObjectId = readPinnedCaptureObjectId(sourceTables);
|
|
32
39
|
}
|
|
33
40
|
|
|
34
41
|
updateSourceTable(updated: SourceTable): void {
|
|
@@ -37,6 +44,7 @@ export class MSSQLSourceTable {
|
|
|
37
44
|
throw new ServiceAssertionError(`No SourceTable found for table: ${updated.id}`);
|
|
38
45
|
}
|
|
39
46
|
this.sourceTables[index] = updated;
|
|
47
|
+
this.captureObjectId = readPinnedCaptureObjectId(this.sourceTables);
|
|
40
48
|
}
|
|
41
49
|
|
|
42
50
|
getReplicatedSourceTables(): SourceTable[] {
|
|
@@ -47,12 +55,19 @@ export class MSSQLSourceTable {
|
|
|
47
55
|
return this.captureInstance !== null;
|
|
48
56
|
}
|
|
49
57
|
|
|
50
|
-
|
|
51
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Persisted capture-table object id, or null for a legacy binding.
|
|
60
|
+
*/
|
|
61
|
+
get pinnedCaptureObjectId(): number | null {
|
|
62
|
+
return this.captureObjectId;
|
|
52
63
|
}
|
|
53
64
|
|
|
54
|
-
|
|
55
|
-
|
|
65
|
+
/**
|
|
66
|
+
* Bind the available capture instance matching this table's persisted identity.
|
|
67
|
+
*/
|
|
68
|
+
setCaptureInstance(availableInstances: readonly CaptureInstance[]): void {
|
|
69
|
+
this.captureInstance =
|
|
70
|
+
availableInstances.find((instance) => instance.objectId === this.pinnedCaptureObjectId) ?? null;
|
|
56
71
|
}
|
|
57
72
|
|
|
58
73
|
get allChangesFunction() {
|
|
@@ -84,3 +99,16 @@ export class MSSQLSourceTable {
|
|
|
84
99
|
return toQualifiedTableName(this.ref.schema, this.ref.name);
|
|
85
100
|
}
|
|
86
101
|
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The pinned capture-table object id shared by these records, or null for a legacy binding.
|
|
105
|
+
*/
|
|
106
|
+
function readPinnedCaptureObjectId(sourceTables: SourceTable[]): number | null {
|
|
107
|
+
for (const sourceTable of sourceTables) {
|
|
108
|
+
const metadata = readCaptureMetadata(sourceTable.sourceMetadata);
|
|
109
|
+
if (metadata != null) {
|
|
110
|
+
return metadata.captureTableObjectId;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
@@ -41,7 +41,8 @@ export class MSSQLModule extends replication.ReplicationModule<types.MSSQLConnec
|
|
|
41
41
|
metricsEngine: context.metricsEngine,
|
|
42
42
|
connectionFactory: connectionFactory,
|
|
43
43
|
rateLimiter: new MSSQLErrorRateLimiter(),
|
|
44
|
-
additionalConfig: normalisedConfig.additionalConfig
|
|
44
|
+
additionalConfig: normalisedConfig.additionalConfig,
|
|
45
|
+
heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
|
|
45
46
|
});
|
|
46
47
|
}
|
|
47
48
|
|