@powersync/service-core 0.0.0-dev-20241007145127 → 0.0.0-dev-20241015210820
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -5
- package/dist/api/RouteAPI.d.ts +6 -4
- package/dist/api/diagnostics.js +169 -105
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/schema.js +2 -2
- package/dist/api/schema.js.map +1 -1
- package/dist/entry/commands/compact-action.js +73 -9
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js +31 -0
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js.map +1 -0
- package/dist/replication/AbstractReplicationJob.d.ts +1 -1
- package/dist/replication/AbstractReplicationJob.js +2 -2
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +2 -2
- package/dist/replication/AbstractReplicator.js +66 -3
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationEngine.js.map +1 -1
- package/dist/replication/ReplicationModule.js +3 -0
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/replication/replication-index.d.ts +1 -1
- package/dist/replication/replication-index.js +1 -1
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/routes/configure-fastify.js +12 -12
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/configure-rsocket.js +4 -1
- package/dist/routes/configure-rsocket.js.map +1 -1
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +5 -2
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/router.d.ts +8 -1
- package/dist/routes/router.js.map +1 -1
- package/dist/runner/teardown.js +66 -4
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +41 -18
- package/dist/storage/BucketStorage.js +6 -0
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +12 -5
- package/dist/storage/MongoBucketStorage.js +44 -23
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +14 -0
- package/dist/storage/ReplicationEventPayload.js +2 -0
- package/dist/storage/ReplicationEventPayload.js.map +1 -0
- package/dist/storage/SourceTable.d.ts +8 -0
- package/dist/storage/SourceTable.js +9 -1
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +10 -2
- package/dist/storage/StorageEngine.js +23 -3
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +9 -2
- package/dist/storage/mongo/MongoBucketBatch.d.ts +12 -4
- package/dist/storage/mongo/MongoBucketBatch.js +60 -21
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/MongoStorageProvider.d.ts +1 -1
- package/dist/storage/mongo/MongoStorageProvider.js +3 -2
- package/dist/storage/mongo/MongoStorageProvider.js.map +1 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +4 -5
- package/dist/storage/mongo/MongoSyncBucketStorage.js +74 -12
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +18 -0
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js +90 -0
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -0
- package/dist/storage/mongo/db.d.ts +3 -2
- package/dist/storage/mongo/db.js +1 -0
- package/dist/storage/mongo/db.js.map +1 -1
- package/dist/storage/mongo/models.d.ts +7 -1
- package/dist/storage/storage-index.d.ts +2 -0
- package/dist/storage/storage-index.js +2 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/write-checkpoint.d.ts +55 -0
- package/dist/storage/write-checkpoint.js +16 -0
- package/dist/storage/write-checkpoint.js.map +1 -0
- package/dist/util/protocol-types.d.ts +2 -1
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +7 -4
- package/src/api/diagnostics.ts +4 -2
- package/src/api/schema.ts +3 -3
- package/src/entry/commands/compact-action.ts +4 -2
- package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts +37 -0
- package/src/replication/AbstractReplicationJob.ts +4 -4
- package/src/replication/AbstractReplicator.ts +5 -4
- package/src/replication/ReplicationEngine.ts +1 -1
- package/src/replication/ReplicationModule.ts +4 -0
- package/src/replication/replication-index.ts +1 -1
- package/src/routes/configure-fastify.ts +16 -17
- package/src/routes/configure-rsocket.ts +7 -2
- package/src/routes/endpoints/admin.ts +2 -2
- package/src/routes/endpoints/checkpointing.ts +5 -2
- package/src/routes/endpoints/sync-rules.ts +1 -0
- package/src/routes/router.ts +7 -1
- package/src/runner/teardown.ts +3 -3
- package/src/storage/BucketStorage.ts +50 -19
- package/src/storage/MongoBucketStorage.ts +70 -29
- package/src/storage/ReplicationEventPayload.ts +16 -0
- package/src/storage/SourceTable.ts +10 -1
- package/src/storage/StorageEngine.ts +34 -5
- package/src/storage/StorageProvider.ts +10 -2
- package/src/storage/mongo/MongoBucketBatch.ts +83 -27
- package/src/storage/mongo/MongoStorageProvider.ts +4 -3
- package/src/storage/mongo/MongoSyncBucketStorage.ts +22 -18
- package/src/storage/mongo/MongoWriteCheckpointAPI.ts +136 -0
- package/src/storage/mongo/db.ts +4 -1
- package/src/storage/mongo/models.ts +8 -1
- package/src/storage/storage-index.ts +2 -0
- package/src/storage/write-checkpoint.ts +67 -0
- package/src/util/protocol-types.ts +1 -1
- package/test/src/compacting.test.ts +13 -15
- package/test/src/data_storage.test.ts +95 -63
- package/test/src/sync.test.ts +10 -9
- package/test/src/util.ts +1 -2
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# @powersync/service-core
|
|
2
2
|
|
|
3
|
-
## 0.0.0-dev-
|
|
3
|
+
## 0.0.0-dev-20241015210820
|
|
4
4
|
|
|
5
5
|
### Minor Changes
|
|
6
6
|
|
|
7
|
+
- d2ece1b: Added ability to emit data replication events
|
|
7
8
|
- d51921f: - Introduced modules to the powersync service architecture
|
|
8
9
|
- Core functionality has been moved to "engine" classes. Modules can register additional functionality with these engines.
|
|
9
10
|
- The sync API functionality used by the routes has been abstracted to an interface. API routes are now managed by the RouterEngine.
|
|
@@ -16,14 +17,17 @@
|
|
|
16
17
|
- e4d19b2: Moved tag variable initialization in diagnostics route to ensure it is initialized before usage
|
|
17
18
|
- 1fd50a5: Fix checksum cache edge case with compacting
|
|
18
19
|
- aa4eb0a: Fix "JavaScript heap out of memory" on startup (slot health check)
|
|
20
|
+
- Updated dependencies [d2ece1b]
|
|
21
|
+
- Updated dependencies [d2ece1b]
|
|
19
22
|
- Updated dependencies [9e78ff1]
|
|
23
|
+
- Updated dependencies [d2ece1b]
|
|
20
24
|
- Updated dependencies [d51921f]
|
|
21
25
|
- Updated dependencies [4ecaee2]
|
|
22
26
|
- Updated dependencies [0e16938]
|
|
23
|
-
- @powersync/
|
|
24
|
-
- @powersync/
|
|
25
|
-
- @powersync/service-
|
|
26
|
-
- @powersync/service-
|
|
27
|
+
- @powersync/lib-services-framework@0.0.0-dev-20241015210820
|
|
28
|
+
- @powersync/service-sync-rules@0.0.0-dev-20241015210820
|
|
29
|
+
- @powersync/service-rsocket-router@0.0.0-dev-20241015210820
|
|
30
|
+
- @powersync/service-types@0.0.0-dev-20241015210820
|
|
27
31
|
|
|
28
32
|
## 0.8.4
|
|
29
33
|
|
package/dist/api/RouteAPI.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as types from '@powersync/service-types';
|
|
3
|
-
import { ParseSyncRulesOptions } from '../storage/BucketStorage.js';
|
|
3
|
+
import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/BucketStorage.js';
|
|
4
4
|
export interface PatternResult {
|
|
5
5
|
schema: string;
|
|
6
6
|
pattern: string;
|
|
@@ -8,6 +8,9 @@ export interface PatternResult {
|
|
|
8
8
|
tables?: types.TableInfo[];
|
|
9
9
|
table?: types.TableInfo;
|
|
10
10
|
}
|
|
11
|
+
export interface ReplicationLagOptions {
|
|
12
|
+
bucketStorage: SyncRulesBucketStorage;
|
|
13
|
+
}
|
|
11
14
|
/**
|
|
12
15
|
* Describes all the methods currently required to service the sync API endpoints.
|
|
13
16
|
*/
|
|
@@ -15,7 +18,7 @@ export interface RouteAPI {
|
|
|
15
18
|
/**
|
|
16
19
|
* @returns basic identification of the connection
|
|
17
20
|
*/
|
|
18
|
-
getSourceConfig(): Promise<types.configFile.
|
|
21
|
+
getSourceConfig(): Promise<types.configFile.ResolvedDataSourceConfig>;
|
|
19
22
|
/**
|
|
20
23
|
* Checks the current connection status of the data source.
|
|
21
24
|
* This is usually some test query to verify the source can be reached.
|
|
@@ -37,9 +40,8 @@ export interface RouteAPI {
|
|
|
37
40
|
/**
|
|
38
41
|
* @returns The replication lag: that is the amount of data which has not been
|
|
39
42
|
* replicated yet, in bytes.
|
|
40
|
-
* @param {string} syncRulesId An identifier representing which set of sync rules the lag is required for.
|
|
41
43
|
*/
|
|
42
|
-
getReplicationLag(
|
|
44
|
+
getReplicationLag(options: ReplicationLagOptions): Promise<number | undefined>;
|
|
43
45
|
/**
|
|
44
46
|
* Get the current LSN or equivalent replication HEAD position identifier
|
|
45
47
|
*/
|
package/dist/api/diagnostics.js
CHANGED
|
@@ -1,119 +1,183 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
1
53
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
54
|
import { DEFAULT_TAG } from '@powersync/service-sync-rules';
|
|
3
55
|
export const DEFAULT_DATASOURCE_ID = 'default';
|
|
4
56
|
export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules, options) {
|
|
5
|
-
|
|
6
|
-
return undefined;
|
|
7
|
-
}
|
|
8
|
-
const include_content = options.include_content ?? false;
|
|
9
|
-
const live_status = options.live_status ?? false;
|
|
10
|
-
const check_connection = options.check_connection ?? false;
|
|
11
|
-
let rules;
|
|
12
|
-
let persisted;
|
|
57
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
13
58
|
try {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
59
|
+
if (sync_rules == null) {
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
const include_content = options.include_content ?? false;
|
|
63
|
+
const live_status = options.live_status ?? false;
|
|
64
|
+
const check_connection = options.check_connection ?? false;
|
|
65
|
+
let rules;
|
|
66
|
+
let persisted;
|
|
67
|
+
try {
|
|
68
|
+
persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
|
|
69
|
+
rules = persisted.sync_rules;
|
|
70
|
+
}
|
|
71
|
+
catch (e) {
|
|
72
|
+
return {
|
|
73
|
+
content: include_content ? sync_rules.sync_rules_content : undefined,
|
|
74
|
+
connections: [],
|
|
75
|
+
errors: [{ level: 'fatal', message: e.message }]
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
const sourceConfig = await apiHandler.getSourceConfig();
|
|
79
|
+
// This method can run under some situations if no connection is configured yet.
|
|
80
|
+
// It will return a default tag in such a case. This default tag is not module specific.
|
|
81
|
+
const tag = sourceConfig.tag ?? DEFAULT_TAG;
|
|
82
|
+
const systemStorage = __addDisposableResource(env_1, live_status ? bucketStorage.getInstance(sync_rules) : undefined, false);
|
|
83
|
+
const status = await systemStorage?.getStatus();
|
|
84
|
+
let replication_lag_bytes = undefined;
|
|
85
|
+
let tables_flat = [];
|
|
86
|
+
if (check_connection) {
|
|
87
|
+
const source_table_patterns = rules.getSourceTables();
|
|
88
|
+
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
|
|
89
|
+
tables_flat = resolved_tables.flatMap((info) => {
|
|
90
|
+
if (info.table) {
|
|
91
|
+
return [info.table];
|
|
92
|
+
}
|
|
93
|
+
else if (info.tables) {
|
|
94
|
+
return info.tables;
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
if (systemStorage) {
|
|
101
|
+
try {
|
|
102
|
+
replication_lag_bytes = await apiHandler.getReplicationLag({
|
|
103
|
+
bucketStorage: systemStorage
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
catch (e) {
|
|
107
|
+
// Ignore
|
|
108
|
+
logger.warn(`Unable to get replication lag`, e);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
const source_table_patterns = rules.getSourceTables();
|
|
114
|
+
tables_flat = source_table_patterns.map((pattern) => {
|
|
115
|
+
if (pattern.isWildcard) {
|
|
116
|
+
return {
|
|
117
|
+
schema: pattern.schema,
|
|
118
|
+
name: pattern.tablePrefix,
|
|
119
|
+
pattern: pattern.isWildcard ? pattern.tablePattern : undefined,
|
|
120
|
+
data_queries: false,
|
|
121
|
+
parameter_queries: false,
|
|
122
|
+
replication_id: [],
|
|
123
|
+
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
const source = {
|
|
128
|
+
connectionTag: tag,
|
|
129
|
+
schema: pattern.schema,
|
|
130
|
+
table: pattern.tablePattern
|
|
131
|
+
};
|
|
132
|
+
const syncData = rules.tableSyncsData(source);
|
|
133
|
+
const syncParameters = rules.tableSyncsParameters(source);
|
|
134
|
+
return {
|
|
135
|
+
schema: pattern.schema,
|
|
136
|
+
name: pattern.name,
|
|
137
|
+
data_queries: syncData,
|
|
138
|
+
parameter_queries: syncParameters,
|
|
139
|
+
replication_id: [],
|
|
140
|
+
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
const errors = tables_flat.flatMap((info) => info.errors);
|
|
146
|
+
if (sync_rules.last_fatal_error) {
|
|
147
|
+
errors.push({ level: 'fatal', message: sync_rules.last_fatal_error });
|
|
148
|
+
}
|
|
149
|
+
errors.push(...rules.errors.map((e) => {
|
|
150
|
+
return {
|
|
151
|
+
level: e.type,
|
|
152
|
+
message: e.message
|
|
153
|
+
};
|
|
154
|
+
}));
|
|
18
155
|
return {
|
|
19
156
|
content: include_content ? sync_rules.sync_rules_content : undefined,
|
|
20
|
-
connections: [
|
|
21
|
-
|
|
157
|
+
connections: [
|
|
158
|
+
{
|
|
159
|
+
id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
|
|
160
|
+
tag: tag,
|
|
161
|
+
slot_name: sync_rules.slot_name,
|
|
162
|
+
initial_replication_done: status?.snapshot_done ?? false,
|
|
163
|
+
// TODO: Rename?
|
|
164
|
+
last_lsn: status?.checkpoint_lsn ?? undefined,
|
|
165
|
+
last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
|
|
166
|
+
last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
|
|
167
|
+
replication_lag_bytes: replication_lag_bytes,
|
|
168
|
+
tables: tables_flat
|
|
169
|
+
}
|
|
170
|
+
],
|
|
171
|
+
errors: deduplicate(errors)
|
|
22
172
|
};
|
|
23
173
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const tag = sourceConfig.tag ?? DEFAULT_TAG;
|
|
28
|
-
const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
|
|
29
|
-
const status = await systemStorage?.getStatus();
|
|
30
|
-
let replication_lag_bytes = undefined;
|
|
31
|
-
let tables_flat = [];
|
|
32
|
-
if (check_connection) {
|
|
33
|
-
const source_table_patterns = rules.getSourceTables();
|
|
34
|
-
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
|
|
35
|
-
tables_flat = resolved_tables.flatMap((info) => {
|
|
36
|
-
if (info.table) {
|
|
37
|
-
return [info.table];
|
|
38
|
-
}
|
|
39
|
-
else if (info.tables) {
|
|
40
|
-
return info.tables;
|
|
41
|
-
}
|
|
42
|
-
else {
|
|
43
|
-
return [];
|
|
44
|
-
}
|
|
45
|
-
});
|
|
46
|
-
if (systemStorage) {
|
|
47
|
-
try {
|
|
48
|
-
replication_lag_bytes = await apiHandler.getReplicationLag(systemStorage.slot_name);
|
|
49
|
-
}
|
|
50
|
-
catch (e) {
|
|
51
|
-
// Ignore
|
|
52
|
-
logger.warn(`Unable to get replication lag`, e);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
174
|
+
catch (e_1) {
|
|
175
|
+
env_1.error = e_1;
|
|
176
|
+
env_1.hasError = true;
|
|
55
177
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
tables_flat = source_table_patterns.map((pattern) => {
|
|
59
|
-
if (pattern.isWildcard) {
|
|
60
|
-
return {
|
|
61
|
-
schema: pattern.schema,
|
|
62
|
-
name: pattern.tablePrefix,
|
|
63
|
-
pattern: pattern.isWildcard ? pattern.tablePattern : undefined,
|
|
64
|
-
data_queries: false,
|
|
65
|
-
parameter_queries: false,
|
|
66
|
-
replication_id: [],
|
|
67
|
-
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
const source = {
|
|
72
|
-
connectionTag: tag,
|
|
73
|
-
schema: pattern.schema,
|
|
74
|
-
table: pattern.tablePattern
|
|
75
|
-
};
|
|
76
|
-
const syncData = rules.tableSyncsData(source);
|
|
77
|
-
const syncParameters = rules.tableSyncsParameters(source);
|
|
78
|
-
return {
|
|
79
|
-
schema: pattern.schema,
|
|
80
|
-
name: pattern.name,
|
|
81
|
-
data_queries: syncData,
|
|
82
|
-
parameter_queries: syncParameters,
|
|
83
|
-
replication_id: [],
|
|
84
|
-
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
});
|
|
178
|
+
finally {
|
|
179
|
+
__disposeResources(env_1);
|
|
88
180
|
}
|
|
89
|
-
const errors = tables_flat.flatMap((info) => info.errors);
|
|
90
|
-
if (sync_rules.last_fatal_error) {
|
|
91
|
-
errors.push({ level: 'fatal', message: sync_rules.last_fatal_error });
|
|
92
|
-
}
|
|
93
|
-
errors.push(...rules.errors.map((e) => {
|
|
94
|
-
return {
|
|
95
|
-
level: e.type,
|
|
96
|
-
message: e.message
|
|
97
|
-
};
|
|
98
|
-
}));
|
|
99
|
-
return {
|
|
100
|
-
content: include_content ? sync_rules.sync_rules_content : undefined,
|
|
101
|
-
connections: [
|
|
102
|
-
{
|
|
103
|
-
id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
|
|
104
|
-
tag: tag,
|
|
105
|
-
slot_name: sync_rules.slot_name,
|
|
106
|
-
initial_replication_done: status?.snapshot_done ?? false,
|
|
107
|
-
// TODO: Rename?
|
|
108
|
-
last_lsn: status?.checkpoint_lsn ?? undefined,
|
|
109
|
-
last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
|
|
110
|
-
last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
|
|
111
|
-
replication_lag_bytes: replication_lag_bytes,
|
|
112
|
-
tables: tables_flat
|
|
113
|
-
}
|
|
114
|
-
],
|
|
115
|
-
errors: deduplicate(errors)
|
|
116
|
-
};
|
|
117
181
|
}
|
|
118
182
|
function deduplicate(errors) {
|
|
119
183
|
let seen = new Set();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAsC,MAAM,+BAA+B,CAAC;AAyBhG,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,aAA2C,EAC3C,UAAoB,EACpB,UAAoD,EACpD,OAA2B
|
|
1
|
+
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAsC,MAAM,+BAA+B,CAAC;AAyBhG,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,aAA2C,EAC3C,UAAoB,EACpB,UAAoD,EACpD,OAA2B;;;QAE3B,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;QACzD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;QACjD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,KAAK,CAAC;QAE3D,IAAI,KAAmB,CAAC;QACxB,IAAI,SAAqC,CAAC;QAC1C,IAAI,CAAC;YACH,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;YACrE,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;gBACpE,WAAW,EAAE,EAAE;gBACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;QACxD,gFAAgF;QAChF,wFAAwF;QACxF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,WAAW,CAAC;QAC5C,MAAM,aAAa,kCAAG,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,QAAA,CAAC;QACtF,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,SAAS,EAAE,CAAC;QAChD,IAAI,qBAAqB,GAAuB,SAAS,CAAC;QAE1D,IAAI,WAAW,GAAgB,EAAE,CAAC;QAElC,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;YACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;YAC1F,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC7C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACtB,CAAC;qBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC,MAAM,CAAC;gBACrB,CAAC;qBAAM,CAAC;oBACN,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,aAAa,EAAE,CAAC;gBAClB,IAAI,CAAC;oBACH,qBAAqB,GAAG,MAAM,UAAU,CAAC,iBAAiB,CAAC;wBACzD,aAAa,EAAE,aAAa;qBAC7B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,SAAS;oBACT,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;gBAClD,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;YAEtD,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAa,EAAE;gBAC7D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;oBACvB,OAAO;wBACL,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,IAAI,EAAE,OAAO,CAAC,WAAW;wBACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;wBAE9D,YAAY,EAAE,KAAK;wBACnB,iBAAiB,EAAE,KAAK;wBACxB,cAAc,EAAE,EAAE;wBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;qBAC3D,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,GAAyB;wBACnC,aAAa,EAAE,GAAG;wBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,KAAK,EAAE,OAAO,CAAC,YAAY;qBAC5B,CAAC;oBACF,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;oBAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;oBAC1D,OAAO;wBACL,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,YAAY,EAAE,QAAQ;wBACtB,iBAAiB,EAAE,cAAc;wBACjC,cAAc,EAAE,EAAE;wBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;qBAC3D,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,CAAC,IAAI,CACT,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACxB,OAAO;gBACL,KAAK,EAAE,CAAC,CAAC,IAAI;gBACb,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,OAAO;YACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACpE,WAAW,EAAE;gBACX;oBACE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,qBAAqB;oBAC5C,GAAG,EAAE,GAAG;oBACR,SAAS,EAAE,UAAU,CAAC,SAAS;oBAC/B,wBAAwB,EAAE,MAAM,EAAE,aAAa,IAAI,KAAK;oBACxD,gBAAgB;oBAChB,QAAQ,EAAE,MAAM,EAAE,cAAc,IAAI,SAAS;oBAC7C,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,EAAE,WAAW,EAAE;oBAChE,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,EAAE,WAAW,EAAE;oBAC9D,qBAAqB,EAAE,qBAAqB;oBAC5C,MAAM,EAAE,WAAW;iBACpB;aACF;YACD,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;SAC5B,CAAC;;;;;;;;;CACH;AAED,SAAS,WAAW,CAAC,MAAyD;IAC5E,IAAI,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7B,IAAI,MAAM,GAAsD,EAAE,CAAC;IACnE,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/api/schema.js
CHANGED
|
@@ -10,9 +10,9 @@ export async function getConnectionsSchema(api) {
|
|
|
10
10
|
return {
|
|
11
11
|
connections: [
|
|
12
12
|
{
|
|
13
|
-
|
|
13
|
+
id: baseConfig.id,
|
|
14
14
|
tag: baseConfig.tag,
|
|
15
|
-
|
|
15
|
+
schemas: await api.getConnectionSchema()
|
|
16
16
|
}
|
|
17
17
|
],
|
|
18
18
|
defaultConnectionTag: baseConfig.tag,
|
package/dist/api/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/api/schema.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAiB;IAC1D,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,WAAW,EAAE,EAAE;YACf,oBAAoB,EAAE,SAAS;YAC/B,aAAa,EAAE,EAAE;SAClB,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE;YACX;gBACE,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/api/schema.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAiB;IAC1D,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,WAAW,EAAE,EAAE;YACf,oBAAoB,EAAE,SAAS;YAC/B,aAAa,EAAE,EAAE;SAClB,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,UAAU,CAAC,EAAE;gBACjB,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,OAAO,EAAE,MAAM,GAAG,CAAC,mBAAmB,EAAE;aACzC;SACF;QACD,oBAAoB,EAAE,UAAU,CAAC,GAAI;QACrC,aAAa,EAAE,GAAG,CAAC,wBAAwB,EAAE,CAAC,aAAa;KAC5D,CAAC;AACJ,CAAC"}
|
|
@@ -1,3 +1,55 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
1
53
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
54
|
import * as v8 from 'v8';
|
|
3
55
|
import * as storage from '../../storage/storage-index.js';
|
|
@@ -28,16 +80,28 @@ export function registerCompactAction(program) {
|
|
|
28
80
|
const client = psdb.client;
|
|
29
81
|
await client.connect();
|
|
30
82
|
try {
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
83
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
84
|
+
try {
|
|
85
|
+
const bucketStorage = new storage.MongoBucketStorage(psdb, {
|
|
86
|
+
slot_name_prefix: configuration.slot_name_prefix
|
|
87
|
+
});
|
|
88
|
+
const active = await bucketStorage.getActiveSyncRulesContent();
|
|
89
|
+
if (active == null) {
|
|
90
|
+
logger.info('No active instance to compact');
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
const p = __addDisposableResource(env_1, bucketStorage.getInstance(active), false);
|
|
94
|
+
logger.info('Performing compaction...');
|
|
95
|
+
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
|
|
96
|
+
logger.info('Successfully compacted storage.');
|
|
97
|
+
}
|
|
98
|
+
catch (e_1) {
|
|
99
|
+
env_1.error = e_1;
|
|
100
|
+
env_1.hasError = true;
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
__disposeResources(env_1);
|
|
36
104
|
}
|
|
37
|
-
const p = bucketStorage.getInstance(active);
|
|
38
|
-
logger.info('Performing compaction...');
|
|
39
|
-
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
|
|
40
|
-
logger.info('Successfully compacted storage.');
|
|
41
105
|
}
|
|
42
106
|
catch (e) {
|
|
43
107
|
logger.error(`Failed to compact: ${e.toString()}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,OAAO,MAAM,gCAAgC,CAAC;AAC1D,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAErD,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACpD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,aAAa,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC
|
|
1
|
+
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,OAAO,MAAM,gCAAgC,CAAC;AAC1D,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAErD,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACpD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,aAAa,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC;;;gBACH,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE;oBACzD,gBAAgB,EAAE,aAAa,CAAC,gBAAgB;iBACjD,CAAC,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,yBAAyB,EAAE,CAAC;gBAC/D,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;oBAC7C,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,kCAAG,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,QAAA,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBACxC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,uBAAuB,EAAE,CAAC,CAAC;gBAC5D,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;;;;;;;;;SAChD;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import * as storage from '../../../storage/storage-index.js';
|
|
2
|
+
import * as utils from '../../../util/util-index.js';
|
|
3
|
+
const INDEX_NAME = 'user_sync_rule_unique';
|
|
4
|
+
export const up = async (context) => {
|
|
5
|
+
const { runner_config } = context;
|
|
6
|
+
const config = await utils.loadConfig(runner_config);
|
|
7
|
+
const db = storage.createPowerSyncMongo(config.storage);
|
|
8
|
+
try {
|
|
9
|
+
await db.custom_write_checkpoints.createIndex({
|
|
10
|
+
user_id: 1,
|
|
11
|
+
sync_rules_id: 1
|
|
12
|
+
}, { name: INDEX_NAME, unique: true });
|
|
13
|
+
}
|
|
14
|
+
finally {
|
|
15
|
+
await db.client.close();
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
export const down = async (context) => {
|
|
19
|
+
const { runner_config } = context;
|
|
20
|
+
const config = await utils.loadConfig(runner_config);
|
|
21
|
+
const db = storage.createPowerSyncMongo(config.storage);
|
|
22
|
+
try {
|
|
23
|
+
if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
|
|
24
|
+
await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
finally {
|
|
28
|
+
await db.client.close();
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
//# sourceMappingURL=1727099539247-custom-write-checkpoint-index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"1727099539247-custom-write-checkpoint-index.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAC7D,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAErD,MAAM,UAAU,GAAG,uBAAuB,CAAC;AAE3C,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,EAAE,OAA+B,EAAE,EAAE;IAC1D,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAExD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAC3C;YACE,OAAO,EAAE,CAAC;YACV,aAAa,EAAE,CAAC;SACjB,EACD,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CACnC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,OAA+B,EAAE,EAAE;IAC5D,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAErD,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAExD,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import winston from 'winston';
|
|
1
2
|
import * as storage from '../storage/storage-index.js';
|
|
2
3
|
import { ErrorRateLimiter } from './ErrorRateLimiter.js';
|
|
3
|
-
import winston from 'winston';
|
|
4
4
|
export interface AbstractReplicationJobOptions {
|
|
5
5
|
id: string;
|
|
6
6
|
storage: storage.SyncRulesBucketStorage;
|
|
@@ -4,7 +4,7 @@ export class AbstractReplicationJob {
|
|
|
4
4
|
this.options = options;
|
|
5
5
|
this.abortController = new AbortController();
|
|
6
6
|
this.isReplicatingPromise = null;
|
|
7
|
-
this.logger = logger.child({ name: `ReplicationJob: ${
|
|
7
|
+
this.logger = logger.child({ name: `ReplicationJob: ${this.id}` });
|
|
8
8
|
}
|
|
9
9
|
/**
|
|
10
10
|
* Start the replication process
|
|
@@ -28,7 +28,7 @@ export class AbstractReplicationJob {
|
|
|
28
28
|
* Safely stop the replication process
|
|
29
29
|
*/
|
|
30
30
|
async stop() {
|
|
31
|
-
this.logger.info(`Stopping
|
|
31
|
+
this.logger.info(`Stopping replication job for sync rule iteration: ${this.storage.group_id}`);
|
|
32
32
|
this.abortController.abort();
|
|
33
33
|
await this.isReplicatingPromise;
|
|
34
34
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractReplicationJob.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicationJob.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"AbstractReplicationJob.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAYtE,MAAM,OAAgB,sBAAsB;IAK1C,YAAgC,OAAsC;QAAtC,YAAO,GAAP,OAAO,CAA+B;QAH5D,oBAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QACxC,yBAAoB,GAAyB,IAAI,CAAC;QAG1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACrE,CAAC;IAYD;;OAEG;IACI,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,SAAS,EAAE;aACzC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;YACZ,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE;gBACtC,QAAQ,EAAE;oBACR,UAAU,EAAE,IAAI,CAAC,EAAE;iBACpB;aACF,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC;aACD,OAAO,CAAC,KAAK,IAAI,EAAE;YAClB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,oBAAoB,CAAC;IAClC,CAAC;IAED,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAW,OAAO;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IAC9B,CAAC;IAED,IAAc,IAAI;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAW,SAAS;QAClB,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;IAC7C,CAAC;CACF"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import winston from 'winston';
|
|
1
2
|
import * as storage from '../storage/storage-index.js';
|
|
3
|
+
import { StorageEngine } from '../storage/storage-index.js';
|
|
2
4
|
import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
|
|
3
|
-
import winston from 'winston';
|
|
4
5
|
import { AbstractReplicationJob } from './AbstractReplicationJob.js';
|
|
5
|
-
import { StorageEngine } from '../storage/storage-index.js';
|
|
6
6
|
import { ErrorRateLimiter } from './ErrorRateLimiter.js';
|
|
7
7
|
export interface CreateJobOptions {
|
|
8
8
|
lock: storage.ReplicationLock;
|