@powersync/service-core 0.18.1 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/api/RouteAPI.d.ts +1 -1
- package/dist/api/diagnostics.js +107 -169
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +10 -73
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +1 -1
- package/dist/replication/AbstractReplicator.js +8 -76
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +3 -2
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +5 -5
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +5 -5
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/runner/teardown.js +3 -65
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +8 -441
- package/dist/storage/BucketStorage.js +9 -10
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +130 -0
- package/dist/storage/BucketStorageBatch.js +10 -0
- package/dist/storage/BucketStorageBatch.js.map +1 -0
- package/dist/storage/BucketStorageFactory.d.ts +145 -0
- package/dist/storage/BucketStorageFactory.js +2 -0
- package/dist/storage/BucketStorageFactory.js.map +1 -0
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +20 -0
- package/dist/storage/PersistedSyncRulesContent.js +2 -0
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -0
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/ReplicationLock.d.ts +4 -0
- package/dist/storage/ReplicationLock.js +2 -0
- package/dist/storage/ReplicationLock.js.map +1 -0
- package/dist/storage/SourceEntity.d.ts +6 -2
- package/dist/storage/SourceTable.d.ts +2 -2
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +4 -4
- package/dist/storage/StorageEngine.js +2 -2
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +4 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +207 -0
- package/dist/storage/SyncRulesBucketStorage.js +7 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -0
- package/dist/storage/bson.d.ts +14 -3
- package/dist/storage/bson.js +18 -2
- package/dist/storage/bson.js.map +1 -1
- package/dist/storage/storage-index.d.ts +5 -0
- package/dist/storage/storage-index.js +5 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +91 -0
- package/dist/sync/BucketChecksumState.js +313 -0
- package/dist/sync/BucketChecksumState.js.map +1 -0
- package/dist/sync/sync-index.d.ts +1 -0
- package/dist/sync/sync-index.js +1 -0
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.d.ts +7 -3
- package/dist/sync/sync.js +139 -135
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +9 -0
- package/dist/sync/util.js +44 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/util/checkpointing.d.ts +1 -1
- package/dist/util/checkpointing.js +15 -78
- package/dist/util/checkpointing.js.map +1 -1
- package/dist/util/protocol-types.d.ts +13 -4
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +1 -1
- package/src/api/diagnostics.ts +1 -1
- package/src/entry/commands/compact-action.ts +2 -3
- package/src/modules/AbstractModule.ts +1 -1
- package/src/replication/AbstractReplicator.ts +7 -12
- package/src/routes/endpoints/checkpointing.ts +3 -3
- package/src/routes/endpoints/socket-route.ts +7 -5
- package/src/routes/endpoints/sync-stream.ts +8 -5
- package/src/runner/teardown.ts +1 -1
- package/src/storage/BucketStorage.ts +8 -550
- package/src/storage/BucketStorageBatch.ts +158 -0
- package/src/storage/BucketStorageFactory.ts +166 -0
- package/src/storage/ChecksumCache.ts +1 -0
- package/src/storage/PersistedSyncRulesContent.ts +26 -0
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/ReplicationLock.ts +5 -0
- package/src/storage/SourceEntity.ts +6 -2
- package/src/storage/SourceTable.ts +1 -1
- package/src/storage/StorageEngine.ts +4 -4
- package/src/storage/StorageProvider.ts +4 -1
- package/src/storage/SyncRulesBucketStorage.ts +265 -0
- package/src/storage/bson.ts +22 -4
- package/src/storage/storage-index.ts +5 -0
- package/src/sync/BucketChecksumState.ts +392 -0
- package/src/sync/sync-index.ts +1 -0
- package/src/sync/sync.ts +182 -157
- package/src/sync/util.ts +54 -0
- package/src/util/checkpointing.ts +4 -6
- package/src/util/protocol-types.ts +16 -4
- package/test/src/auth.test.ts +5 -5
- package/test/src/sync/BucketChecksumState.test.ts +565 -0
- package/test/src/sync/util.test.ts +34 -0
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
# @powersync/service-core
|
|
2
2
|
|
|
3
|
+
## 1.7.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- b4fe4ae: Upgrade mongodb and bson packages, removing the need for some workarounds.
|
|
8
|
+
- 88ab679: Keep serving current data when restarting replication due to errors.
|
|
9
|
+
- 2f75fd7: Improve handling of some edge cases which could trigger truncating of synced tables.
|
|
10
|
+
- 346382e: Unified compacting options between storage providers.
|
|
11
|
+
- 9b1868d: Fix missing checkpoint complete line for empty sync iterations.
|
|
12
|
+
- Updated dependencies [b4fe4ae]
|
|
13
|
+
- @powersync/service-rsocket-router@0.0.20
|
|
14
|
+
- @powersync/lib-services-framework@0.5.3
|
|
15
|
+
|
|
16
|
+
## 1.7.0
|
|
17
|
+
|
|
18
|
+
### Minor Changes
|
|
19
|
+
|
|
20
|
+
- 436eee6: Minor optimizations to new checkpoint calulations.
|
|
21
|
+
|
|
22
|
+
### Patch Changes
|
|
23
|
+
|
|
24
|
+
- 15283d4: Stream changes in priority order.
|
|
25
|
+
- 88d4cb3: Fix signed integer overflow issue for int64 values from MongoDB.
|
|
26
|
+
- Updated dependencies [436eee6]
|
|
27
|
+
- Updated dependencies [f55e36a]
|
|
28
|
+
- @powersync/service-sync-rules@0.24.0
|
|
29
|
+
- @powersync/lib-services-framework@0.5.2
|
|
30
|
+
- @powersync/service-rsocket-router@0.0.19
|
|
31
|
+
|
|
3
32
|
## 0.18.1
|
|
4
33
|
|
|
5
34
|
### Patch Changes
|
package/dist/api/RouteAPI.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as types from '@powersync/service-types';
|
|
3
|
-
import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/
|
|
3
|
+
import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
|
|
4
4
|
export interface PatternResult {
|
|
5
5
|
schema: string;
|
|
6
6
|
pattern: string;
|
package/dist/api/diagnostics.js
CHANGED
|
@@ -1,183 +1,121 @@
|
|
|
1
|
-
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
-
if (value !== null && value !== void 0) {
|
|
3
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
-
var dispose, inner;
|
|
5
|
-
if (async) {
|
|
6
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
-
dispose = value[Symbol.asyncDispose];
|
|
8
|
-
}
|
|
9
|
-
if (dispose === void 0) {
|
|
10
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
-
dispose = value[Symbol.dispose];
|
|
12
|
-
if (async) inner = dispose;
|
|
13
|
-
}
|
|
14
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
-
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
-
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
-
}
|
|
18
|
-
else if (async) {
|
|
19
|
-
env.stack.push({ async: true });
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
22
|
-
};
|
|
23
|
-
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
-
return function (env) {
|
|
25
|
-
function fail(e) {
|
|
26
|
-
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
-
env.hasError = true;
|
|
28
|
-
}
|
|
29
|
-
var r, s = 0;
|
|
30
|
-
function next() {
|
|
31
|
-
while (r = env.stack.pop()) {
|
|
32
|
-
try {
|
|
33
|
-
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
-
if (r.dispose) {
|
|
35
|
-
var result = r.dispose.call(r.value);
|
|
36
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
-
}
|
|
38
|
-
else s |= 1;
|
|
39
|
-
}
|
|
40
|
-
catch (e) {
|
|
41
|
-
fail(e);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
-
if (env.hasError) throw env.error;
|
|
46
|
-
}
|
|
47
|
-
return next();
|
|
48
|
-
};
|
|
49
|
-
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
-
var e = new Error(message);
|
|
51
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
-
});
|
|
53
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
54
2
|
import { DEFAULT_TAG } from '@powersync/service-sync-rules';
|
|
55
3
|
export const DEFAULT_DATASOURCE_ID = 'default';
|
|
56
4
|
export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules, options) {
|
|
57
|
-
|
|
5
|
+
if (sync_rules == null) {
|
|
6
|
+
return undefined;
|
|
7
|
+
}
|
|
8
|
+
const include_content = options.include_content ?? false;
|
|
9
|
+
const live_status = options.live_status ?? false;
|
|
10
|
+
const check_connection = options.check_connection ?? false;
|
|
11
|
+
let rules;
|
|
12
|
+
let persisted;
|
|
58
13
|
try {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const live_status = options.live_status ?? false;
|
|
64
|
-
const check_connection = options.check_connection ?? false;
|
|
65
|
-
let rules;
|
|
66
|
-
let persisted;
|
|
67
|
-
try {
|
|
68
|
-
persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
|
|
69
|
-
rules = persisted.sync_rules;
|
|
70
|
-
}
|
|
71
|
-
catch (e) {
|
|
72
|
-
return {
|
|
73
|
-
content: include_content ? sync_rules.sync_rules_content : undefined,
|
|
74
|
-
connections: [],
|
|
75
|
-
errors: [{ level: 'fatal', message: e.message }]
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
const sourceConfig = await apiHandler.getSourceConfig();
|
|
79
|
-
// This method can run under some situations if no connection is configured yet.
|
|
80
|
-
// It will return a default tag in such a case. This default tag is not module specific.
|
|
81
|
-
const tag = sourceConfig.tag ?? DEFAULT_TAG;
|
|
82
|
-
const systemStorage = __addDisposableResource(env_1, live_status ? bucketStorage.getInstance(sync_rules) : undefined, false);
|
|
83
|
-
const status = await systemStorage?.getStatus();
|
|
84
|
-
let replication_lag_bytes = undefined;
|
|
85
|
-
let tables_flat = [];
|
|
86
|
-
if (check_connection) {
|
|
87
|
-
const source_table_patterns = rules.getSourceTables();
|
|
88
|
-
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
|
|
89
|
-
tables_flat = resolved_tables.flatMap((info) => {
|
|
90
|
-
if (info.table) {
|
|
91
|
-
return [info.table];
|
|
92
|
-
}
|
|
93
|
-
else if (info.tables) {
|
|
94
|
-
return info.tables;
|
|
95
|
-
}
|
|
96
|
-
else {
|
|
97
|
-
return [];
|
|
98
|
-
}
|
|
99
|
-
});
|
|
100
|
-
if (systemStorage) {
|
|
101
|
-
try {
|
|
102
|
-
replication_lag_bytes = await apiHandler.getReplicationLag({
|
|
103
|
-
bucketStorage: systemStorage
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
catch (e) {
|
|
107
|
-
// Ignore
|
|
108
|
-
logger.warn(`Unable to get replication lag`, e);
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
else {
|
|
113
|
-
const source_table_patterns = rules.getSourceTables();
|
|
114
|
-
tables_flat = source_table_patterns.map((pattern) => {
|
|
115
|
-
if (pattern.isWildcard) {
|
|
116
|
-
return {
|
|
117
|
-
schema: pattern.schema,
|
|
118
|
-
name: pattern.tablePrefix,
|
|
119
|
-
pattern: pattern.isWildcard ? pattern.tablePattern : undefined,
|
|
120
|
-
data_queries: false,
|
|
121
|
-
parameter_queries: false,
|
|
122
|
-
replication_id: [],
|
|
123
|
-
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
else {
|
|
127
|
-
const source = {
|
|
128
|
-
connectionTag: tag,
|
|
129
|
-
schema: pattern.schema,
|
|
130
|
-
table: pattern.tablePattern
|
|
131
|
-
};
|
|
132
|
-
const syncData = rules.tableSyncsData(source);
|
|
133
|
-
const syncParameters = rules.tableSyncsParameters(source);
|
|
134
|
-
return {
|
|
135
|
-
schema: pattern.schema,
|
|
136
|
-
name: pattern.name,
|
|
137
|
-
data_queries: syncData,
|
|
138
|
-
parameter_queries: syncParameters,
|
|
139
|
-
replication_id: [],
|
|
140
|
-
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
|
-
const errors = tables_flat.flatMap((info) => info.errors);
|
|
146
|
-
if (sync_rules.last_fatal_error) {
|
|
147
|
-
errors.push({ level: 'fatal', message: sync_rules.last_fatal_error });
|
|
148
|
-
}
|
|
149
|
-
errors.push(...rules.errors.map((e) => {
|
|
150
|
-
return {
|
|
151
|
-
level: e.type,
|
|
152
|
-
message: e.message
|
|
153
|
-
};
|
|
154
|
-
}));
|
|
14
|
+
persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
|
|
15
|
+
rules = persisted.sync_rules;
|
|
16
|
+
}
|
|
17
|
+
catch (e) {
|
|
155
18
|
return {
|
|
156
19
|
content: include_content ? sync_rules.sync_rules_content : undefined,
|
|
157
|
-
connections: [
|
|
158
|
-
|
|
159
|
-
id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
|
|
160
|
-
tag: tag,
|
|
161
|
-
slot_name: sync_rules.slot_name,
|
|
162
|
-
initial_replication_done: status?.snapshot_done ?? false,
|
|
163
|
-
// TODO: Rename?
|
|
164
|
-
last_lsn: status?.checkpoint_lsn ?? undefined,
|
|
165
|
-
last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
|
|
166
|
-
last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
|
|
167
|
-
replication_lag_bytes: replication_lag_bytes,
|
|
168
|
-
tables: tables_flat
|
|
169
|
-
}
|
|
170
|
-
],
|
|
171
|
-
errors: deduplicate(errors)
|
|
20
|
+
connections: [],
|
|
21
|
+
errors: [{ level: 'fatal', message: e.message }]
|
|
172
22
|
};
|
|
173
23
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
24
|
+
const sourceConfig = await apiHandler.getSourceConfig();
|
|
25
|
+
// This method can run under some situations if no connection is configured yet.
|
|
26
|
+
// It will return a default tag in such a case. This default tag is not module specific.
|
|
27
|
+
const tag = sourceConfig.tag ?? DEFAULT_TAG;
|
|
28
|
+
const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
|
|
29
|
+
const status = await systemStorage?.getStatus();
|
|
30
|
+
let replication_lag_bytes = undefined;
|
|
31
|
+
let tables_flat = [];
|
|
32
|
+
if (check_connection) {
|
|
33
|
+
const source_table_patterns = rules.getSourceTables();
|
|
34
|
+
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
|
|
35
|
+
tables_flat = resolved_tables.flatMap((info) => {
|
|
36
|
+
if (info.table) {
|
|
37
|
+
return [info.table];
|
|
38
|
+
}
|
|
39
|
+
else if (info.tables) {
|
|
40
|
+
return info.tables;
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
return [];
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
if (systemStorage) {
|
|
47
|
+
try {
|
|
48
|
+
replication_lag_bytes = await apiHandler.getReplicationLag({
|
|
49
|
+
bucketStorage: systemStorage
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
catch (e) {
|
|
53
|
+
// Ignore
|
|
54
|
+
logger.warn(`Unable to get replication lag`, e);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
177
57
|
}
|
|
178
|
-
|
|
179
|
-
|
|
58
|
+
else {
|
|
59
|
+
const source_table_patterns = rules.getSourceTables();
|
|
60
|
+
tables_flat = source_table_patterns.map((pattern) => {
|
|
61
|
+
if (pattern.isWildcard) {
|
|
62
|
+
return {
|
|
63
|
+
schema: pattern.schema,
|
|
64
|
+
name: pattern.tablePrefix,
|
|
65
|
+
pattern: pattern.isWildcard ? pattern.tablePattern : undefined,
|
|
66
|
+
data_queries: false,
|
|
67
|
+
parameter_queries: false,
|
|
68
|
+
replication_id: [],
|
|
69
|
+
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
const source = {
|
|
74
|
+
connectionTag: tag,
|
|
75
|
+
schema: pattern.schema,
|
|
76
|
+
table: pattern.tablePattern
|
|
77
|
+
};
|
|
78
|
+
const syncData = rules.tableSyncsData(source);
|
|
79
|
+
const syncParameters = rules.tableSyncsParameters(source);
|
|
80
|
+
return {
|
|
81
|
+
schema: pattern.schema,
|
|
82
|
+
name: pattern.name,
|
|
83
|
+
data_queries: syncData,
|
|
84
|
+
parameter_queries: syncParameters,
|
|
85
|
+
replication_id: [],
|
|
86
|
+
errors: [{ level: 'fatal', message: 'connection failed' }]
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
});
|
|
180
90
|
}
|
|
91
|
+
const errors = tables_flat.flatMap((info) => info.errors);
|
|
92
|
+
if (sync_rules.last_fatal_error) {
|
|
93
|
+
errors.push({ level: 'fatal', message: sync_rules.last_fatal_error });
|
|
94
|
+
}
|
|
95
|
+
errors.push(...rules.errors.map((e) => {
|
|
96
|
+
return {
|
|
97
|
+
level: e.type,
|
|
98
|
+
message: e.message
|
|
99
|
+
};
|
|
100
|
+
}));
|
|
101
|
+
return {
|
|
102
|
+
content: include_content ? sync_rules.sync_rules_content : undefined,
|
|
103
|
+
connections: [
|
|
104
|
+
{
|
|
105
|
+
id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
|
|
106
|
+
tag: tag,
|
|
107
|
+
slot_name: sync_rules.slot_name,
|
|
108
|
+
initial_replication_done: status?.snapshot_done ?? false,
|
|
109
|
+
// TODO: Rename?
|
|
110
|
+
last_lsn: status?.checkpoint_lsn ?? undefined,
|
|
111
|
+
last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
|
|
112
|
+
last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
|
|
113
|
+
replication_lag_bytes: replication_lag_bytes,
|
|
114
|
+
tables: tables_flat
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
errors: deduplicate(errors)
|
|
118
|
+
};
|
|
181
119
|
}
|
|
182
120
|
function deduplicate(errors) {
|
|
183
121
|
let seen = new Set();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAsC,MAAM,+BAA+B,CAAC;AAyBhG,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,aAA2C,EAC3C,UAAoB,EACpB,UAAoD,EACpD,OAA2B;IAE3B,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IACzD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IACjD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,KAAK,CAAC;IAE3D,IAAI,KAAmB,CAAC;IACxB,IAAI,SAAqC,CAAC;IAC1C,IAAI,CAAC;QACH,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACrE,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC;IAC/B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACpE,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;SACjD,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;IACxD,gFAAgF;IAChF,wFAAwF;IACxF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,WAAW,CAAC;IAC5C,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtF,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,SAAS,EAAE,CAAC;IAChD,IAAI,qBAAqB,GAAuB,SAAS,CAAC;IAE1D,IAAI,WAAW,GAAgB,EAAE,CAAC;IAElC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC1F,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC7C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC,MAAM,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,qBAAqB,GAAG,MAAM,UAAU,CAAC,iBAAiB,CAAC;oBACzD,aAAa,EAAE,aAAa;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,SAAS;gBACT,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QAEtD,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAa,EAAE;YAC7D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,WAAW;oBACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;oBAE9D,YAAY,EAAE,KAAK;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;iBAC3D,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAyB;oBACnC,aAAa,EAAE,GAAG;oBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,OAAO,CAAC,YAAY;iBAC5B,CAAC;gBACF,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,QAAQ;oBACtB,iBAAiB,EAAE,cAAc;oBACjC,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;iBAC3D,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,CAAC,IAAI,CACT,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,IAAI;YACb,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;QACpE,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,qBAAqB;gBAC5C,GAAG,EAAE,GAAG;gBACR,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,wBAAwB,EAAE,MAAM,EAAE,aAAa,IAAI,KAAK;gBACxD,gBAAgB;gBAChB,QAAQ,EAAE,MAAM,EAAE,cAAc,IAAI,SAAS;gBAC7C,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,EAAE,WAAW,EAAE;gBAChE,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC9D,qBAAqB,EAAE,qBAAqB;gBAC5C,MAAM,EAAE,WAAW;aACpB;SACF;QACD,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAAyD;IAC5E,IAAI,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7B,IAAI,MAAM,GAAsD,EAAE,CAAC;IACnE,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,55 +1,3 @@
|
|
|
1
|
-
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
-
if (value !== null && value !== void 0) {
|
|
3
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
-
var dispose, inner;
|
|
5
|
-
if (async) {
|
|
6
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
-
dispose = value[Symbol.asyncDispose];
|
|
8
|
-
}
|
|
9
|
-
if (dispose === void 0) {
|
|
10
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
-
dispose = value[Symbol.dispose];
|
|
12
|
-
if (async) inner = dispose;
|
|
13
|
-
}
|
|
14
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
-
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
-
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
-
}
|
|
18
|
-
else if (async) {
|
|
19
|
-
env.stack.push({ async: true });
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
22
|
-
};
|
|
23
|
-
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
-
return function (env) {
|
|
25
|
-
function fail(e) {
|
|
26
|
-
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
-
env.hasError = true;
|
|
28
|
-
}
|
|
29
|
-
var r, s = 0;
|
|
30
|
-
function next() {
|
|
31
|
-
while (r = env.stack.pop()) {
|
|
32
|
-
try {
|
|
33
|
-
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
-
if (r.dispose) {
|
|
35
|
-
var result = r.dispose.call(r.value);
|
|
36
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
-
}
|
|
38
|
-
else s |= 1;
|
|
39
|
-
}
|
|
40
|
-
catch (e) {
|
|
41
|
-
fail(e);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
-
if (env.hasError) throw env.error;
|
|
46
|
-
}
|
|
47
|
-
return next();
|
|
48
|
-
};
|
|
49
|
-
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
-
var e = new Error(message);
|
|
51
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
-
});
|
|
53
1
|
import { container, logger } from '@powersync/lib-services-framework';
|
|
54
2
|
import * as v8 from 'v8';
|
|
55
3
|
import * as system from '../../system/system-index.js';
|
|
@@ -87,28 +35,17 @@ export function registerCompactAction(program) {
|
|
|
87
35
|
await moduleManager.initialize(serviceContext);
|
|
88
36
|
logger.info('Connecting to storage...');
|
|
89
37
|
try {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
logger.info('No active instance to compact');
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
const p = __addDisposableResource(env_1, bucketStorage.getInstance(active), false);
|
|
101
|
-
logger.info('Performing compaction...');
|
|
102
|
-
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
|
|
103
|
-
logger.info('Successfully compacted storage.');
|
|
104
|
-
}
|
|
105
|
-
catch (e_1) {
|
|
106
|
-
env_1.error = e_1;
|
|
107
|
-
env_1.hasError = true;
|
|
108
|
-
}
|
|
109
|
-
finally {
|
|
110
|
-
__disposeResources(env_1);
|
|
38
|
+
// Start the storage engine in order to create the appropriate BucketStorage
|
|
39
|
+
await serviceContext.lifeCycleEngine.start();
|
|
40
|
+
const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
|
|
41
|
+
const active = await bucketStorage.getActiveStorage();
|
|
42
|
+
if (active == null) {
|
|
43
|
+
logger.info('No active instance to compact');
|
|
44
|
+
return;
|
|
111
45
|
}
|
|
46
|
+
logger.info('Performing compaction...');
|
|
47
|
+
await active.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
|
|
48
|
+
logger.info('Successfully compacted storage.');
|
|
112
49
|
}
|
|
113
50
|
catch (e) {
|
|
114
51
|
logger.error(`Failed to compact: ${e.toString()}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO;SAC3B,OAAO,CAAC,YAAY,CAAC;SACrB,MAAM,CAAC,yBAAyB,EAAE,4EAA4E,CAAC,CAAC;IAEnH,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,0BAA0B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,IAAI,MAAM,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAElE,8DAA8D;QAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,aAAa,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,4EAA4E;YAC5E,MAAM,cAAc,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,aAAa,CAAC,mBAAmB,CAAC;YAEvE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;gBAC7C,OAAO;YACT,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YACxC,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,uBAAuB,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;YAC1F,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACnD,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ServiceContextContainer } from '../system/ServiceContext.js';
|
|
2
2
|
import winston from 'winston';
|
|
3
|
-
import { PersistedSyncRulesContent } from '../storage/
|
|
3
|
+
import { PersistedSyncRulesContent } from '../storage/storage-index.js';
|
|
4
4
|
export interface TearDownOptions {
|
|
5
5
|
/**
|
|
6
6
|
* If required, tear down any configuration/state for the specific sync rules
|
|
@@ -1,55 +1,3 @@
|
|
|
1
|
-
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
-
if (value !== null && value !== void 0) {
|
|
3
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
-
var dispose, inner;
|
|
5
|
-
if (async) {
|
|
6
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
-
dispose = value[Symbol.asyncDispose];
|
|
8
|
-
}
|
|
9
|
-
if (dispose === void 0) {
|
|
10
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
-
dispose = value[Symbol.dispose];
|
|
12
|
-
if (async) inner = dispose;
|
|
13
|
-
}
|
|
14
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
-
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
-
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
-
}
|
|
18
|
-
else if (async) {
|
|
19
|
-
env.stack.push({ async: true });
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
22
|
-
};
|
|
23
|
-
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
-
return function (env) {
|
|
25
|
-
function fail(e) {
|
|
26
|
-
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
-
env.hasError = true;
|
|
28
|
-
}
|
|
29
|
-
var r, s = 0;
|
|
30
|
-
function next() {
|
|
31
|
-
while (r = env.stack.pop()) {
|
|
32
|
-
try {
|
|
33
|
-
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
-
if (r.dispose) {
|
|
35
|
-
var result = r.dispose.call(r.value);
|
|
36
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
-
}
|
|
38
|
-
else s |= 1;
|
|
39
|
-
}
|
|
40
|
-
catch (e) {
|
|
41
|
-
fail(e);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
-
if (env.hasError) throw env.error;
|
|
46
|
-
}
|
|
47
|
-
return next();
|
|
48
|
-
};
|
|
49
|
-
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
-
var e = new Error(message);
|
|
51
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
-
});
|
|
53
1
|
import { container, logger } from '@powersync/lib-services-framework';
|
|
54
2
|
import { hrtime } from 'node:process';
|
|
55
3
|
// 5 minutes
|
|
@@ -203,35 +151,24 @@ export class AbstractReplicator {
|
|
|
203
151
|
}
|
|
204
152
|
}
|
|
205
153
|
this.replicationJobs = newJobs;
|
|
206
|
-
//
|
|
154
|
+
// Stop any orphaned jobs that no longer have sync rules.
|
|
155
|
+
// Termination happens below
|
|
207
156
|
for (let job of existingJobs.values()) {
|
|
208
157
|
// Old - stop and clean up
|
|
209
158
|
try {
|
|
210
159
|
await job.stop();
|
|
211
|
-
await this.terminateSyncRules(job.storage);
|
|
212
|
-
job.storage[Symbol.dispose]();
|
|
213
160
|
}
|
|
214
161
|
catch (e) {
|
|
215
162
|
// This will be retried
|
|
216
|
-
this.logger.warn('Failed to
|
|
163
|
+
this.logger.warn('Failed to stop old replication job}', e);
|
|
217
164
|
}
|
|
218
165
|
}
|
|
219
166
|
// Sync rules stopped previously or by a different process.
|
|
220
167
|
const stopped = await this.storage.getStoppedSyncRules();
|
|
221
168
|
for (let syncRules of stopped) {
|
|
222
169
|
try {
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
const syncRuleStorage = __addDisposableResource(env_1, this.storage.getInstance(syncRules), false);
|
|
226
|
-
await this.terminateSyncRules(syncRuleStorage);
|
|
227
|
-
}
|
|
228
|
-
catch (e_1) {
|
|
229
|
-
env_1.error = e_1;
|
|
230
|
-
env_1.hasError = true;
|
|
231
|
-
}
|
|
232
|
-
finally {
|
|
233
|
-
__disposeResources(env_1);
|
|
234
|
-
}
|
|
170
|
+
const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
|
|
171
|
+
await this.terminateSyncRules(syncRuleStorage);
|
|
235
172
|
}
|
|
236
173
|
catch (e) {
|
|
237
174
|
this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
|
|
@@ -243,14 +180,9 @@ export class AbstractReplicator {
|
|
|
243
180
|
}
|
|
244
181
|
async terminateSyncRules(syncRuleStorage) {
|
|
245
182
|
this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
|
|
250
|
-
}
|
|
251
|
-
catch (e) {
|
|
252
|
-
this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
|
|
253
|
-
}
|
|
183
|
+
await this.cleanUp(syncRuleStorage);
|
|
184
|
+
await syncRuleStorage.terminate();
|
|
185
|
+
this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
|
|
254
186
|
}
|
|
255
187
|
}
|
|
256
188
|
//# sourceMappingURL=AbstractReplicator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAStC,YAAY;AACZ,MAAM,aAAa,GAAG,WAAc,GAAG,IAAI,CAAC;AAiB5C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IAaR;IAZpB,MAAM,CAAiB;IACjC;;;;OAIG;IACK,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IACvC,OAAO,GAAG,KAAK,CAAC;IAExB,wDAAwD;IAChD,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAEnC,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAAC;YAC5D,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QAEpD,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACtC,IAAI,CAAC;gBACH,kDAAkD;gBAClD,2GAA2G;gBAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACrD,OAAO,EAAE,SAAS;oBAClB,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW;iBAC5C,CAAC,CAAC;gBACH,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,4BAA4B;gBAC5B,4EAA4E;gBAC5E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;gBACvE,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;gBACxD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,CAAC;wBACzC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;4BACtD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;wBAC9B,CAAC;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,eAAe,CAAC;QAE9C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,KAAK,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1C,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAChD,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,IAAI,CAAC;oBACH,IAAI,IAA6B,CAAC;oBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;wBAClD,IAAI,GAAG,cAAc,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC5B,IAAI,EAAE,IAAI;wBACV,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,qCAAqC;oBACrC,uDAAuD;oBACvD,gDAAgD;oBAChD,gFAAgF;oBAChF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAE/B,yDAAyD;QACzD,4BAA4B;QAC5B,KAAK,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,0BAA0B;YAC1B,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,uBAAuB;gBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QACzD,KAAK,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1F,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IAES,WAAW,CAAC,UAAkB;QACtC,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAA+C;QAChF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,eAAe,CAAC,QAAQ,KAAK,CAAC,CAAC;QAC3E,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACpC,MAAM,eAAe,CAAC,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtF,CAAC;CAGF"}
|
|
@@ -21,8 +21,9 @@ export const writeCheckpoint = routeDefinition({
|
|
|
21
21
|
const timeout = 50_000;
|
|
22
22
|
logger.info(`Waiting for LSN checkpoint: ${head}`);
|
|
23
23
|
while (Date.now() - start < timeout) {
|
|
24
|
-
const
|
|
25
|
-
|
|
24
|
+
const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
|
|
25
|
+
const cp = await bucketStorage?.getCheckpoint();
|
|
26
|
+
if (cp == null) {
|
|
26
27
|
throw new Error('No sync rules available');
|
|
27
28
|
}
|
|
28
29
|
if (cp.lsn && cp.lsn >= head) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"checkpointing.js","sourceRoot":"","sources":["../../../src/routes/endpoints/checkpointing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"checkpointing.js","sourceRoot":"","sources":["../../../src/routes/endpoints/checkpointing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,eAAe,CAAC;IAC7C,IAAI,EAAE,wBAAwB;IAC9B,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG;IAC7B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,sBAAsB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC3F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EACJ,OAAO,EAAE,EAAE,eAAe,EAAE,EAC7B,GAAG,OAAO,CAAC;QACZ,MAAM,UAAU,GAAG,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;QAE1D,gDAAgD;QAChD,wEAAwE;QACxE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,qBAAqB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,+BAA+B,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE,CAAC;YACpC,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,CAAC;YACjG,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,aAAa,EAAE,CAAC;YAChD,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YACD,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;gBAChE,OAAO,EAAE,UAAU,EAAE,EAAE,CAAC,UAAU,EAAE,CAAC;YACvC,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,eAAe,CAAC;IAC9C,IAAI,EAAE,yBAAyB;IAC/B,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG;IAC7B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,sBAAsB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC3F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAErD,MAAM,UAAU,GAAG,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;QAE1D,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;YAC5E,MAAM,EAAE,OAAO;YACf,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS;YAClC,GAAG,EAAE,UAAU;YACf,OAAO,EAAE,eAAe,CAAC,aAAa,CAAC,mBAAmB;SAC3D,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,KAAK,eAAe,MAAM,eAAe,EAAE,CAAC,CAAC;QAEpH,OAAO;YACL,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC;SAC1C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC"}
|