@powersync/service-module-postgres-storage 0.0.0-dev-20250116115804

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/LICENSE +67 -0
  3. package/README.md +67 -0
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/@types/index.d.ts +7 -0
  6. package/dist/@types/migrations/PostgresMigrationAgent.d.ts +12 -0
  7. package/dist/@types/migrations/PostgresMigrationStore.d.ts +14 -0
  8. package/dist/@types/migrations/migration-utils.d.ts +3 -0
  9. package/dist/@types/migrations/scripts/1684951997326-init.d.ts +3 -0
  10. package/dist/@types/module/PostgresStorageModule.d.ts +6 -0
  11. package/dist/@types/storage/PostgresBucketStorageFactory.d.ts +42 -0
  12. package/dist/@types/storage/PostgresCompactor.d.ts +40 -0
  13. package/dist/@types/storage/PostgresStorageProvider.d.ts +5 -0
  14. package/dist/@types/storage/PostgresSyncRulesStorage.d.ts +46 -0
  15. package/dist/@types/storage/PostgresTestStorageFactoryGenerator.d.ts +13 -0
  16. package/dist/@types/storage/batch/OperationBatch.d.ts +47 -0
  17. package/dist/@types/storage/batch/PostgresBucketBatch.d.ts +90 -0
  18. package/dist/@types/storage/batch/PostgresPersistedBatch.d.ts +64 -0
  19. package/dist/@types/storage/checkpoints/PostgresWriteCheckpointAPI.d.ts +20 -0
  20. package/dist/@types/storage/storage-index.d.ts +5 -0
  21. package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +17 -0
  22. package/dist/@types/types/codecs.d.ts +61 -0
  23. package/dist/@types/types/models/ActiveCheckpoint.d.ts +12 -0
  24. package/dist/@types/types/models/ActiveCheckpointNotification.d.ts +19 -0
  25. package/dist/@types/types/models/BucketData.d.ts +22 -0
  26. package/dist/@types/types/models/BucketParameters.d.ts +11 -0
  27. package/dist/@types/types/models/CurrentData.d.ts +22 -0
  28. package/dist/@types/types/models/Instance.d.ts +6 -0
  29. package/dist/@types/types/models/Migration.d.ts +12 -0
  30. package/dist/@types/types/models/SourceTable.d.ts +31 -0
  31. package/dist/@types/types/models/SyncRules.d.ts +47 -0
  32. package/dist/@types/types/models/WriteCheckpoint.d.ts +15 -0
  33. package/dist/@types/types/models/models-index.d.ts +10 -0
  34. package/dist/@types/types/types.d.ts +96 -0
  35. package/dist/@types/utils/bson.d.ts +6 -0
  36. package/dist/@types/utils/bucket-data.d.ts +18 -0
  37. package/dist/@types/utils/db.d.ts +8 -0
  38. package/dist/@types/utils/ts-codec.d.ts +5 -0
  39. package/dist/@types/utils/utils-index.d.ts +4 -0
  40. package/dist/index.js +8 -0
  41. package/dist/index.js.map +1 -0
  42. package/dist/migrations/PostgresMigrationAgent.js +36 -0
  43. package/dist/migrations/PostgresMigrationAgent.js.map +1 -0
  44. package/dist/migrations/PostgresMigrationStore.js +60 -0
  45. package/dist/migrations/PostgresMigrationStore.js.map +1 -0
  46. package/dist/migrations/migration-utils.js +13 -0
  47. package/dist/migrations/migration-utils.js.map +1 -0
  48. package/dist/migrations/scripts/1684951997326-init.js +196 -0
  49. package/dist/migrations/scripts/1684951997326-init.js.map +1 -0
  50. package/dist/module/PostgresStorageModule.js +23 -0
  51. package/dist/module/PostgresStorageModule.js.map +1 -0
  52. package/dist/storage/PostgresBucketStorageFactory.js +433 -0
  53. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -0
  54. package/dist/storage/PostgresCompactor.js +298 -0
  55. package/dist/storage/PostgresCompactor.js.map +1 -0
  56. package/dist/storage/PostgresStorageProvider.js +35 -0
  57. package/dist/storage/PostgresStorageProvider.js.map +1 -0
  58. package/dist/storage/PostgresSyncRulesStorage.js +619 -0
  59. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -0
  60. package/dist/storage/PostgresTestStorageFactoryGenerator.js +110 -0
  61. package/dist/storage/PostgresTestStorageFactoryGenerator.js.map +1 -0
  62. package/dist/storage/batch/OperationBatch.js +93 -0
  63. package/dist/storage/batch/OperationBatch.js.map +1 -0
  64. package/dist/storage/batch/PostgresBucketBatch.js +732 -0
  65. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -0
  66. package/dist/storage/batch/PostgresPersistedBatch.js +367 -0
  67. package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -0
  68. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +148 -0
  69. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -0
  70. package/dist/storage/storage-index.js +6 -0
  71. package/dist/storage/storage-index.js.map +1 -0
  72. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +58 -0
  73. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +1 -0
  74. package/dist/types/codecs.js +97 -0
  75. package/dist/types/codecs.js.map +1 -0
  76. package/dist/types/models/ActiveCheckpoint.js +12 -0
  77. package/dist/types/models/ActiveCheckpoint.js.map +1 -0
  78. package/dist/types/models/ActiveCheckpointNotification.js +8 -0
  79. package/dist/types/models/ActiveCheckpointNotification.js.map +1 -0
  80. package/dist/types/models/BucketData.js +23 -0
  81. package/dist/types/models/BucketData.js.map +1 -0
  82. package/dist/types/models/BucketParameters.js +11 -0
  83. package/dist/types/models/BucketParameters.js.map +1 -0
  84. package/dist/types/models/CurrentData.js +16 -0
  85. package/dist/types/models/CurrentData.js.map +1 -0
  86. package/dist/types/models/Instance.js +5 -0
  87. package/dist/types/models/Instance.js.map +1 -0
  88. package/dist/types/models/Migration.js +12 -0
  89. package/dist/types/models/Migration.js.map +1 -0
  90. package/dist/types/models/SourceTable.js +24 -0
  91. package/dist/types/models/SourceTable.js.map +1 -0
  92. package/dist/types/models/SyncRules.js +47 -0
  93. package/dist/types/models/SyncRules.js.map +1 -0
  94. package/dist/types/models/WriteCheckpoint.js +13 -0
  95. package/dist/types/models/WriteCheckpoint.js.map +1 -0
  96. package/dist/types/models/models-index.js +11 -0
  97. package/dist/types/models/models-index.js.map +1 -0
  98. package/dist/types/types.js +46 -0
  99. package/dist/types/types.js.map +1 -0
  100. package/dist/utils/bson.js +16 -0
  101. package/dist/utils/bson.js.map +1 -0
  102. package/dist/utils/bucket-data.js +25 -0
  103. package/dist/utils/bucket-data.js.map +1 -0
  104. package/dist/utils/db.js +24 -0
  105. package/dist/utils/db.js.map +1 -0
  106. package/dist/utils/ts-codec.js +11 -0
  107. package/dist/utils/ts-codec.js.map +1 -0
  108. package/dist/utils/utils-index.js +5 -0
  109. package/dist/utils/utils-index.js.map +1 -0
  110. package/package.json +50 -0
  111. package/src/index.ts +10 -0
  112. package/src/migrations/PostgresMigrationAgent.ts +46 -0
  113. package/src/migrations/PostgresMigrationStore.ts +70 -0
  114. package/src/migrations/migration-utils.ts +14 -0
  115. package/src/migrations/scripts/1684951997326-init.ts +141 -0
  116. package/src/module/PostgresStorageModule.ts +30 -0
  117. package/src/storage/PostgresBucketStorageFactory.ts +496 -0
  118. package/src/storage/PostgresCompactor.ts +366 -0
  119. package/src/storage/PostgresStorageProvider.ts +42 -0
  120. package/src/storage/PostgresSyncRulesStorage.ts +666 -0
  121. package/src/storage/PostgresTestStorageFactoryGenerator.ts +61 -0
  122. package/src/storage/batch/OperationBatch.ts +101 -0
  123. package/src/storage/batch/PostgresBucketBatch.ts +885 -0
  124. package/src/storage/batch/PostgresPersistedBatch.ts +441 -0
  125. package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +176 -0
  126. package/src/storage/storage-index.ts +5 -0
  127. package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +67 -0
  128. package/src/types/codecs.ts +136 -0
  129. package/src/types/models/ActiveCheckpoint.ts +15 -0
  130. package/src/types/models/ActiveCheckpointNotification.ts +14 -0
  131. package/src/types/models/BucketData.ts +26 -0
  132. package/src/types/models/BucketParameters.ts +14 -0
  133. package/src/types/models/CurrentData.ts +23 -0
  134. package/src/types/models/Instance.ts +8 -0
  135. package/src/types/models/Migration.ts +19 -0
  136. package/src/types/models/SourceTable.ts +32 -0
  137. package/src/types/models/SyncRules.ts +50 -0
  138. package/src/types/models/WriteCheckpoint.ts +20 -0
  139. package/src/types/models/models-index.ts +10 -0
  140. package/src/types/types.ts +73 -0
  141. package/src/utils/bson.ts +17 -0
  142. package/src/utils/bucket-data.ts +25 -0
  143. package/src/utils/db.ts +27 -0
  144. package/src/utils/ts-codec.ts +14 -0
  145. package/src/utils/utils-index.ts +4 -0
  146. package/test/src/__snapshots__/storage.test.ts.snap +9 -0
  147. package/test/src/__snapshots__/storage_sync.test.ts.snap +332 -0
  148. package/test/src/env.ts +6 -0
  149. package/test/src/migrations.test.ts +34 -0
  150. package/test/src/setup.ts +16 -0
  151. package/test/src/storage.test.ts +131 -0
  152. package/test/src/storage_compacting.test.ts +5 -0
  153. package/test/src/storage_sync.test.ts +12 -0
  154. package/test/src/util.ts +34 -0
  155. package/test/tsconfig.json +20 -0
  156. package/tsconfig.json +36 -0
  157. package/vitest.config.ts +13 -0
@@ -0,0 +1,619 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
53
+ import { DisposableObserver } from '@powersync/lib-services-framework';
54
+ import { storage, utils } from '@powersync/service-core';
55
+ import { JSONBig } from '@powersync/service-jsonbig';
56
+ import * as uuid from 'uuid';
57
+ import { BIGINT_MAX } from '../types/codecs.js';
58
+ import { models } from '../types/types.js';
59
+ import { replicaIdToSubkey } from '../utils/bson.js';
60
+ import { mapOpEntry } from '../utils/bucket-data.js';
61
+ import { pick } from '../utils/ts-codec.js';
62
+ import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js';
63
+ import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js';
64
+ import { PostgresCompactor } from './PostgresCompactor.js';
65
+ export class PostgresSyncRulesStorage extends DisposableObserver {
66
+ options;
67
+ group_id;
68
+ sync_rules;
69
+ slot_name;
70
+ factory;
71
+ db;
72
+ writeCheckpointAPI;
73
+ // TODO we might be able to share this in an abstract class
74
+ parsedSyncRulesCache;
75
+ checksumCache = new storage.ChecksumCache({
76
+ fetchChecksums: (batch) => {
77
+ return this.getChecksumsInternal(batch);
78
+ }
79
+ });
80
+ constructor(options) {
81
+ super();
82
+ this.options = options;
83
+ this.group_id = options.sync_rules.id;
84
+ this.db = options.db;
85
+ this.sync_rules = options.sync_rules;
86
+ this.slot_name = options.sync_rules.slot_name;
87
+ this.factory = options.factory;
88
+ this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
89
+ db: this.db,
90
+ mode: options.write_checkpoint_mode ?? storage.WriteCheckpointMode.MANAGED
91
+ });
92
+ }
93
+ get writeCheckpointMode() {
94
+ return this.writeCheckpointAPI.writeCheckpointMode;
95
+ }
96
+ // TODO we might be able to share this in an abstract class
97
+ getParsedSyncRules(options) {
98
+ const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
99
+ /**
100
+ * Check if the cached sync rules, if present, had the same options.
101
+ * Parse sync rules if the options are different or if there is no cached value.
102
+ */
103
+ if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
104
+ this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).sync_rules, options };
105
+ }
106
+ return this.parsedSyncRulesCache.parsed;
107
+ }
108
+ async reportError(e) {
109
+ const message = String(e.message ?? 'Replication failure');
110
+ await this.db.sql `
111
+ UPDATE sync_rules
112
+ SET
113
+ last_fatal_error = ${{ type: 'varchar', value: message }}
114
+ WHERE
115
+ id = ${{ type: 'int4', value: this.group_id }};
116
+ `.execute();
117
+ }
118
+ compact(options) {
119
+ return new PostgresCompactor(this.db, this.group_id, options).compact();
120
+ }
121
+ batchCreateCustomWriteCheckpoints(checkpoints) {
122
+ return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints.map((c) => ({ ...c, sync_rules_id: this.group_id })));
123
+ }
124
+ createCustomWriteCheckpoint(checkpoint) {
125
+ return this.writeCheckpointAPI.createCustomWriteCheckpoint({
126
+ ...checkpoint,
127
+ sync_rules_id: this.group_id
128
+ });
129
+ }
130
+ lastWriteCheckpoint(filters) {
131
+ return this.writeCheckpointAPI.lastWriteCheckpoint({
132
+ ...filters,
133
+ sync_rules_id: this.group_id
134
+ });
135
+ }
136
+ setWriteCheckpointMode(mode) {
137
+ return this.writeCheckpointAPI.setWriteCheckpointMode(mode);
138
+ }
139
+ createManagedWriteCheckpoint(checkpoint) {
140
+ return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
141
+ }
142
+ async getCheckpoint() {
143
+ const checkpointRow = await this.db.sql `
144
+ SELECT
145
+ last_checkpoint,
146
+ last_checkpoint_lsn
147
+ FROM
148
+ sync_rules
149
+ WHERE
150
+ id = ${{ type: 'int4', value: this.group_id }}
151
+ `
152
+ .decoded(pick(models.SyncRules, ['last_checkpoint', 'last_checkpoint_lsn']))
153
+ .first();
154
+ return {
155
+ checkpoint: utils.timestampToOpId(checkpointRow?.last_checkpoint ?? 0n),
156
+ lsn: checkpointRow?.last_checkpoint_lsn ?? null
157
+ };
158
+ }
159
+ async resolveTable(options) {
160
+ const { group_id, connection_id, connection_tag, entity_descriptor } = options;
161
+ const { schema, name: table, objectId, replicationColumns } = entity_descriptor;
162
+ const columns = replicationColumns.map((column) => ({
163
+ name: column.name,
164
+ type: column.type,
165
+ // The PGWire returns this as a BigInt. We want to store this as JSONB
166
+ type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
167
+ }));
168
+ return this.db.transaction(async (db) => {
169
+ let sourceTableRow = await db.sql `
170
+ SELECT
171
+ *
172
+ FROM
173
+ source_tables
174
+ WHERE
175
+ group_id = ${{ type: 'int4', value: group_id }}
176
+ AND connection_id = ${{ type: 'int4', value: connection_id }}
177
+ AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } }}
178
+ AND schema_name = ${{ type: 'varchar', value: schema }}
179
+ AND table_name = ${{ type: 'varchar', value: table }}
180
+ AND replica_id_columns = ${{ type: 'jsonb', value: columns }}
181
+ `
182
+ .decoded(models.SourceTable)
183
+ .first();
184
+ if (sourceTableRow == null) {
185
+ const row = await db.sql `
186
+ INSERT INTO
187
+ source_tables (
188
+ id,
189
+ group_id,
190
+ connection_id,
191
+ relation_id,
192
+ schema_name,
193
+ table_name,
194
+ replica_id_columns
195
+ )
196
+ VALUES
197
+ (
198
+ ${{ type: 'varchar', value: uuid.v4() }},
199
+ ${{ type: 'int4', value: group_id }},
200
+ ${{ type: 'int4', value: connection_id }},
201
+ --- The objectId can be string | number, we store it as jsonb value
202
+ ${{ type: 'jsonb', value: { object_id: objectId } }},
203
+ ${{ type: 'varchar', value: schema }},
204
+ ${{ type: 'varchar', value: table }},
205
+ ${{ type: 'jsonb', value: columns }}
206
+ )
207
+ RETURNING
208
+ *
209
+ `
210
+ .decoded(models.SourceTable)
211
+ .first();
212
+ sourceTableRow = row;
213
+ }
214
+ const sourceTable = new storage.SourceTable(sourceTableRow.id, connection_tag, objectId, schema, table, replicationColumns, sourceTableRow.snapshot_done ?? true);
215
+ sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable);
216
+ sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable);
217
+ sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable);
218
+ const truncatedTables = await db.sql `
219
+ SELECT
220
+ *
221
+ FROM
222
+ source_tables
223
+ WHERE
224
+ group_id = ${{ type: 'int4', value: group_id }}
225
+ AND connection_id = ${{ type: 'int4', value: connection_id }}
226
+ AND id != ${{ type: 'varchar', value: sourceTableRow.id }}
227
+ AND (
228
+ relation_id = ${{ type: 'jsonb', value: { object_id: objectId } }}
229
+ OR (
230
+ schema_name = ${{ type: 'varchar', value: schema }}
231
+ AND table_name = ${{ type: 'varchar', value: table }}
232
+ )
233
+ )
234
+ `
235
+ .decoded(models.SourceTable)
236
+ .rows();
237
+ return {
238
+ table: sourceTable,
239
+ dropTables: truncatedTables.map((doc) => new storage.SourceTable(doc.id, connection_tag, doc.relation_id?.object_id ?? 0, doc.schema_name, doc.table_name, doc.replica_id_columns?.map((c) => ({
240
+ name: c.name,
241
+ typeOid: c.typeId,
242
+ type: c.type
243
+ })) ?? [], doc.snapshot_done ?? true))
244
+ };
245
+ });
246
+ }
247
+ async startBatch(options, callback) {
248
+ const env_1 = { stack: [], error: void 0, hasError: false };
249
+ try {
250
+ const syncRules = await this.db.sql `
251
+ SELECT
252
+ last_checkpoint_lsn,
253
+ no_checkpoint_before,
254
+ keepalive_op
255
+ FROM
256
+ sync_rules
257
+ WHERE
258
+ id = ${{ type: 'int4', value: this.group_id }}
259
+ `
260
+ .decoded(pick(models.SyncRules, ['last_checkpoint_lsn', 'no_checkpoint_before', 'keepalive_op']))
261
+ .first();
262
+ const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null;
263
+ const batch = __addDisposableResource(env_1, new PostgresBucketBatch({
264
+ db: this.db,
265
+ sync_rules: this.sync_rules.parsed(options).sync_rules,
266
+ group_id: this.group_id,
267
+ slot_name: this.slot_name,
268
+ last_checkpoint_lsn: checkpoint_lsn,
269
+ keep_alive_op: syncRules?.keepalive_op,
270
+ no_checkpoint_before_lsn: syncRules?.no_checkpoint_before ?? options.zeroLSN,
271
+ store_current_data: options.storeCurrentData,
272
+ skip_existing_rows: options.skipExistingRows ?? false,
273
+ batch_limits: this.options.batchLimits
274
+ }), true);
275
+ this.iterateListeners((cb) => cb.batchStarted?.(batch));
276
+ await callback(batch);
277
+ await batch.flush();
278
+ if (batch.last_flushed_op) {
279
+ return { flushed_op: String(batch.last_flushed_op) };
280
+ }
281
+ else {
282
+ return null;
283
+ }
284
+ }
285
+ catch (e_1) {
286
+ env_1.error = e_1;
287
+ env_1.hasError = true;
288
+ }
289
+ finally {
290
+ const result_1 = __disposeResources(env_1);
291
+ if (result_1)
292
+ await result_1;
293
+ }
294
+ }
295
+ async getParameterSets(checkpoint, lookups) {
296
+ const rows = await this.db.sql `
297
+ SELECT DISTINCT
298
+ ON (lookup, source_table, source_key) lookup,
299
+ source_table,
300
+ source_key,
301
+ id,
302
+ bucket_parameters
303
+ FROM
304
+ bucket_parameters
305
+ WHERE
306
+ group_id = ${{ type: 'int4', value: this.group_id }}
307
+ AND lookup = ANY (
308
+ SELECT
309
+ decode((FILTER ->> 0)::text, 'hex') -- Decode the hex string to bytea
310
+ FROM
311
+ jsonb_array_elements(${{
312
+ type: 'jsonb',
313
+ value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex'))
314
+ }}) AS FILTER
315
+ )
316
+ AND id <= ${{ type: 'int8', value: BigInt(checkpoint) }}
317
+ ORDER BY
318
+ lookup,
319
+ source_table,
320
+ source_key,
321
+ id DESC
322
+ `
323
+ .decoded(pick(models.BucketParameters, ['bucket_parameters']))
324
+ .rows();
325
+ const groupedParameters = rows.map((row) => {
326
+ return JSONBig.parse(row.bucket_parameters);
327
+ });
328
+ return groupedParameters.flat();
329
+ }
330
+ async *getBucketDataBatch(checkpoint, dataBuckets, options) {
331
+ if (dataBuckets.size == 0) {
332
+ return;
333
+ }
334
+ const end = checkpoint ?? BIGINT_MAX;
335
+ const filters = Array.from(dataBuckets.entries()).map(([name, start]) => ({
336
+ bucket_name: name,
337
+ start: start
338
+ }));
339
+ const rowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
340
+ const sizeLimit = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
341
+ let batchSize = 0;
342
+ let currentBatch = null;
343
+ let targetOp = null;
344
+ let rowCount = 0;
345
+ /**
346
+ * It is possible to perform this query with JSONB join. e.g.
347
+ * ```sql
348
+ * WITH
349
+ * filter_data AS (
350
+ * SELECT
351
+ * FILTER ->> 'bucket_name' AS bucket_name,
352
+ * (FILTER ->> 'start')::BIGINT AS start_op_id
353
+ * FROM
354
+ * jsonb_array_elements($1::jsonb) AS FILTER
355
+ * )
356
+ * SELECT
357
+ * b.*,
358
+ * octet_length(b.data) AS data_size
359
+ * FROM
360
+ * bucket_data b
361
+ * JOIN filter_data f ON b.bucket_name = f.bucket_name
362
+ * AND b.op_id > f.start_op_id
363
+ * AND b.op_id <= $2
364
+ * WHERE
365
+ * b.group_id = $3
366
+ * ORDER BY
367
+ * b.bucket_name ASC,
368
+ * b.op_id ASC
369
+ * LIMIT
370
+ * $4;
371
+ * ```
372
+ * Which might be better for large volumes of buckets, but in testing the JSON method
373
+ * was significantly slower than the method below. Syncing 2.5 million rows in a single
374
+ * bucket takes 2 minutes and 11 seconds with the method below. With the JSON method
375
+ * 1 million rows were only synced before a 5 minute timeout.
376
+ */
377
+ for await (const rows of this.db.streamRows({
378
+ statement: `
379
+ SELECT
380
+ *
381
+ FROM
382
+ bucket_data
383
+ WHERE
384
+ group_id = $1
385
+ and op_id <= $2
386
+ and (
387
+ ${filters.map((f, index) => `(bucket_name = $${index * 2 + 4} and op_id > $${index * 2 + 5})`).join(' OR ')}
388
+ )
389
+ ORDER BY
390
+ bucket_name ASC,
391
+ op_id ASC
392
+ LIMIT
393
+ $3;`,
394
+ params: [
395
+ { type: 'int4', value: this.group_id },
396
+ { type: 'int8', value: end },
397
+ { type: 'int4', value: rowLimit + 1 },
398
+ ...filters.flatMap((f) => [
399
+ { type: 'varchar', value: f.bucket_name },
400
+ { type: 'int8', value: f.start }
401
+ ])
402
+ ]
403
+ })) {
404
+ const decodedRows = rows.map((r) => models.BucketData.decode(r));
405
+ for (const row of decodedRows) {
406
+ const { bucket_name } = row;
407
+ const rowSize = row.data ? row.data.length : 0;
408
+ if (currentBatch == null ||
409
+ currentBatch.bucket != bucket_name ||
410
+ batchSize >= sizeLimit ||
411
+ (currentBatch?.data.length && batchSize + rowSize > sizeLimit) ||
412
+ currentBatch.data.length >= rowLimit) {
413
+ let start = undefined;
414
+ if (currentBatch != null) {
415
+ if (currentBatch.bucket == bucket_name) {
416
+ currentBatch.has_more = true;
417
+ }
418
+ const yieldBatch = currentBatch;
419
+ start = currentBatch.after;
420
+ currentBatch = null;
421
+ batchSize = 0;
422
+ yield { batch: yieldBatch, targetOp: targetOp };
423
+ targetOp = null;
424
+ if (rowCount >= rowLimit) {
425
+ // We've yielded all the requested rows
426
+ break;
427
+ }
428
+ }
429
+ start ??= dataBuckets.get(bucket_name);
430
+ if (start == null) {
431
+ throw new Error(`data for unexpected bucket: ${bucket_name}`);
432
+ }
433
+ currentBatch = {
434
+ bucket: bucket_name,
435
+ after: start,
436
+ has_more: false,
437
+ data: [],
438
+ next_after: start
439
+ };
440
+ targetOp = null;
441
+ }
442
+ const entry = mapOpEntry(row);
443
+ if (row.source_table && row.source_key) {
444
+ entry.subkey = replicaIdToSubkey(row.source_table, storage.deserializeReplicaId(row.source_key));
445
+ }
446
+ if (row.target_op != null) {
447
+ // MOVE, CLEAR
448
+ const rowTargetOp = row.target_op;
449
+ if (targetOp == null || rowTargetOp > targetOp) {
450
+ targetOp = rowTargetOp;
451
+ }
452
+ }
453
+ currentBatch.data.push(entry);
454
+ currentBatch.next_after = entry.op_id;
455
+ batchSize += rowSize;
456
+ // Manually track the total rows yielded
457
+ rowCount++;
458
+ }
459
+ }
460
+ if (currentBatch != null) {
461
+ const yieldBatch = currentBatch;
462
+ currentBatch = null;
463
+ yield { batch: yieldBatch, targetOp: targetOp };
464
+ targetOp = null;
465
+ }
466
+ }
467
+ async getChecksums(checkpoint, buckets) {
468
+ return this.checksumCache.getChecksumMap(checkpoint, buckets);
469
+ }
470
+ async terminate(options) {
471
+ if (!options || options?.clearStorage) {
472
+ await this.clear();
473
+ }
474
+ await this.db.sql `
475
+ UPDATE sync_rules
476
+ SET
477
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }},
478
+ snapshot_done = ${{ type: 'bool', value: false }}
479
+ WHERE
480
+ id = ${{ type: 'int4', value: this.group_id }}
481
+ `.execute();
482
+ }
483
+ async getStatus() {
484
+ const syncRulesRow = await this.db.sql `
485
+ SELECT
486
+ snapshot_done,
487
+ last_checkpoint_lsn,
488
+ state
489
+ FROM
490
+ sync_rules
491
+ WHERE
492
+ id = ${{ type: 'int4', value: this.group_id }}
493
+ `
494
+ .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state']))
495
+ .first();
496
+ if (syncRulesRow == null) {
497
+ throw new Error('Cannot find sync rules status');
498
+ }
499
+ return {
500
+ snapshot_done: syncRulesRow.snapshot_done,
501
+ active: syncRulesRow.state == storage.SyncRuleState.ACTIVE,
502
+ checkpoint_lsn: syncRulesRow.last_checkpoint_lsn ?? null
503
+ };
504
+ }
505
+ async clear() {
506
+ await this.db.sql `
507
+ UPDATE sync_rules
508
+ SET
509
+ snapshot_done = FALSE,
510
+ last_checkpoint_lsn = NULL,
511
+ last_checkpoint = NULL,
512
+ no_checkpoint_before = NULL
513
+ WHERE
514
+ id = ${{ type: 'int4', value: this.group_id }}
515
+ `.execute();
516
+ await this.db.sql `
517
+ DELETE FROM bucket_data
518
+ WHERE
519
+ group_id = ${{ type: 'int4', value: this.group_id }}
520
+ `.execute();
521
+ await this.db.sql `
522
+ DELETE FROM bucket_parameters
523
+ WHERE
524
+ group_id = ${{ type: 'int4', value: this.group_id }}
525
+ `.execute();
526
+ await this.db.sql `
527
+ DELETE FROM current_data
528
+ WHERE
529
+ group_id = ${{ type: 'int4', value: this.group_id }}
530
+ `.execute();
531
+ await this.db.sql `
532
+ DELETE FROM source_tables
533
+ WHERE
534
+ group_id = ${{ type: 'int4', value: this.group_id }}
535
+ `.execute();
536
+ }
537
+ async autoActivate() {
538
+ await this.db.transaction(async (db) => {
539
+ const syncRulesRow = await db.sql `
540
+ SELECT
541
+ state
542
+ FROM
543
+ sync_rules
544
+ WHERE
545
+ id = ${{ type: 'int4', value: this.group_id }}
546
+ `
547
+ .decoded(pick(models.SyncRules, ['state']))
548
+ .first();
549
+ if (syncRulesRow && syncRulesRow.state == storage.SyncRuleState.PROCESSING) {
550
+ await db.sql `
551
+ UPDATE sync_rules
552
+ SET
553
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }}
554
+ WHERE
555
+ id = ${{ type: 'int4', value: this.group_id }}
556
+ `.execute();
557
+ }
558
+ await db.sql `
559
+ UPDATE sync_rules
560
+ SET
561
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }}
562
+ WHERE
563
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }}
564
+ AND id != ${{ type: 'int4', value: this.group_id }}
565
+ `.execute();
566
+ });
567
+ }
568
+ async getChecksumsInternal(batch) {
569
+ if (batch.length == 0) {
570
+ return new Map();
571
+ }
572
+ const rangedBatch = batch.map((b) => ({
573
+ ...b,
574
+ start: b.start ?? 0
575
+ }));
576
+ const results = await this.db.sql `
577
+ WITH
578
+ filter_data AS (
579
+ SELECT
580
+ FILTER ->> 'bucket' AS bucket_name,
581
+ (FILTER ->> 'start')::BIGINT AS start_op_id,
582
+ (FILTER ->> 'end')::BIGINT AS end_op_id
583
+ FROM
584
+ jsonb_array_elements(${{ type: 'jsonb', value: rangedBatch }}::jsonb) AS FILTER
585
+ )
586
+ SELECT
587
+ b.bucket_name AS bucket,
588
+ SUM(b.checksum) AS checksum_total,
589
+ COUNT(*) AS total,
590
+ MAX(
591
+ CASE
592
+ WHEN b.op = 'CLEAR' THEN 1
593
+ ELSE 0
594
+ END
595
+ ) AS has_clear_op
596
+ FROM
597
+ bucket_data b
598
+ JOIN filter_data f ON b.bucket_name = f.bucket_name
599
+ AND b.op_id > f.start_op_id
600
+ AND b.op_id <= f.end_op_id
601
+ WHERE
602
+ b.group_id = ${{ type: 'int4', value: this.group_id }}
603
+ GROUP BY
604
+ b.bucket_name;
605
+ `.rows();
606
+ return new Map(results.map((doc) => {
607
+ return [
608
+ doc.bucket,
609
+ {
610
+ bucket: doc.bucket,
611
+ partialCount: Number(doc.total),
612
+ partialChecksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff,
613
+ isFullChecksum: doc.has_clear_op == 1
614
+ }
615
+ ];
616
+ }));
617
+ }
618
+ }
619
+ //# sourceMappingURL=PostgresSyncRulesStorage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"PostgresSyncRulesStorage.js","sourceRoot":"","sources":["../../src/storage/PostgresSyncRulesStorage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAErD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAgC,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAIrD,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,0BAA0B,EAAE,MAAM,6CAA6C,CAAC;AAEzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAU3D,MAAM,OAAO,wBACX,SAAQ,kBAA0D;IAmB5C;IAhBN,QAAQ,CAAS;IACjB,UAAU,CAAoC;IAC9C,SAAS,CAAS;IAClB,OAAO,CAA+B;IAE5C,EAAE,CAA8B;IAChC,kBAAkB,CAA6B;IAEzD,6DAA6D;IACrD,oBAAoB,CAA0F;IAC9G,aAAa,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC;QAChD,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,OAAO,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;KACF,CAAC,CAAC;IAEH,YAAsB,OAAwC;QAC5D,KAAK,EAAE,CAAC;QADY,YAAO,GAAP,OAAO,CAAiC;QAE5D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACtC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;QAC9C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAE/B,IAAI,CAAC,kBAAkB,GAAG,IAAI,0BAA0B,CAAC;YACvD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,OAAO,CAAC,qBAAqB,IAAI,OAAO,CAAC,mBAAmB,CAAC,OAAO;SAC3E,CAAC,CAAC;IACL,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,kBAAkB,CAAC,mBAAmB,CAAC;IACrD,CAAC;IAED,6DAA6D;IAC7D,kBAAkB,CAAC,OAAsC;QACvD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,oBAAoB,IAAI,EAAE,CAAC;QAC3E;;;WAGG;QACH,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,aAAa,IAAI,aAAa,EAAE,aAAa,EAAE,CAAC;YACrE,IAAI,CAAC,oBAAoB,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC;QAC9F,CAAC;QAED,OAAO,IAAI,CAAC,oBAAqB,CAAC,MAAM,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,CAAM;QACtB,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,qBAAqB,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;6BAGQ,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;;eAEjD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KAChD,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,CAAC,OAAgC;QACtC,OAAO,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;IAC1E,CAAC;IAED,iCAAiC,CAAC,WAA0D;QAC1F,OAAO,IAAI,CAAC,kBAAkB,CAAC,iCAAiC,CAC9D,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CACjE,CAAC;IACJ,CAAC;IAED,2BAA2B,CAAC,UAAuD;QACjF,OAAO,IAAI,CAAC,kBAAkB,CAAC,2BAA2B,CAAC;YACzD,GAAG,UAAU;YACb,aAAa,EAAE,IAAI,CAAC,QAAQ;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB,CAAC,OAAsD;QACxE,OAAO,IAAI,CAAC,kBAAkB,CAAC,mBAAmB,CAAC;YACjD,GAAG,OAAO;YACV,aAAa,EAAE,IAAI,CAAC,QAAQ;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,OAAO,IAAI,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,4BAA4B,CAAC,UAAiD;QAC5E,OAAO,IAAI,CAAC,kBAAkB,CAAC,4BAA4B,CAAC,UAAU,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;eAO5B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KAChD;aACE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,iBAAiB,EAAE,qBAAqB,CAAC,CAAC,CAAC;aAC3E,KAAK,EAAE,CAAC;QAEX,OAAO;YACL,UAAU,EAAE,KAAK,CAAC,eAAe,CAAC,aAAa,EAAE,eAAe,IAAI,EAAE,CAAC;YACvE,GAAG,EAAE,aAAa,EAAE,mBAAmB,IAAI,IAAI;SAChD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAAoC;QACrD,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QAE/E,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,GAAG,iBAAiB,CAAC;QAEhF,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClD,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,sEAAsE;YACtE,QAAQ,EAAE,OAAO,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;SACvF,CAAC,CAAC,CAAC;QACJ,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACtC,IAAI,cAAc,GAAG,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;uBAMhB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;gCACxB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE;8BACxC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,QAAQ,EAA6B,EAAE;8BAC5E,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;6BACnC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE;qCACzB,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE;OAC/D;iBACE,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;iBAC3B,KAAK,EAAE,CAAC;YAEX,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;;gBAahB,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE;gBACrC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;gBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE;;gBAEtC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,QAAQ,EAA6B,EAAE;gBAC5E,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;gBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE;gBACjC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE;;;;SAIxC;qBACE,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;qBAC3B,KAAK,EAAE,CAAC;gBACX,cAAc,GAAG,GAAG,CAAC;YACvB,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,WAAW,CACzC,cAAe,CAAC,EAAE,EAClB,cAAc,EACd,QAAQ,EACR,MAAM,EACN,KAAK,EACL,kBAAkB,EAClB,cAAe,CAAC,aAAa,IAAI,IAAI,CACtC,CAAC;YACF,WAAW,CAAC,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;YAC3E,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YACtE,WAAW,CAAC,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC;YAElF,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;uBAMnB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;gCACxB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE;sBAChD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,cAAe,CAAC,EAAE,EAAE;;4BAExC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,QAAQ,EAA6B,EAAE;;8BAE1E,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;iCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE;;;OAG3D;iBACE,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;iBAC3B,IAAI,EAAE,CAAC;YAEV,OAAO;gBACL,KAAK,EAAE,WAAW;gBAClB,UAAU,EAAE,eAAe,CAAC,GAAG,CAC7B,CAAC,GAAG,EAAE,EAAE,CACN,IAAI,OAAO,CAAC,WAAW,CACrB,GAAG,CAAC,EAAE,EACN,cAAc,EACd,GAAG,CAAC,WAAW,EAAE,SAAS,IAAI,CAAC,EAC/B,GAAG,CAAC,WAAW,EACf,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAClC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,MAAM;oBACjB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC,CAAC,IAAI,EAAE,EACT,GAAG,CAAC,aAAa,IAAI,IAAI,CAC1B,CACJ;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CACd,OAAkC,EAClC,QAA8D;;;YAE9D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;eAQxB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KAChD;iBACE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,cAAc,CAAC,CAAC,CAAC;iBAChG,KAAK,EAAE,CAAC;YAEX,MAAM,cAAc,GAAG,SAAS,EAAE,mBAAmB,IAAI,IAAI,CAAC;YAE9D,MAAY,KAAK,kCAAG,IAAI,mBAAmB,CAAC;gBAC1C,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,UAAU;gBACtD,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,mBAAmB,EAAE,cAAc;gBACnC,aAAa,EAAE,SAAS,EAAE,YAAY;gBACtC,wBAAwB,EAAE,SAAS,EAAE,oBAAoB,IAAI,OAAO,CAAC,OAAO;gBAC5E,kBAAkB,EAAE,OAAO,CAAC,gBAAgB;gBAC5C,kBAAkB,EAAE,OAAO,CAAC,gBAAgB,IAAI,KAAK;gBACrD,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;aACvC,CAAC,OAAA,CAAC;YACH,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YAExD,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAC;YACd,CAAC;;;;;;;;;;;KACF;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAsB,EACtB,OAAuC;QAEvC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;qBAUb,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;;;;;mCAKxB;YAC3B,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC5E;;oBAEa,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE;;;;;;KAM1D;aACE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC;aAC7D,IAAI,EAAE,CAAC;QAEV,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACzC,OAAO,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,CAA6B,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,OAAO,iBAAiB,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,CAAC,kBAAkB,CACvB,UAAsB,EACtB,WAAgC,EAChC,OAAwC;QAExC,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,UAAU,IAAI,UAAU,CAAC;QACrC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YACxE,WAAW,EAAE,IAAI;YACjB,KAAK,EAAE,KAAK;SACb,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,OAAO,EAAE,KAAK,IAAI,OAAO,CAAC,4BAA4B,CAAC;QACxE,MAAM,SAAS,GAAG,OAAO,EAAE,eAAe,IAAI,OAAO,CAAC,kCAAkC,CAAC;QAEzF,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,YAAY,GAAgC,IAAI,CAAC;QACrD,IAAI,QAAQ,GAAkB,IAAI,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WA+BG;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC;YAC1C,SAAS,EAAE;;;;;;;;;cASH,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,mBAAmB,KAAK,GAAG,CAAC,GAAG,CAAC,iBAAiB,KAAK,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;;;;;;gBAMvG;YACV,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;gBACtC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE;gBAC5B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,CAAC,EAAE;gBACrC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oBACxB,EAAE,IAAI,EAAE,SAAkB,EAAE,KAAK,EAAE,CAAC,CAAC,WAAW,EAAE;oBAClD,EAAE,IAAI,EAAE,MAAe,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAA2B;iBACnE,CAAC;aACH;SACF,CAAC,EAAE,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAQ,CAAC,CAAC,CAAC;YAExE,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,MAAM,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC;gBAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE/C,IACE,YAAY,IAAI,IAAI;oBACpB,YAAY,CAAC,MAAM,IAAI,WAAW;oBAClC,SAAS,IAAI,SAAS;oBACtB,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,IAAI,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;oBAC9D,YAAY,CAAC,IAAI,CAAC,MAAM,IAAI,QAAQ,EACpC,CAAC;oBACD,IAAI,KAAK,GAAuB,SAAS,CAAC;oBAC1C,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;wBACzB,IAAI,YAAY,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;4BACvC,YAAY,CAAC,QAAQ,GAAG,IAAI,CAAC;wBAC/B,CAAC;wBAED,MAAM,UAAU,GAAG,YAAY,CAAC;wBAChC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;wBAC3B,YAAY,GAAG,IAAI,CAAC;wBACpB,SAAS,GAAG,CAAC,CAAC;wBACd,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;wBAChD,QAAQ,GAAG,IAAI,CAAC;wBAChB,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;4BACzB,uCAAuC;4BACvC,MAAM;wBACR,CAAC;oBACH,CAAC;oBAED,KAAK,KAAK,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;oBACvC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;wBAClB,MAAM,IAAI,KAAK,CAAC,+BAA+B,WAAW,EAAE,CAAC,CAAC;oBAChE,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,KAAK;wBACZ,QAAQ,EAAE,KAAK;wBACf,IAAI,EAAE,EAAE;wBACR,UAAU,EAAE,KAAK;qBAClB,CAAC;oBACF,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;gBAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;gBAE9B,IAAI,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;oBACvC,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;gBACnG,CAAC;gBAED,IAAI,GAAG,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;oBAC1B,cAAc;oBACd,MAAM,WAAW,GAAG,GAAG,CAAC,SAAS,CAAC;oBAClC,IAAI,QAAQ,IAAI,IAAI,IAAI,WAAW,GAAG,QAAQ,EAAE,CAAC;wBAC/C,QAAQ,GAAG,WAAW,CAAC;oBACzB,CAAC;gBACH,CAAC;gBAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,YAAY,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;gBAEtC,SAAS,IAAI,OAAO,CAAC;gBAErB,wCAAwC;gBACxC,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;QAED,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,YAAY,CAAC;YAChC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;YAChD,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAsB,EAAE,OAAiB;QAC1D,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAkC;QAChD,IAAI,CAAC,OAAO,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;kBAGH,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE;0BACpD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE;;eAEzC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KAChD,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;eAQ3B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KAChD;aACE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,eAAe,EAAE,qBAAqB,EAAE,OAAO,CAAC,CAAC,CAAC;aAClF,KAAK,EAAE,CAAC;QAEX,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,OAAO;YACL,aAAa,EAAE,YAAY,CAAC,aAAa;YACzC,MAAM,EAAE,YAAY,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM;YAC1D,cAAc,EAAE,YAAY,CAAC,mBAAmB,IAAI,IAAI;SACzD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;eAQN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KAChD,CAAC,OAAO,EAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;qBAGA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KACtD,CAAC,OAAO,EAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;qBAGA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KACtD,CAAC,OAAO,EAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;qBAGA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KACtD,CAAC,OAAO,EAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;qBAGA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;KACtD,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACrC,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;iBAMtB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;OAChD;iBACE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;iBAC1C,KAAK,EAAE,CAAC;YAEX,IAAI,YAAY,IAAI,YAAY,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;gBAC3E,MAAM,EAAE,CAAC,GAAG,CAAA;;;sBAGE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE;;mBAE3D,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;SAChD,CAAC,OAAO,EAAE,CAAC;YACd,CAAC;YAED,MAAM,EAAE,CAAC,GAAG,CAAA;;;oBAGE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE;;oBAEtD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE;sBACtD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;OACrD,CAAC,OAAO,EAAE,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,KAA2C;QAC5E,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,CAAC;YACJ,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;SACpB,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;mCAQF,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE;;;;;;;;;;;;;;;;;;uBAkBjD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;;;KAGxD,CAAC,IAAI,EAAmF,CAAC;QAE1F,OAAO,IAAI,GAAG,CACZ,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,OAAO;gBACL,GAAG,CAAC,MAAM;gBACV;oBACE,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;oBAC/B,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU;oBAC9E,cAAc,EAAE,GAAG,CAAC,YAAY,IAAI,CAAC;iBACJ;aACpC,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}