@powersync/service-core 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219091224

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/CHANGELOG.md +63 -4
  2. package/dist/auth/KeySpec.d.ts +1 -0
  3. package/dist/auth/KeySpec.js +10 -8
  4. package/dist/auth/KeySpec.js.map +1 -1
  5. package/dist/auth/RemoteJWKSCollector.js +2 -2
  6. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  7. package/dist/entry/commands/compact-action.js +15 -15
  8. package/dist/entry/commands/compact-action.js.map +1 -1
  9. package/dist/entry/commands/migrate-action.js +15 -4
  10. package/dist/entry/commands/migrate-action.js.map +1 -1
  11. package/dist/index.d.ts +1 -3
  12. package/dist/index.js +1 -3
  13. package/dist/index.js.map +1 -1
  14. package/dist/migrations/PowerSyncMigrationManager.d.ts +17 -0
  15. package/dist/migrations/PowerSyncMigrationManager.js +22 -0
  16. package/dist/migrations/PowerSyncMigrationManager.js.map +1 -0
  17. package/dist/migrations/ensure-automatic-migrations.d.ts +4 -0
  18. package/dist/migrations/ensure-automatic-migrations.js +14 -0
  19. package/dist/migrations/ensure-automatic-migrations.js.map +1 -0
  20. package/dist/migrations/migrations-index.d.ts +2 -3
  21. package/dist/migrations/migrations-index.js +2 -3
  22. package/dist/migrations/migrations-index.js.map +1 -1
  23. package/dist/routes/RouterEngine.js +2 -1
  24. package/dist/routes/RouterEngine.js.map +1 -1
  25. package/dist/routes/configure-fastify.d.ts +28 -28
  26. package/dist/routes/endpoints/admin.d.ts +24 -24
  27. package/dist/storage/BucketStorage.d.ts +41 -1
  28. package/dist/storage/BucketStorage.js +26 -0
  29. package/dist/storage/BucketStorage.js.map +1 -1
  30. package/dist/storage/storage-index.d.ts +2 -14
  31. package/dist/storage/storage-index.js +2 -14
  32. package/dist/storage/storage-index.js.map +1 -1
  33. package/dist/sync/sync.js +12 -3
  34. package/dist/sync/sync.js.map +1 -1
  35. package/dist/system/ServiceContext.d.ts +3 -0
  36. package/dist/system/ServiceContext.js +11 -3
  37. package/dist/system/ServiceContext.js.map +1 -1
  38. package/dist/util/config/types.d.ts +2 -2
  39. package/dist/util/utils.d.ts +14 -1
  40. package/dist/util/utils.js +56 -0
  41. package/dist/util/utils.js.map +1 -1
  42. package/package.json +6 -7
  43. package/src/auth/KeySpec.ts +12 -9
  44. package/src/auth/RemoteJWKSCollector.ts +2 -2
  45. package/src/entry/commands/compact-action.ts +20 -15
  46. package/src/entry/commands/migrate-action.ts +17 -4
  47. package/src/index.ts +1 -4
  48. package/src/migrations/PowerSyncMigrationManager.ts +43 -0
  49. package/src/migrations/ensure-automatic-migrations.ts +15 -0
  50. package/src/migrations/migrations-index.ts +2 -3
  51. package/src/routes/RouterEngine.ts +2 -1
  52. package/src/storage/BucketStorage.ts +44 -1
  53. package/src/storage/storage-index.ts +3 -15
  54. package/src/sync/sync.ts +12 -3
  55. package/src/system/ServiceContext.ts +17 -4
  56. package/src/util/config/types.ts +2 -2
  57. package/src/util/utils.ts +59 -1
  58. package/test/src/auth.test.ts +54 -21
  59. package/test/src/env.ts +0 -1
  60. package/tsconfig.tsbuildinfo +1 -1
  61. package/dist/db/db-index.d.ts +0 -1
  62. package/dist/db/db-index.js +0 -2
  63. package/dist/db/db-index.js.map +0 -1
  64. package/dist/db/mongo.d.ts +0 -35
  65. package/dist/db/mongo.js +0 -73
  66. package/dist/db/mongo.js.map +0 -1
  67. package/dist/locks/LockManager.d.ts +0 -10
  68. package/dist/locks/LockManager.js +0 -7
  69. package/dist/locks/LockManager.js.map +0 -1
  70. package/dist/locks/MongoLocks.d.ts +0 -36
  71. package/dist/locks/MongoLocks.js +0 -81
  72. package/dist/locks/MongoLocks.js.map +0 -1
  73. package/dist/locks/locks-index.d.ts +0 -2
  74. package/dist/locks/locks-index.js +0 -3
  75. package/dist/locks/locks-index.js.map +0 -1
  76. package/dist/migrations/db/migrations/1684951997326-init.d.ts +0 -3
  77. package/dist/migrations/db/migrations/1684951997326-init.js +0 -33
  78. package/dist/migrations/db/migrations/1684951997326-init.js.map +0 -1
  79. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.d.ts +0 -2
  80. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js +0 -5
  81. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map +0 -1
  82. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +0 -3
  83. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +0 -56
  84. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +0 -1
  85. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +0 -3
  86. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +0 -29
  87. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +0 -1
  88. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.d.ts +0 -3
  89. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js +0 -31
  90. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js.map +0 -1
  91. package/dist/migrations/definitions.d.ts +0 -18
  92. package/dist/migrations/definitions.js +0 -6
  93. package/dist/migrations/definitions.js.map +0 -1
  94. package/dist/migrations/executor.d.ts +0 -16
  95. package/dist/migrations/executor.js +0 -64
  96. package/dist/migrations/executor.js.map +0 -1
  97. package/dist/migrations/migrations.d.ts +0 -18
  98. package/dist/migrations/migrations.js +0 -110
  99. package/dist/migrations/migrations.js.map +0 -1
  100. package/dist/migrations/store/migration-store.d.ts +0 -11
  101. package/dist/migrations/store/migration-store.js +0 -46
  102. package/dist/migrations/store/migration-store.js.map +0 -1
  103. package/dist/storage/MongoBucketStorage.d.ts +0 -48
  104. package/dist/storage/MongoBucketStorage.js +0 -426
  105. package/dist/storage/MongoBucketStorage.js.map +0 -1
  106. package/dist/storage/mongo/MongoBucketBatch.d.ts +0 -67
  107. package/dist/storage/mongo/MongoBucketBatch.js +0 -643
  108. package/dist/storage/mongo/MongoBucketBatch.js.map +0 -1
  109. package/dist/storage/mongo/MongoCompactor.d.ts +0 -40
  110. package/dist/storage/mongo/MongoCompactor.js +0 -309
  111. package/dist/storage/mongo/MongoCompactor.js.map +0 -1
  112. package/dist/storage/mongo/MongoIdSequence.d.ts +0 -12
  113. package/dist/storage/mongo/MongoIdSequence.js +0 -21
  114. package/dist/storage/mongo/MongoIdSequence.js.map +0 -1
  115. package/dist/storage/mongo/MongoPersistedSyncRules.d.ts +0 -9
  116. package/dist/storage/mongo/MongoPersistedSyncRules.js +0 -9
  117. package/dist/storage/mongo/MongoPersistedSyncRules.js.map +0 -1
  118. package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +0 -20
  119. package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +0 -26
  120. package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +0 -1
  121. package/dist/storage/mongo/MongoStorageProvider.d.ts +0 -5
  122. package/dist/storage/mongo/MongoStorageProvider.js +0 -26
  123. package/dist/storage/mongo/MongoStorageProvider.js.map +0 -1
  124. package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +0 -38
  125. package/dist/storage/mongo/MongoSyncBucketStorage.js +0 -531
  126. package/dist/storage/mongo/MongoSyncBucketStorage.js.map +0 -1
  127. package/dist/storage/mongo/MongoSyncRulesLock.d.ts +0 -16
  128. package/dist/storage/mongo/MongoSyncRulesLock.js +0 -65
  129. package/dist/storage/mongo/MongoSyncRulesLock.js.map +0 -1
  130. package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +0 -20
  131. package/dist/storage/mongo/MongoWriteCheckpointAPI.js +0 -103
  132. package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +0 -1
  133. package/dist/storage/mongo/OperationBatch.d.ts +0 -35
  134. package/dist/storage/mongo/OperationBatch.js +0 -119
  135. package/dist/storage/mongo/OperationBatch.js.map +0 -1
  136. package/dist/storage/mongo/PersistedBatch.d.ts +0 -46
  137. package/dist/storage/mongo/PersistedBatch.js +0 -213
  138. package/dist/storage/mongo/PersistedBatch.js.map +0 -1
  139. package/dist/storage/mongo/config.d.ts +0 -19
  140. package/dist/storage/mongo/config.js +0 -26
  141. package/dist/storage/mongo/config.js.map +0 -1
  142. package/dist/storage/mongo/db.d.ts +0 -36
  143. package/dist/storage/mongo/db.js +0 -47
  144. package/dist/storage/mongo/db.js.map +0 -1
  145. package/dist/storage/mongo/models.d.ts +0 -156
  146. package/dist/storage/mongo/models.js +0 -27
  147. package/dist/storage/mongo/models.js.map +0 -1
  148. package/dist/storage/mongo/util.d.ts +0 -40
  149. package/dist/storage/mongo/util.js +0 -151
  150. package/dist/storage/mongo/util.js.map +0 -1
  151. package/src/db/db-index.ts +0 -1
  152. package/src/db/mongo.ts +0 -81
  153. package/src/locks/LockManager.ts +0 -16
  154. package/src/locks/MongoLocks.ts +0 -142
  155. package/src/locks/locks-index.ts +0 -2
  156. package/src/migrations/db/migrations/1684951997326-init.ts +0 -38
  157. package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts +0 -5
  158. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +0 -102
  159. package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +0 -34
  160. package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts +0 -37
  161. package/src/migrations/definitions.ts +0 -21
  162. package/src/migrations/executor.ts +0 -87
  163. package/src/migrations/migrations.ts +0 -142
  164. package/src/migrations/store/migration-store.ts +0 -63
  165. package/src/storage/MongoBucketStorage.ts +0 -540
  166. package/src/storage/mongo/MongoBucketBatch.ts +0 -841
  167. package/src/storage/mongo/MongoCompactor.ts +0 -392
  168. package/src/storage/mongo/MongoIdSequence.ts +0 -24
  169. package/src/storage/mongo/MongoPersistedSyncRules.ts +0 -16
  170. package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +0 -50
  171. package/src/storage/mongo/MongoStorageProvider.ts +0 -31
  172. package/src/storage/mongo/MongoSyncBucketStorage.ts +0 -636
  173. package/src/storage/mongo/MongoSyncRulesLock.ts +0 -85
  174. package/src/storage/mongo/MongoWriteCheckpointAPI.ts +0 -151
  175. package/src/storage/mongo/OperationBatch.ts +0 -131
  176. package/src/storage/mongo/PersistedBatch.ts +0 -272
  177. package/src/storage/mongo/config.ts +0 -40
  178. package/src/storage/mongo/db.ts +0 -88
  179. package/src/storage/mongo/models.ts +0 -179
  180. package/src/storage/mongo/util.ts +0 -158
  181. package/test/src/__snapshots__/sync.test.ts.snap +0 -332
  182. package/test/src/bucket_validation.test.ts +0 -142
  183. package/test/src/bucket_validation.ts +0 -116
  184. package/test/src/compacting.test.ts +0 -295
  185. package/test/src/data_storage.test.ts +0 -1499
  186. package/test/src/stream_utils.ts +0 -42
  187. package/test/src/sync.test.ts +0 -511
  188. package/test/src/util.ts +0 -148
@@ -1,110 +0,0 @@
1
- import * as fs from 'fs/promises';
2
- import * as path from 'path';
3
- import { fileURLToPath } from 'url';
4
- import { logger } from '@powersync/lib-services-framework';
5
- import * as db from '../db/db-index.js';
6
- import * as locks from '../locks/locks-index.js';
7
- import * as util from '../util/util-index.js';
8
- import { Direction } from './definitions.js';
9
- import { execute, writeLogsToStore } from './executor.js';
10
- import { createMongoMigrationStore } from './store/migration-store.js';
11
- const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
12
- const MONGO_LOCK_PROCESS = 'migrations';
13
- const __filename = fileURLToPath(import.meta.url);
14
- const __dirname = path.dirname(__filename);
15
- const MIGRATIONS_DIR = path.join(__dirname, '/db/migrations');
16
- /**
17
- * Loads migrations and injects a custom context for loading the specified
18
- * runner configuration.
19
- */
20
- const loadMigrations = async (dir, runnerConfig) => {
21
- const files = await fs.readdir(dir);
22
- const migrations = files.filter((file) => {
23
- return path.extname(file) === '.js';
24
- });
25
- const context = {
26
- runner_config: runnerConfig
27
- };
28
- return await Promise.all(migrations.map(async (migration) => {
29
- const module = await import(path.resolve(dir, migration));
30
- return {
31
- name: path.basename(migration).replace(path.extname(migration), ''),
32
- up: () => module.up(context),
33
- down: () => module.down(context)
34
- };
35
- }));
36
- };
37
- /**
38
- * Runs migration scripts exclusively using Mongo locks
39
- */
40
- export const migrate = async (options) => {
41
- const { direction, runner_config } = options;
42
- const config = await util.loadConfig(runner_config);
43
- const { storage } = config;
44
- /**
45
- * Try and get Mongo from config file.
46
- * But this might not be available in Journey Micro as we use the standard Mongo.
47
- */
48
- const client = db.mongo.createMongoClient(storage);
49
- logger.info('Connecting to MongoDB');
50
- await client.connect();
51
- const clientDB = client.db(storage.database);
52
- const collection = clientDB.collection(DEFAULT_MONGO_LOCK_COLLECTION);
53
- const manager = locks.createMongoLockManager(collection, {
54
- name: MONGO_LOCK_PROCESS
55
- });
56
- // Only one process should execute this at a time.
57
- logger.info('Acquiring lock');
58
- const lockId = await manager.acquire();
59
- if (!lockId) {
60
- throw new Error('Could not acquire lock_id');
61
- }
62
- let isReleased = false;
63
- const releaseLock = async () => {
64
- if (isReleased) {
65
- return;
66
- }
67
- await manager.release(lockId);
68
- isReleased = true;
69
- };
70
- // For the case where the migration is terminated
71
- process.addListener('beforeExit', releaseLock);
72
- try {
73
- logger.info('Loading migrations');
74
- const migrations = await loadMigrations(MIGRATIONS_DIR, runner_config);
75
- // Use the provided config to connect to Mongo
76
- const store = createMongoMigrationStore(clientDB);
77
- const state = await store.load();
78
- logger.info('Running migrations');
79
- const logStream = execute({
80
- direction: direction,
81
- migrations,
82
- state
83
- });
84
- await writeLogsToStore({
85
- log_stream: logStream,
86
- store,
87
- state
88
- });
89
- }
90
- finally {
91
- logger.info('Releasing lock');
92
- await releaseLock();
93
- logger.info('Closing database');
94
- await client.close(true);
95
- process.removeListener('beforeExit', releaseLock);
96
- logger.info('Done with migrations');
97
- }
98
- };
99
- /**
100
- * Ensures automatic migrations are executed
101
- */
102
- export const ensureAutomaticMigrations = async (params) => {
103
- if (!params.config.migrations?.disable_auto_migration) {
104
- await migrate({
105
- direction: Direction.Up,
106
- runner_config: params.runner_config
107
- });
108
- }
109
- };
110
- //# sourceMappingURL=migrations.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/migrations/migrations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxC,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAEvE,MAAM,6BAA6B,GAAG,OAAO,CAAC;AAC9C,MAAM,kBAAkB,GAAG,YAAY,CAAC;AAExC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;AAY9D;;;GAGG;AACH,MAAM,cAAc,GAAG,KAAK,EAAE,GAAW,EAAE,YAA+B,EAAE,EAAE;IAC5E,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAA0B;QACrC,aAAa,EAAE,YAAY;KAC5B,CAAC;IAEF,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;QACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;QAC1D,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;YACnE,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC;YAC5B,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;SACjC,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAyB,EAAE,EAAE;IACzD,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAE7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC3B;;;OAGG;IAEH,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACnD,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACrC,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IAEvB,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAa,6BAA6B,CAAC,CAAC;IAElF,MAAM,OAAO,GAAG,KAAK,CAAC,sBAAsB,CAAC,UAAU,EAAE;QACvD,IAAI,EAAE,kBAAkB;KACzB,CAAC,CAAC;IAEH,kDAAkD;IAClD,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;QAC7B,IAAI,UAAU,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9B,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC,CAAC;IAEF,iDAAiD;IACjD,OAAO,CAAC,WAAW,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAE/C,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;QAEvE,8CAA8C;QAC9C,MAAM,KAAK,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAElD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEjC,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAClC,MAAM,SAAS,GAAG,OAAO,CAAC;YACxB,SAAS,EAAE,SAAS;YACpB,UAAU;YACV,KAAK;SACN,CAAC,CAAC;QAEH,MAAM,gBAAgB,CAAC;YACrB,UAAU,EAAE,SAAS;YACrB,KAAK;YACL,KAAK;SACN,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC9B,MAAM,WAAW,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAChC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzB,OAAO,CAAC,cAAc,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,KAAK,EAAE,MAAgC,EAAE,EAAE;IAClF,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,sBAAsB,EAAE,CAAC;QACtD,MAAM,OAAO,CAAC;YACZ,SAAS,EAAE,SAAS,CAAC,EAAE;YACvB,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC"}
@@ -1,11 +0,0 @@
1
- import { Db } from 'mongodb';
2
- import * as defs from '../definitions.js';
3
- export type MigrationStore = {
4
- load: () => Promise<defs.MigrationState | undefined>;
5
- save: (state: defs.MigrationState) => Promise<void>;
6
- };
7
- /**
8
- * A custom store for node-migrate which is used to save and load migrations that have
9
- * been operated on to mongo.
10
- */
11
- export declare const createMongoMigrationStore: (db: Db) => MigrationStore;
@@ -1,46 +0,0 @@
1
- import * as path from 'path';
2
- /**
3
- * A custom store for node-migrate which is used to save and load migrations that have
4
- * been operated on to mongo.
5
- */
6
- export const createMongoMigrationStore = (db) => {
7
- const collection = db.collection('migrations');
8
- return {
9
- load: async () => {
10
- const state_entry = await collection.findOne();
11
- if (!state_entry) {
12
- return;
13
- }
14
- const { _id, ...state } = state_entry;
15
- /**
16
- * This is for backwards compatibility. A previous version of the migration tool used to save
17
- * state as `lastRun`.
18
- */
19
- let last_run = state.last_run;
20
- if ('lastRun' in state) {
21
- last_run = state.lastRun;
22
- }
23
- /**
24
- * This is for backwards compatibility. A previous version of the migration tool used to include the
25
- * file extension in migration names. This strips that extension off if it exists
26
- */
27
- const extension = path.extname(last_run);
28
- if (extension) {
29
- last_run = last_run.replace(extension, '');
30
- }
31
- return {
32
- last_run,
33
- log: state.log || []
34
- };
35
- },
36
- save: async (state) => {
37
- await collection.replaceOne({}, {
38
- last_run: state.last_run,
39
- log: state.log
40
- }, {
41
- upsert: true
42
- });
43
- }
44
- };
45
- };
46
- //# sourceMappingURL=migration-store.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"migration-store.js","sourceRoot":"","sources":["../../../src/migrations/store/migration-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAQ7B;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,EAAM,EAAkB,EAAE;IAClE,MAAM,UAAU,GAAG,EAAE,CAAC,UAAU,CAAsB,YAAY,CAAC,CAAC;IAEpE,OAAO;QACL,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;YAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,OAAO;YACT,CAAC;YAED,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,WAAW,CAAC;YAEtC;;;eAGG;YACH,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;YAC9B,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;gBACvB,QAAQ,GAAI,KAAa,CAAC,OAAO,CAAC;YACpC,CAAC;YAED;;;eAGG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzC,IAAI,SAAS,EAAE,CAAC;gBACd,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,OAAO;gBACL,QAAQ;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,EAAE;aACrB,CAAC;QACJ,CAAC;QAED,IAAI,EAAE,KAAK,EAAE,KAA0B,EAAE,EAAE;YACzC,MAAM,UAAU,CAAC,UAAU,CACzB,EAAE,EACF;gBACE,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,GAAG,EAAE,KAAK,CAAC,GAAG;aACf,EACD;gBACE,MAAM,EAAE,IAAI;aACb,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC,CAAC"}
@@ -1,48 +0,0 @@
1
- import { DisposableObserver } from '@powersync/lib-services-framework';
2
- import { ActiveCheckpoint, BucketStorageFactory, BucketStorageFactoryListener, ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent, StorageMetrics, UpdateSyncRulesOptions, WriteCheckpoint } from './BucketStorage.js';
3
- import { PowerSyncMongo } from './mongo/db.js';
4
- import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
5
- import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
6
- export declare class MongoBucketStorage extends DisposableObserver<BucketStorageFactoryListener> implements BucketStorageFactory {
7
- private readonly client;
8
- private readonly session;
9
- readonly slot_name_prefix: string;
10
- private readonly storageCache;
11
- readonly db: PowerSyncMongo;
12
- constructor(db: PowerSyncMongo, options: {
13
- slot_name_prefix: string;
14
- });
15
- getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage;
16
- configureSyncRules(sync_rules: string, options?: {
17
- lock?: boolean;
18
- }): Promise<{
19
- updated: boolean;
20
- persisted_sync_rules?: undefined;
21
- lock?: undefined;
22
- } | {
23
- updated: boolean;
24
- persisted_sync_rules: MongoPersistedSyncRulesContent;
25
- lock: import("./storage-index.js").MongoSyncRulesLock | undefined;
26
- }>;
27
- slotRemoved(slot_name: string): Promise<void>;
28
- updateSyncRules(options: UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent>;
29
- getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null>;
30
- getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
31
- getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null>;
32
- getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
33
- getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
34
- getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
35
- getActiveCheckpoint(): Promise<ActiveCheckpoint>;
36
- getStorageMetrics(): Promise<StorageMetrics>;
37
- getPowerSyncInstanceId(): Promise<string>;
38
- private makeActiveCheckpoint;
39
- /**
40
- * Instance-wide watch on the latest available checkpoint (op_id + lsn).
41
- */
42
- private watchActiveCheckpoint;
43
- private readonly sharedIter;
44
- /**
45
- * User-specific watch on the latest checkpoint and/or write checkpoint.
46
- */
47
- watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable<WriteCheckpoint>;
48
- }
@@ -1,426 +0,0 @@
1
- import { SqlSyncRules } from '@powersync/service-sync-rules';
2
- import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
3
- import { LRUCache } from 'lru-cache/min';
4
- import * as mongo from 'mongodb';
5
- import * as timers from 'timers/promises';
6
- import * as locks from '../locks/locks-index.js';
7
- import * as sync from '../sync/sync-index.js';
8
- import * as util from '../util/util-index.js';
9
- import { DisposableObserver, logger } from '@powersync/lib-services-framework';
10
- import { v4 as uuid } from 'uuid';
11
- import { SyncRuleState } from './mongo/models.js';
12
- import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
13
- import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
14
- import { generateSlotName } from './mongo/util.js';
15
- export class MongoBucketStorage extends DisposableObserver {
16
- constructor(db, options) {
17
- super();
18
- this.storageCache = new LRUCache({
19
- max: 3,
20
- fetchMethod: async (id) => {
21
- const doc2 = await this.db.sync_rules.findOne({
22
- _id: id
23
- }, { limit: 1 });
24
- if (doc2 == null) {
25
- // Deleted in the meantime?
26
- return undefined;
27
- }
28
- const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
29
- return this.getInstance(rules);
30
- },
31
- dispose: (storage) => {
32
- storage[Symbol.dispose]();
33
- }
34
- });
35
- // Nothing is done here until a subscriber starts to iterate
36
- this.sharedIter = new sync.BroadcastIterable((signal) => {
37
- return this.watchActiveCheckpoint(signal);
38
- });
39
- this.client = db.client;
40
- this.db = db;
41
- this.session = this.client.startSession();
42
- this.slot_name_prefix = options.slot_name_prefix;
43
- }
44
- getInstance(options) {
45
- let { id, slot_name } = options;
46
- if (typeof id == 'bigint') {
47
- id = Number(id);
48
- }
49
- const storage = new MongoSyncBucketStorage(this, id, options, slot_name);
50
- this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
51
- storage.registerListener({
52
- batchStarted: (batch) => {
53
- // This nested listener will be automatically disposed when the storage is disposed
54
- batch.registerManagedListener(storage, {
55
- replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
56
- });
57
- }
58
- });
59
- return storage;
60
- }
61
- async configureSyncRules(sync_rules, options) {
62
- const next = await this.getNextSyncRulesContent();
63
- const active = await this.getActiveSyncRulesContent();
64
- if (next?.sync_rules_content == sync_rules) {
65
- logger.info('Sync rules from configuration unchanged');
66
- return { updated: false };
67
- }
68
- else if (next == null && active?.sync_rules_content == sync_rules) {
69
- logger.info('Sync rules from configuration unchanged');
70
- return { updated: false };
71
- }
72
- else {
73
- logger.info('Sync rules updated from configuration');
74
- const persisted_sync_rules = await this.updateSyncRules({
75
- content: sync_rules,
76
- lock: options?.lock
77
- });
78
- return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
79
- }
80
- }
81
- async slotRemoved(slot_name) {
82
- const next = await this.getNextSyncRulesContent();
83
- const active = await this.getActiveSyncRulesContent();
84
- // In both the below cases, we create a new sync rules instance.
85
- // The current one will continue erroring until the next one has finished processing.
86
- // TODO: Update
87
- if (next != null && next.slot_name == slot_name) {
88
- // We need to redo the "next" sync rules
89
- await this.updateSyncRules({
90
- content: next.sync_rules_content
91
- });
92
- // Pro-actively stop replicating
93
- await this.db.sync_rules.updateOne({
94
- _id: next.id,
95
- state: SyncRuleState.PROCESSING
96
- }, {
97
- $set: {
98
- state: SyncRuleState.STOP
99
- }
100
- });
101
- }
102
- else if (next == null && active?.slot_name == slot_name) {
103
- // Slot removed for "active" sync rules, while there is no "next" one.
104
- await this.updateSyncRules({
105
- content: active.sync_rules_content
106
- });
107
- // Pro-actively stop replicating
108
- await this.db.sync_rules.updateOne({
109
- _id: active.id,
110
- state: SyncRuleState.ACTIVE
111
- }, {
112
- $set: {
113
- state: SyncRuleState.STOP
114
- }
115
- });
116
- }
117
- }
118
- async updateSyncRules(options) {
119
- // Parse and validate before applying any changes
120
- const parsed = SqlSyncRules.fromYaml(options.content, {
121
- // No schema-based validation at this point
122
- schema: undefined,
123
- defaultSchema: 'not_applicable', // Not needed for validation
124
- throwOnError: true
125
- });
126
- let rules = undefined;
127
- await this.session.withTransaction(async () => {
128
- // Only have a single set of sync rules with PROCESSING.
129
- await this.db.sync_rules.updateMany({
130
- state: SyncRuleState.PROCESSING
131
- }, { $set: { state: SyncRuleState.STOP } });
132
- const id_doc = await this.db.op_id_sequence.findOneAndUpdate({
133
- _id: 'sync_rules'
134
- }, {
135
- $inc: {
136
- op_id: 1n
137
- }
138
- }, {
139
- upsert: true,
140
- returnDocument: 'after'
141
- });
142
- const id = Number(id_doc.op_id);
143
- const slot_name = generateSlotName(this.slot_name_prefix, id);
144
- const doc = {
145
- _id: id,
146
- content: options.content,
147
- last_checkpoint: null,
148
- last_checkpoint_lsn: null,
149
- no_checkpoint_before: null,
150
- snapshot_done: false,
151
- state: SyncRuleState.PROCESSING,
152
- slot_name: slot_name,
153
- last_checkpoint_ts: null,
154
- last_fatal_error: null,
155
- last_keepalive_ts: null
156
- };
157
- await this.db.sync_rules.insertOne(doc);
158
- rules = new MongoPersistedSyncRulesContent(this.db, doc);
159
- if (options.lock) {
160
- const lock = await rules.lock();
161
- }
162
- });
163
- return rules;
164
- }
165
- async getActiveSyncRulesContent() {
166
- const doc = await this.db.sync_rules.findOne({
167
- state: SyncRuleState.ACTIVE
168
- }, { sort: { _id: -1 }, limit: 1 });
169
- if (doc == null) {
170
- return null;
171
- }
172
- return new MongoPersistedSyncRulesContent(this.db, doc);
173
- }
174
- async getActiveSyncRules(options) {
175
- const content = await this.getActiveSyncRulesContent();
176
- return content?.parsed(options) ?? null;
177
- }
178
- async getNextSyncRulesContent() {
179
- const doc = await this.db.sync_rules.findOne({
180
- state: SyncRuleState.PROCESSING
181
- }, { sort: { _id: -1 }, limit: 1 });
182
- if (doc == null) {
183
- return null;
184
- }
185
- return new MongoPersistedSyncRulesContent(this.db, doc);
186
- }
187
- async getNextSyncRules(options) {
188
- const content = await this.getNextSyncRulesContent();
189
- return content?.parsed(options) ?? null;
190
- }
191
- async getReplicatingSyncRules() {
192
- const docs = await this.db.sync_rules
193
- .find({
194
- $or: [{ state: SyncRuleState.ACTIVE }, { state: SyncRuleState.PROCESSING }]
195
- })
196
- .toArray();
197
- return docs.map((doc) => {
198
- return new MongoPersistedSyncRulesContent(this.db, doc);
199
- });
200
- }
201
- async getStoppedSyncRules() {
202
- const docs = await this.db.sync_rules
203
- .find({
204
- state: SyncRuleState.STOP
205
- })
206
- .toArray();
207
- return docs.map((doc) => {
208
- return new MongoPersistedSyncRulesContent(this.db, doc);
209
- });
210
- }
211
- async getActiveCheckpoint() {
212
- const doc = await this.db.sync_rules.findOne({
213
- state: SyncRuleState.ACTIVE
214
- }, {
215
- sort: { _id: -1 },
216
- limit: 1,
217
- projection: { _id: 1, last_checkpoint: 1, last_checkpoint_lsn: 1 }
218
- });
219
- return this.makeActiveCheckpoint(doc);
220
- }
221
- async getStorageMetrics() {
222
- const ignoreNotExiting = (e) => {
223
- if (e instanceof mongo.MongoServerError && e.codeName == 'NamespaceNotFound') {
224
- // Collection doesn't exist - return 0
225
- return [{ storageStats: { size: 0 } }];
226
- }
227
- else {
228
- return Promise.reject(e);
229
- }
230
- };
231
- const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
232
- if (active_sync_rules == null) {
233
- return {
234
- operations_size_bytes: 0,
235
- parameters_size_bytes: 0,
236
- replication_size_bytes: 0
237
- };
238
- }
239
- const operations_aggregate = await this.db.bucket_data
240
- .aggregate([
241
- {
242
- $collStats: {
243
- storageStats: {}
244
- }
245
- }
246
- ])
247
- .toArray()
248
- .catch(ignoreNotExiting);
249
- const parameters_aggregate = await this.db.bucket_parameters
250
- .aggregate([
251
- {
252
- $collStats: {
253
- storageStats: {}
254
- }
255
- }
256
- ])
257
- .toArray()
258
- .catch(ignoreNotExiting);
259
- const replication_aggregate = await this.db.current_data
260
- .aggregate([
261
- {
262
- $collStats: {
263
- storageStats: {}
264
- }
265
- }
266
- ])
267
- .toArray()
268
- .catch(ignoreNotExiting);
269
- return {
270
- operations_size_bytes: operations_aggregate[0].storageStats.size,
271
- parameters_size_bytes: parameters_aggregate[0].storageStats.size,
272
- replication_size_bytes: replication_aggregate[0].storageStats.size
273
- };
274
- }
275
- async getPowerSyncInstanceId() {
276
- let instance = await this.db.instance.findOne({
277
- _id: { $exists: true }
278
- });
279
- if (!instance) {
280
- const manager = locks.createMongoLockManager(this.db.locks, {
281
- name: `instance-id-insertion-lock`
282
- });
283
- await manager.lock(async () => {
284
- await this.db.instance.insertOne({
285
- _id: uuid()
286
- });
287
- });
288
- instance = await this.db.instance.findOne({
289
- _id: { $exists: true }
290
- });
291
- }
292
- return instance._id;
293
- }
294
- makeActiveCheckpoint(doc) {
295
- return {
296
- checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
297
- lsn: doc?.last_checkpoint_lsn ?? null,
298
- hasSyncRules() {
299
- return doc != null;
300
- },
301
- getBucketStorage: async () => {
302
- if (doc == null) {
303
- return null;
304
- }
305
- return (await this.storageCache.fetch(doc._id)) ?? null;
306
- }
307
- };
308
- }
309
- /**
310
- * Instance-wide watch on the latest available checkpoint (op_id + lsn).
311
- */
312
- async *watchActiveCheckpoint(signal) {
313
- const pipeline = [
314
- {
315
- $match: {
316
- 'fullDocument.state': 'ACTIVE',
317
- operationType: { $in: ['insert', 'update'] }
318
- }
319
- },
320
- {
321
- $project: {
322
- operationType: 1,
323
- 'fullDocument._id': 1,
324
- 'fullDocument.last_checkpoint': 1,
325
- 'fullDocument.last_checkpoint_lsn': 1
326
- }
327
- }
328
- ];
329
- // Use this form instead of (doc: SyncRuleDocument | null = null),
330
- // otherwise we get weird "doc: never" issues.
331
- let doc = null;
332
- let clusterTime = null;
333
- await this.client.withSession(async (session) => {
334
- doc = await this.db.sync_rules.findOne({
335
- state: SyncRuleState.ACTIVE
336
- }, {
337
- session,
338
- sort: { _id: -1 },
339
- limit: 1,
340
- projection: {
341
- _id: 1,
342
- last_checkpoint: 1,
343
- last_checkpoint_lsn: 1
344
- }
345
- });
346
- const time = session.clusterTime?.clusterTime ?? null;
347
- clusterTime = time;
348
- });
349
- if (clusterTime == null) {
350
- throw new Error('Could not get clusterTime');
351
- }
352
- if (signal.aborted) {
353
- return;
354
- }
355
- if (doc) {
356
- yield this.makeActiveCheckpoint(doc);
357
- }
358
- const stream = this.db.sync_rules.watch(pipeline, {
359
- fullDocument: 'updateLookup',
360
- // Start at the cluster time where we got the initial doc, to make sure
361
- // we don't skip any updates.
362
- // This may result in the first operation being a duplicate, but we filter
363
- // it out anyway.
364
- startAtOperationTime: clusterTime
365
- });
366
- signal.addEventListener('abort', () => {
367
- stream.close();
368
- }, { once: true });
369
- let lastOp = null;
370
- for await (const update of stream.stream()) {
371
- if (signal.aborted) {
372
- break;
373
- }
374
- if (update.operationType != 'insert' && update.operationType != 'update') {
375
- continue;
376
- }
377
- const doc = update.fullDocument;
378
- if (doc == null) {
379
- continue;
380
- }
381
- const op = this.makeActiveCheckpoint(doc);
382
- // Check for LSN / checkpoint changes - ignore other metadata changes
383
- if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
384
- lastOp = op;
385
- yield op;
386
- }
387
- }
388
- }
389
- /**
390
- * User-specific watch on the latest checkpoint and/or write checkpoint.
391
- */
392
- async *watchWriteCheckpoint(user_id, signal) {
393
- let lastCheckpoint = null;
394
- let lastWriteCheckpoint = null;
395
- const iter = wrapWithAbort(this.sharedIter, signal);
396
- for await (const cp of iter) {
397
- const { checkpoint, lsn } = cp;
398
- // lsn changes are not important by itself.
399
- // What is important is:
400
- // 1. checkpoint (op_id) changes.
401
- // 2. write checkpoint changes for the specific user
402
- const bucketStorage = await cp.getBucketStorage();
403
- if (!bucketStorage) {
404
- continue;
405
- }
406
- const lsnFilters = lsn ? { 1: lsn } : {};
407
- const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({
408
- user_id,
409
- heads: {
410
- ...lsnFilters
411
- }
412
- });
413
- if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
414
- // No change - wait for next one
415
- // In some cases, many LSNs may be produced in a short time.
416
- // Add a delay to throttle the write checkpoint lookup a bit.
417
- await timers.setTimeout(20 + 10 * Math.random());
418
- continue;
419
- }
420
- lastWriteCheckpoint = currentWriteCheckpoint;
421
- lastCheckpoint = checkpoint;
422
- yield { base: cp, writeCheckpoint: currentWriteCheckpoint };
423
- }
424
- }
425
- }
426
- //# sourceMappingURL=MongoBucketStorage.js.map