@powersync/service-core 0.12.2 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/dist/auth/KeySpec.d.ts +1 -0
  3. package/dist/auth/KeySpec.js +5 -2
  4. package/dist/auth/KeySpec.js.map +1 -1
  5. package/dist/auth/RemoteJWKSCollector.js +1 -1
  6. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  7. package/dist/entry/commands/compact-action.js +14 -14
  8. package/dist/entry/commands/compact-action.js.map +1 -1
  9. package/dist/entry/commands/migrate-action.js +15 -4
  10. package/dist/entry/commands/migrate-action.js.map +1 -1
  11. package/dist/index.d.ts +1 -3
  12. package/dist/index.js +1 -3
  13. package/dist/index.js.map +1 -1
  14. package/dist/migrations/PowerSyncMigrationManager.d.ts +17 -0
  15. package/dist/migrations/PowerSyncMigrationManager.js +21 -0
  16. package/dist/migrations/PowerSyncMigrationManager.js.map +1 -0
  17. package/dist/migrations/ensure-automatic-migrations.d.ts +4 -0
  18. package/dist/migrations/ensure-automatic-migrations.js +14 -0
  19. package/dist/migrations/ensure-automatic-migrations.js.map +1 -0
  20. package/dist/migrations/migrations-index.d.ts +2 -3
  21. package/dist/migrations/migrations-index.js +2 -3
  22. package/dist/migrations/migrations-index.js.map +1 -1
  23. package/dist/routes/configure-fastify.d.ts +12 -12
  24. package/dist/routes/endpoints/admin.d.ts +24 -24
  25. package/dist/routes/endpoints/probes.d.ts +1 -1
  26. package/dist/routes/endpoints/probes.js +5 -5
  27. package/dist/routes/endpoints/probes.js.map +1 -1
  28. package/dist/storage/BucketStorage.d.ts +49 -1
  29. package/dist/storage/BucketStorage.js +26 -0
  30. package/dist/storage/BucketStorage.js.map +1 -1
  31. package/dist/storage/SourceTable.d.ts +4 -0
  32. package/dist/storage/SourceTable.js +4 -0
  33. package/dist/storage/SourceTable.js.map +1 -1
  34. package/dist/storage/bson.d.ts +24 -0
  35. package/dist/storage/bson.js +73 -0
  36. package/dist/storage/bson.js.map +1 -0
  37. package/dist/storage/storage-index.d.ts +3 -14
  38. package/dist/storage/storage-index.js +3 -14
  39. package/dist/storage/storage-index.js.map +1 -1
  40. package/dist/sync/sync.js +3 -1
  41. package/dist/sync/sync.js.map +1 -1
  42. package/dist/system/ServiceContext.d.ts +3 -0
  43. package/dist/system/ServiceContext.js +11 -3
  44. package/dist/system/ServiceContext.js.map +1 -1
  45. package/dist/util/config/types.d.ts +2 -2
  46. package/dist/util/utils.d.ts +13 -1
  47. package/dist/util/utils.js +20 -1
  48. package/dist/util/utils.js.map +1 -1
  49. package/package.json +7 -8
  50. package/src/auth/KeySpec.ts +5 -3
  51. package/src/auth/RemoteJWKSCollector.ts +1 -1
  52. package/src/entry/commands/compact-action.ts +19 -14
  53. package/src/entry/commands/migrate-action.ts +17 -4
  54. package/src/index.ts +1 -4
  55. package/src/migrations/PowerSyncMigrationManager.ts +42 -0
  56. package/src/migrations/ensure-automatic-migrations.ts +15 -0
  57. package/src/migrations/migrations-index.ts +2 -3
  58. package/src/routes/endpoints/probes.ts +6 -6
  59. package/src/storage/BucketStorage.ts +53 -1
  60. package/src/storage/SourceTable.ts +4 -0
  61. package/src/storage/bson.ts +78 -0
  62. package/src/storage/storage-index.ts +3 -15
  63. package/src/sync/sync.ts +3 -1
  64. package/src/system/ServiceContext.ts +17 -4
  65. package/src/util/config/types.ts +2 -2
  66. package/src/util/utils.ts +21 -1
  67. package/test/src/auth.test.ts +33 -0
  68. package/test/src/env.ts +0 -1
  69. package/test/src/routes/probes.test.ts +6 -1
  70. package/tsconfig.tsbuildinfo +1 -1
  71. package/dist/db/db-index.d.ts +0 -1
  72. package/dist/db/db-index.js +0 -2
  73. package/dist/db/db-index.js.map +0 -1
  74. package/dist/db/mongo.d.ts +0 -35
  75. package/dist/db/mongo.js +0 -73
  76. package/dist/db/mongo.js.map +0 -1
  77. package/dist/locks/LockManager.d.ts +0 -10
  78. package/dist/locks/LockManager.js +0 -7
  79. package/dist/locks/LockManager.js.map +0 -1
  80. package/dist/locks/MongoLocks.d.ts +0 -36
  81. package/dist/locks/MongoLocks.js +0 -81
  82. package/dist/locks/MongoLocks.js.map +0 -1
  83. package/dist/locks/locks-index.d.ts +0 -2
  84. package/dist/locks/locks-index.js +0 -3
  85. package/dist/locks/locks-index.js.map +0 -1
  86. package/dist/migrations/db/migrations/1684951997326-init.d.ts +0 -3
  87. package/dist/migrations/db/migrations/1684951997326-init.js +0 -33
  88. package/dist/migrations/db/migrations/1684951997326-init.js.map +0 -1
  89. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.d.ts +0 -2
  90. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js +0 -5
  91. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map +0 -1
  92. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +0 -3
  93. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +0 -56
  94. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +0 -1
  95. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +0 -3
  96. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +0 -29
  97. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +0 -1
  98. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.d.ts +0 -3
  99. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js +0 -31
  100. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js.map +0 -1
  101. package/dist/migrations/definitions.d.ts +0 -18
  102. package/dist/migrations/definitions.js +0 -6
  103. package/dist/migrations/definitions.js.map +0 -1
  104. package/dist/migrations/executor.d.ts +0 -16
  105. package/dist/migrations/executor.js +0 -64
  106. package/dist/migrations/executor.js.map +0 -1
  107. package/dist/migrations/migrations.d.ts +0 -18
  108. package/dist/migrations/migrations.js +0 -110
  109. package/dist/migrations/migrations.js.map +0 -1
  110. package/dist/migrations/store/migration-store.d.ts +0 -11
  111. package/dist/migrations/store/migration-store.js +0 -46
  112. package/dist/migrations/store/migration-store.js.map +0 -1
  113. package/dist/storage/MongoBucketStorage.d.ts +0 -48
  114. package/dist/storage/MongoBucketStorage.js +0 -427
  115. package/dist/storage/MongoBucketStorage.js.map +0 -1
  116. package/dist/storage/mongo/MongoBucketBatch.d.ts +0 -74
  117. package/dist/storage/mongo/MongoBucketBatch.js +0 -683
  118. package/dist/storage/mongo/MongoBucketBatch.js.map +0 -1
  119. package/dist/storage/mongo/MongoCompactor.d.ts +0 -40
  120. package/dist/storage/mongo/MongoCompactor.js +0 -310
  121. package/dist/storage/mongo/MongoCompactor.js.map +0 -1
  122. package/dist/storage/mongo/MongoIdSequence.d.ts +0 -12
  123. package/dist/storage/mongo/MongoIdSequence.js +0 -21
  124. package/dist/storage/mongo/MongoIdSequence.js.map +0 -1
  125. package/dist/storage/mongo/MongoPersistedSyncRules.d.ts +0 -9
  126. package/dist/storage/mongo/MongoPersistedSyncRules.js +0 -9
  127. package/dist/storage/mongo/MongoPersistedSyncRules.js.map +0 -1
  128. package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +0 -20
  129. package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +0 -26
  130. package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +0 -1
  131. package/dist/storage/mongo/MongoStorageProvider.d.ts +0 -5
  132. package/dist/storage/mongo/MongoStorageProvider.js +0 -26
  133. package/dist/storage/mongo/MongoStorageProvider.js.map +0 -1
  134. package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +0 -38
  135. package/dist/storage/mongo/MongoSyncBucketStorage.js +0 -534
  136. package/dist/storage/mongo/MongoSyncBucketStorage.js.map +0 -1
  137. package/dist/storage/mongo/MongoSyncRulesLock.d.ts +0 -16
  138. package/dist/storage/mongo/MongoSyncRulesLock.js +0 -65
  139. package/dist/storage/mongo/MongoSyncRulesLock.js.map +0 -1
  140. package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +0 -20
  141. package/dist/storage/mongo/MongoWriteCheckpointAPI.js +0 -104
  142. package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +0 -1
  143. package/dist/storage/mongo/OperationBatch.d.ts +0 -35
  144. package/dist/storage/mongo/OperationBatch.js +0 -119
  145. package/dist/storage/mongo/OperationBatch.js.map +0 -1
  146. package/dist/storage/mongo/PersistedBatch.d.ts +0 -46
  147. package/dist/storage/mongo/PersistedBatch.js +0 -223
  148. package/dist/storage/mongo/PersistedBatch.js.map +0 -1
  149. package/dist/storage/mongo/config.d.ts +0 -19
  150. package/dist/storage/mongo/config.js +0 -26
  151. package/dist/storage/mongo/config.js.map +0 -1
  152. package/dist/storage/mongo/db.d.ts +0 -36
  153. package/dist/storage/mongo/db.js +0 -47
  154. package/dist/storage/mongo/db.js.map +0 -1
  155. package/dist/storage/mongo/models.d.ts +0 -163
  156. package/dist/storage/mongo/models.js +0 -27
  157. package/dist/storage/mongo/models.js.map +0 -1
  158. package/dist/storage/mongo/util.d.ts +0 -54
  159. package/dist/storage/mongo/util.js +0 -190
  160. package/dist/storage/mongo/util.js.map +0 -1
  161. package/src/db/db-index.ts +0 -1
  162. package/src/db/mongo.ts +0 -81
  163. package/src/locks/LockManager.ts +0 -16
  164. package/src/locks/MongoLocks.ts +0 -142
  165. package/src/locks/locks-index.ts +0 -2
  166. package/src/migrations/db/migrations/1684951997326-init.ts +0 -38
  167. package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts +0 -5
  168. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +0 -102
  169. package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +0 -34
  170. package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts +0 -37
  171. package/src/migrations/definitions.ts +0 -21
  172. package/src/migrations/executor.ts +0 -87
  173. package/src/migrations/migrations.ts +0 -142
  174. package/src/migrations/store/migration-store.ts +0 -63
  175. package/src/storage/MongoBucketStorage.ts +0 -541
  176. package/src/storage/mongo/MongoBucketBatch.ts +0 -900
  177. package/src/storage/mongo/MongoCompactor.ts +0 -393
  178. package/src/storage/mongo/MongoIdSequence.ts +0 -24
  179. package/src/storage/mongo/MongoPersistedSyncRules.ts +0 -16
  180. package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +0 -50
  181. package/src/storage/mongo/MongoStorageProvider.ts +0 -31
  182. package/src/storage/mongo/MongoSyncBucketStorage.ts +0 -640
  183. package/src/storage/mongo/MongoSyncRulesLock.ts +0 -85
  184. package/src/storage/mongo/MongoWriteCheckpointAPI.ts +0 -154
  185. package/src/storage/mongo/OperationBatch.ts +0 -131
  186. package/src/storage/mongo/PersistedBatch.ts +0 -285
  187. package/src/storage/mongo/config.ts +0 -40
  188. package/src/storage/mongo/db.ts +0 -88
  189. package/src/storage/mongo/models.ts +0 -187
  190. package/src/storage/mongo/util.ts +0 -203
  191. package/test/src/__snapshots__/sync.test.ts.snap +0 -332
  192. package/test/src/bucket_validation.test.ts +0 -143
  193. package/test/src/bucket_validation.ts +0 -60
  194. package/test/src/compacting.test.ts +0 -295
  195. package/test/src/data_storage.test.ts +0 -1569
  196. package/test/src/stream_utils.ts +0 -42
  197. package/test/src/sync.test.ts +0 -511
  198. package/test/src/util.ts +0 -150
@@ -1,6 +1,8 @@
1
- import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
1
+ import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
2
2
 
3
+ import { framework } from '../index.js';
3
4
  import * as metrics from '../metrics/Metrics.js';
5
+ import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
4
6
  import * as replication from '../replication/replication-index.js';
5
7
  import * as routes from '../routes/routes-index.js';
6
8
  import * as storage from '../storage/storage-index.js';
@@ -13,6 +15,7 @@ export interface ServiceContext {
13
15
  replicationEngine: replication.ReplicationEngine | null;
14
16
  routerEngine: routes.RouterEngine | null;
15
17
  storageEngine: storage.StorageEngine;
18
+ migrations: PowerSyncMigrationManager;
16
19
  }
17
20
 
18
21
  /**
@@ -30,13 +33,19 @@ export class ServiceContextContainer implements ServiceContext {
30
33
  this.storageEngine = new storage.StorageEngine({
31
34
  configuration
32
35
  });
36
+
37
+ const migrationManager = new MigrationManager();
38
+ container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager);
39
+
40
+ this.lifeCycleEngine.withLifecycle(migrationManager, {
41
+ // Migrations should be executed before the system starts
42
+ start: () => migrationManager[Symbol.asyncDispose]()
43
+ });
44
+
33
45
  this.lifeCycleEngine.withLifecycle(this.storageEngine, {
34
46
  start: (storageEngine) => storageEngine.start(),
35
47
  stop: (storageEngine) => storageEngine.shutDown()
36
48
  });
37
-
38
- // Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
39
- this.storageEngine.registerProvider(new storage.MongoStorageProvider());
40
49
  }
41
50
 
42
51
  get replicationEngine(): replication.ReplicationEngine | null {
@@ -51,6 +60,10 @@ export class ServiceContextContainer implements ServiceContext {
51
60
  return container.getOptional(metrics.Metrics);
52
61
  }
53
62
 
63
+ get migrations(): PowerSyncMigrationManager {
64
+ return container.getImplementation(framework.ContainerImplementation.MIGRATION_MANAGER);
65
+ }
66
+
54
67
  /**
55
68
  * Allows for registering core and generic implementations of services/helpers.
56
69
  * This uses the framework container under the hood.
@@ -30,8 +30,8 @@ export type SyncRulesConfig = {
30
30
 
31
31
  export type ResolvedPowerSyncConfig = {
32
32
  base_config: PowerSyncConfig;
33
- connections?: configFile.DataSourceConfig[];
34
- storage: configFile.StorageConfig;
33
+ connections?: configFile.GenericDataSourceConfig[];
34
+ storage: configFile.GenericStorageConfig;
35
35
  dev: {
36
36
  demo_auth: boolean;
37
37
  demo_password?: string;
package/src/util/utils.ts CHANGED
@@ -131,7 +131,16 @@ export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
131
131
  return false;
132
132
  }
133
133
 
134
- export function isCompleteRow(row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
134
+ /**
135
+ * Returns true if we have a complete row.
136
+ *
137
+ * If we don't store data, we assume we always have a complete row.
138
+ */
139
+ export function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
140
+ if (!storeData) {
141
+ // Assume the row is complete - no need to check
142
+ return true;
143
+ }
135
144
  return !hasToastedValues(row);
136
145
  }
137
146
 
@@ -199,6 +208,17 @@ export function reduceBucket(operations: OplogEntry[]) {
199
208
  return finalState;
200
209
  }
201
210
 
211
+ /**
212
+ * Flattens string to reduce memory usage (around 320 bytes -> 120 bytes),
213
+ * at the cost of some upfront CPU usage.
214
+ *
215
+ * From: https://github.com/davidmarkclements/flatstr/issues/8
216
+ */
217
+ export function flatstr(s: string) {
218
+ s.match(/\n/g);
219
+ return s;
220
+ }
221
+
202
222
  function rowKey(entry: OplogEntry) {
203
223
  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
204
224
  }
@@ -40,6 +40,17 @@ const privateKeyEdDSA: jose.JWK = {
40
40
  alg: 'EdDSA'
41
41
  };
42
42
 
43
+ const privateKeyECDSA: jose.JWK = {
44
+ use: 'sig',
45
+ kty: 'EC',
46
+ crv: 'P-256',
47
+ kid: 'k3',
48
+ x: 'Y37HQjG1YvlQZ16CzO7UQxgkY_us-NfPxMPcHUDN-PE',
49
+ y: 'W3Jqs5_qlIh2UH79l8L3ApqNu14aFetM5oc9oCjAEaw',
50
+ d: 'p2HQaJApdgaAemVuVsL1hscCFOTd0r9uGxRnzvAelFU',
51
+ alg: 'ES256'
52
+ };
53
+
43
54
  describe('JWT Auth', () => {
44
55
  test('KeyStore basics', async () => {
45
56
  const keys = await StaticKeyCollector.importKeys([sharedKey]);
@@ -372,4 +383,26 @@ describe('JWT Auth', () => {
372
383
 
373
384
  expect(verified.claim).toEqual('test-claim');
374
385
  });
386
+
387
+ test('signing with ECDSA', async () => {
388
+ const keys = await StaticKeyCollector.importKeys([privateKeyECDSA]);
389
+ const store = new KeyStore(keys);
390
+ const signKey = (await jose.importJWK(privateKeyECDSA)) as jose.KeyLike;
391
+
392
+ const signedJwt = await new jose.SignJWT({ claim: 'test-claim-2' })
393
+ .setProtectedHeader({ alg: 'ES256', kid: 'k3' })
394
+ .setSubject('f1')
395
+ .setIssuedAt()
396
+ .setIssuer('tester')
397
+ .setAudience('tests')
398
+ .setExpirationTime('5m')
399
+ .sign(signKey);
400
+
401
+ const verified = (await store.verifyJwt(signedJwt, {
402
+ defaultAudiences: ['tests'],
403
+ maxAge: '6m'
404
+ })) as JwtPayload & { claim: string };
405
+
406
+ expect(verified.claim).toEqual('test-claim-2');
407
+ });
375
408
  });
package/test/src/env.ts CHANGED
@@ -1,6 +1,5 @@
1
1
  import { utils } from '@powersync/lib-services-framework';
2
2
 
3
3
  export const env = utils.collectEnvironmentVariables({
4
- MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
5
4
  CI: utils.type.boolean.default('false')
6
5
  });
@@ -20,7 +20,12 @@ vi.mock('@powersync/lib-services-framework', () => ({
20
20
  afterSend: () => Promise<void>;
21
21
  __micro_router_response = true;
22
22
 
23
- constructor({ status, data, headers, afterSend }: {
23
+ constructor({
24
+ status,
25
+ data,
26
+ headers,
27
+ afterSend
28
+ }: {
24
29
  status?: number;
25
30
  data: any;
26
31
  headers?: Record<string, string>;