@powersync/service-core 0.18.1 → 1.7.1

package/dist/util/protocol-types.d.ts

@@ -1,5 +1,5 @@
 import * as t from 'ts-codec';
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import { BucketDescription, BucketPriority, SqliteJsonValue } from '@powersync/service-sync-rules';
 export declare const BucketRequest: t.ObjectCodec<{
     name: t.IdentityCodec<t.CodecType.String>;
     /**
@@ -52,7 +52,7 @@ export interface StreamingSyncCheckpointDiff {
     checkpoint_diff: {
         last_op_id: OpId;
         write_checkpoint?: OpId;
-        updated_buckets: BucketChecksum[];
+        updated_buckets: BucketChecksumWithDescription[];
         removed_buckets: string[];
     };
 }
@@ -64,9 +64,16 @@ export interface StreamingSyncCheckpointComplete {
         last_op_id: OpId;
     };
 }
+export interface StreamingSyncCheckpointPartiallyComplete {
+    partial_checkpoint_complete: {
+        last_op_id: OpId;
+        priority: BucketPriority;
+    };
+}
 export interface StreamingSyncKeepalive {
+    token_expires_in: number;
 }
-export type StreamingSyncLine = StreamingSyncData | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncKeepalive;
+export type StreamingSyncLine = StreamingSyncData | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncCheckpointPartiallyComplete | StreamingSyncKeepalive;
 /**
  * 64-bit unsigned number, as a base-10 string.
  */
@@ -74,7 +81,7 @@ export type OpId = string;
 export interface Checkpoint {
     last_op_id: OpId;
     write_checkpoint?: OpId;
-    buckets: BucketChecksum[];
+    buckets: BucketChecksumWithDescription[];
 }
 export interface BucketState {
     bucket: string;
@@ -119,3 +126,5 @@ export interface BucketChecksum {
      */
     count: number;
 }
+export interface BucketChecksumWithDescription extends BucketChecksum, BucketDescription {
+}

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.18.1",
+  "version": "1.7.1",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -18,7 +18,7 @@
     "@opentelemetry/sdk-metrics": "1.24.1",
     "async": "^3.2.4",
     "async-mutex": "^0.5.0",
-    "bson": "^6.8.0",
+    "bson": "^6.10.3",
     "commander": "^12.0.0",
     "cors": "^2.8.5",
     "ipaddr.js": "^2.1.0",
@@ -32,10 +32,10 @@
     "uuid": "^9.0.1",
     "winston": "^3.13.0",
     "yaml": "^2.3.2",
-    "@powersync/lib-services-framework": "0.5.1",
+    "@powersync/lib-services-framework": "0.5.3",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-rsocket-router": "0.0.18",
-    "@powersync/service-sync-rules": "0.23.4",
+    "@powersync/service-rsocket-router": "0.0.20",
+    "@powersync/service-sync-rules": "0.24.0",
     "@powersync/service-types": "0.8.0"
   },
   "devDependencies": {

package/src/api/RouteAPI.ts

@@ -1,6 +1,6 @@
 import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
 import * as types from '@powersync/service-types';
-import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/BucketStorage.js';
+import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
 
 export interface PatternResult {
   schema: string;

package/src/api/diagnostics.ts

@@ -57,7 +57,7 @@ export async function getSyncRulesStatus(
   // This method can run under some situations if no connection is configured yet.
   // It will return a default tag in such a case. This default tag is not module specific.
   const tag = sourceConfig.tag ?? DEFAULT_TAG;
-  using systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
+  const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
   const status = await systemStorage?.getStatus();
   let replication_lag_bytes: number | undefined = undefined;
 

package/src/entry/commands/compact-action.ts

@@ -50,14 +50,13 @@ export function registerCompactAction(program: Command) {
       await serviceContext.lifeCycleEngine.start();
       const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
 
-      const active = await bucketStorage.getActiveSyncRulesContent();
+      const active = await bucketStorage.getActiveStorage();
       if (active == null) {
         logger.info('No active instance to compact');
         return;
       }
-      using p = bucketStorage.getInstance(active);
       logger.info('Performing compaction...');
-      await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
+      await active.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
       logger.info('Successfully compacted storage.');
     } catch (e) {
       logger.error(`Failed to compact: ${e.toString()}`);

package/src/modules/AbstractModule.ts

@@ -1,7 +1,7 @@
 import { ServiceContextContainer } from '../system/ServiceContext.js';
 import { logger } from '@powersync/lib-services-framework';
 import winston from 'winston';
-import { PersistedSyncRulesContent } from '../storage/BucketStorage.js';
+import { PersistedSyncRulesContent } from '../storage/storage-index.js';
 
 export interface TearDownOptions {
   /**

package/src/replication/AbstractReplicator.ts

@@ -193,16 +193,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
 
     this.replicationJobs = newJobs;
 
-    // Terminate any orphaned jobs that no longer have sync rules
+    // Stop any orphaned jobs that no longer have sync rules.
+    // Termination happens below
     for (let job of existingJobs.values()) {
       // Old - stop and clean up
       try {
         await job.stop();
-        await this.terminateSyncRules(job.storage);
-        job.storage[Symbol.dispose]();
       } catch (e) {
         // This will be retried
-        this.logger.warn('Failed to terminate old replication job}', e);
+        this.logger.warn('Failed to stop old replication job}', e);
       }
     }
 
@@ -210,7 +209,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     const stopped = await this.storage.getStoppedSyncRules();
     for (let syncRules of stopped) {
       try {
-        using syncRuleStorage = this.storage.getInstance(syncRules);
+        const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
         await this.terminateSyncRules(syncRuleStorage);
       } catch (e) {
         this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
@@ -224,13 +223,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
 
   protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
     this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
-    try {
-      await this.cleanUp(syncRuleStorage);
-      await syncRuleStorage.terminate();
-      this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
-    } catch (e) {
-      this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
-    }
+    await this.cleanUp(syncRuleStorage);
+    await syncRuleStorage.terminate();
+    this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
   }
 
   abstract testConnection(): Promise<ConnectionTestResult>;

package/src/routes/endpoints/checkpointing.ts

@@ -1,7 +1,6 @@
 import { logger, router, schema } from '@powersync/lib-services-framework';
 import * as t from 'ts-codec';
 
-import * as framework from '@powersync/lib-services-framework';
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
@@ -31,8 +30,9 @@ export const writeCheckpoint = routeDefinition({
 
     logger.info(`Waiting for LSN checkpoint: ${head}`);
     while (Date.now() - start < timeout) {
-      const cp = await service_context.storageEngine.activeBucketStorage.getActiveCheckpoint();
-      if (!cp.hasSyncRules()) {
+      const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
+      const cp = await bucketStorage?.getCheckpoint();
+      if (cp == null) {
         throw new Error('No sync rules available');
       }
       if (cp.lsn && cp.lsn >= head) {

package/src/routes/endpoints/socket-route.ts

@@ -49,9 +49,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       const {
         storageEngine: { activeBucketStorage }
       } = service_context;
-      // Sanity check before we start the stream
-      const cp = await activeBucketStorage.getActiveCheckpoint();
-      if (!cp.hasSyncRules()) {
+
+      const bucketStorage = await activeBucketStorage.getActiveStorage();
+      if (bucketStorage == null) {
         responder.onError(
           new errors.ServiceError({
             status: 500,
@@ -63,6 +63,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
+      const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions());
+
       const removeStopHandler = routerEngine!.addStopHandler(() => {
         controller.abort();
       });
@@ -71,8 +73,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       const tracker = new sync.RequestTracker();
       try {
         for await (const data of sync.streamResponse({
-          storage: activeBucketStorage,
-          parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
+          bucketStorage: bucketStorage,
+          syncRules: syncRules,
           params: {
             ...params,
             binary_data: true // always true for web sockets

package/src/routes/endpoints/sync-stream.ts

@@ -36,15 +36,18 @@ export const syncStreamed = routeDefinition({
     const params: util.StreamingSyncRequest = payload.params;
     const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
 
-    // Sanity check before we start the stream
-    const cp = await storageEngine.activeBucketStorage.getActiveCheckpoint();
-    if (!cp.hasSyncRules()) {
+    const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage();
+
+    if (bucketStorage == null) {
       throw new errors.ServiceError({
         status: 500,
         code: ErrorCode.PSYNC_S2302,
         description: 'No sync rules available'
       });
     }
+
+    const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions());
+
     const controller = new AbortController();
     const tracker = new sync.RequestTracker();
     try {
@@ -53,8 +56,8 @@ export const syncStreamed = routeDefinition({
         sync.transformToBytesTracked(
           sync.ndjson(
             sync.streamResponse({
-              storage: storageEngine.activeBucketStorage,
-              parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
+              bucketStorage,
+              syncRules: syncRules,
               params,
               syncParams,
               token: payload.context.token_payload!,

package/src/runner/teardown.ts

@@ -51,7 +51,7 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
 
       // Mark the sync rules as terminated
       for (let syncRules of combinedSyncRules) {
-        using syncRulesStorage = storageFactory.getInstance(syncRules);
+        const syncRulesStorage = storageFactory.getInstance(syncRules);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here
         await syncRulesStorage.terminate({ clearStorage: false });
       }