@powersync/service-core 0.0.0-dev-20250304151813 → 0.0.0-dev-20250306152715

package/src/sync/BucketChecksumState.ts

@@ -13,7 +13,7 @@ export interface BucketChecksumStateOptions {
   bucketStorage: BucketChecksumStateStorage;
   syncRules: SqlSyncRules;
   syncParams: RequestParameters;
-  initialBucketPositions?: { name: string; after: string }[];
+  initialBucketPositions?: { name: string; after: util.InternalOpId }[];
 }
 
 /**
@@ -78,7 +78,7 @@ export class BucketChecksumState {
     for (let bucket of allBuckets) {
       dataBucketsNew.set(bucket.bucket, {
         description: bucket,
-        start_op_id: this.bucketDataPositions.get(bucket.bucket)?.start_op_id ?? '0'
+        start_op_id: this.bucketDataPositions.get(bucket.bucket)?.start_op_id ?? 0n
       });
     }
     this.bucketDataPositions = dataBucketsNew;
@@ -180,7 +180,7 @@ export class BucketChecksumState {
 
       checkpointLine = {
         checkpoint_diff: {
-          last_op_id: base.checkpoint,
+          last_op_id: util.internalToExternalOpId(base.checkpoint),
           write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
           removed_buckets: diff.removedBuckets,
           updated_buckets: updatedBucketDescriptions
@@ -193,7 +193,7 @@ export class BucketChecksumState {
       bucketsToFetch = allBuckets;
       checkpointLine = {
         checkpoint: {
-          last_op_id: base.checkpoint,
+          last_op_id: util.internalToExternalOpId(base.checkpoint),
           write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
           buckets: [...checksumMap.values()].map((e) => ({
             ...e,
@@ -219,8 +219,8 @@ export class BucketChecksumState {
    * @param bucketsToFetch List of buckets to fetch, typically from buildNextCheckpointLine, or a subset of that
    * @returns
    */
-  getFilteredBucketPositions(bucketsToFetch: BucketDescription[]): Map<string, string> {
-    const filtered = new Map<string, string>();
+  getFilteredBucketPositions(bucketsToFetch: BucketDescription[]): Map<string, util.InternalOpId> {
+    const filtered = new Map<string, util.InternalOpId>();
     for (let bucket of bucketsToFetch) {
       const state = this.bucketDataPositions.get(bucket.bucket);
       if (state) {
@@ -236,7 +236,7 @@ export class BucketChecksumState {
    * @param bucket the bucket name
    * @param nextAfter sync operations >= this value in the next batch
    */
-  updateBucketPosition(options: { bucket: string; nextAfter: string; hasMore: boolean }) {
+  updateBucketPosition(options: { bucket: string; nextAfter: util.InternalOpId; hasMore: boolean }) {
     const state = this.bucketDataPositions.get(options.bucket);
     if (state) {
       state.start_op_id = options.nextAfter;

package/src/sync/RequestTracker.ts

@@ -1,4 +1,6 @@
-import { Metrics } from '../metrics/Metrics.js';
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
+
+import { APIMetric } from '@powersync/service-types';
 
 /**
  * Record sync stats per request stream.
@@ -7,15 +9,19 @@ export class RequestTracker {
   operationsSynced = 0;
   dataSyncedBytes = 0;
 
+  constructor(private metrics: MetricsEngine) {
+    this.metrics = metrics;
+  }
+
   addOperationsSynced(operations: number) {
     this.operationsSynced += operations;
 
-    Metrics.getInstance().operations_synced_total.add(operations);
+    this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED_TOTAL).add(operations);
   }
 
   addDataSynced(bytes: number) {
     this.dataSyncedBytes += bytes;
 
-    Metrics.getInstance().data_synced_bytes.add(bytes);
+    this.metrics.getCounter(APIMetric.DATA_SYNCED_BYTES).add(bytes);
   }
 }

package/src/sync/sync.ts

@@ -79,7 +79,7 @@ export async function* streamResponse(
 
 export type BucketSyncState = {
   description?: BucketDescription; // Undefined if the bucket has not yet been resolved by us.
-  start_op_id: string;
+  start_op_id: util.InternalOpId;
 };
 
 async function* streamResponseInner(
@@ -100,7 +100,10 @@ async function* streamResponseInner(
     bucketStorage,
     syncRules,
     syncParams,
-    initialBucketPositions: params.buckets
+    initialBucketPositions: params.buckets?.map((bucket) => ({
+      name: bucket.name,
+      after: BigInt(bucket.after)
+    }))
   });
   const stream = bucketStorage.watchWriteCheckpoint({
     user_id: checkpointUserId,
@@ -218,7 +221,7 @@ async function* streamResponseInner(
 interface BucketDataRequest {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
-  checkpoint: string;
+  checkpoint: util.InternalOpId;
   bucketsToFetch: BucketDescription[];
   /** Contains current bucket state. Modified by the request as data is sent.  */
   checksumState: BucketChecksumState;
@@ -290,7 +293,6 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     onRowsSent
   } = request;
 
-  const checkpointOp = BigInt(checkpoint);
   let checkpointInvalidated = false;
 
   if (syncContext.syncSemaphore.isLocked()) {
@@ -326,7 +328,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       if (r.has_more) {
         has_more = true;
       }
-      if (targetOp != null && targetOp > checkpointOp) {
+      if (targetOp != null && targetOp > checkpoint) {
         checkpointInvalidated = true;
       }
       if (r.data.length == 0) {
@@ -362,7 +364,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       }
       onRowsSent(r.data.length);
 
-      checksumState.updateBucketPosition({ bucket: r.bucket, nextAfter: r.next_after, hasMore: r.has_more });
+      checksumState.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
 
       // Check if syncing bucket data is supposed to stop before fetching more data
       // from storage.
@@ -381,7 +383,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         if (request.forPriority != null) {
           const line: util.StreamingSyncCheckpointPartiallyComplete = {
             partial_checkpoint_complete: {
-              last_op_id: checkpoint,
+              last_op_id: util.internalToExternalOpId(checkpoint),
               priority: request.forPriority
             }
           };
@@ -389,7 +391,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         } else {
           const line: util.StreamingSyncCheckpointComplete = {
             checkpoint_complete: {
-              last_op_id: checkpoint
+              last_op_id: util.internalToExternalOpId(checkpoint)
             }
           };
           yield { data: line, done: true };

package/src/system/ServiceContext.ts

@@ -1,7 +1,7 @@
 import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
 
 import { framework } from '../index.js';
-import * as metrics from '../metrics/Metrics.js';
+import * as metrics from '../metrics/MetricsEngine.js';
 import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
 import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
@@ -12,7 +12,7 @@ import { SyncContext } from '../sync/SyncContext.js';
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
   lifeCycleEngine: LifeCycledSystem;
-  metrics: metrics.Metrics | null;
+  metricsEngine: metrics.MetricsEngine;
   replicationEngine: replication.ReplicationEngine | null;
   routerEngine: routes.RouterEngine | null;
   storageEngine: storage.StorageEngine;
@@ -37,6 +37,11 @@ export class ServiceContextContainer implements ServiceContext {
       configuration
     });
 
+    this.lifeCycleEngine.withLifecycle(this.storageEngine, {
+      start: (storageEngine) => storageEngine.start(),
+      stop: (storageEngine) => storageEngine.shutDown()
+    });
+
     this.syncContext = new SyncContext({
       maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency,
       maxBuckets: configuration.api_parameters.max_buckets_per_connection,
@@ -65,8 +70,8 @@ export class ServiceContextContainer implements ServiceContext {
     return container.getOptional(routes.RouterEngine);
   }
 
-  get metrics(): metrics.Metrics | null {
-    return container.getOptional(metrics.Metrics);
+  get metricsEngine(): metrics.MetricsEngine {
+    return container.getImplementation(metrics.MetricsEngine);
   }
 
   get migrations(): PowerSyncMigrationManager {

package/src/util/protocol-types.ts

@@ -57,8 +57,8 @@ export interface StreamingSyncCheckpoint {
 
 export interface StreamingSyncCheckpointDiff {
   checkpoint_diff: {
-    last_op_id: OpId;
-    write_checkpoint?: OpId;
+    last_op_id: ProtocolOpId;
+    write_checkpoint?: ProtocolOpId;
     updated_buckets: BucketChecksumWithDescription[];
     removed_buckets: string[];
   };
@@ -70,13 +70,13 @@ export interface StreamingSyncData {
 
 export interface StreamingSyncCheckpointComplete {
   checkpoint_complete: {
-    last_op_id: OpId;
+    last_op_id: ProtocolOpId;
   };
 }
 
 export interface StreamingSyncCheckpointPartiallyComplete {
   partial_checkpoint_complete: {
-    last_op_id: OpId;
+    last_op_id: ProtocolOpId;
     priority: BucketPriority;
   };
 }
@@ -96,11 +96,11 @@ export type StreamingSyncLine =
 /**
  * 64-bit unsigned number, as a base-10 string.
  */
-export type OpId = string;
+export type ProtocolOpId = string;
 
 export interface Checkpoint {
-  last_op_id: OpId;
-  write_checkpoint?: OpId;
+  last_op_id: ProtocolOpId;
+  write_checkpoint?: ProtocolOpId;
   buckets: BucketChecksumWithDescription[];
 }
 
@@ -123,15 +123,15 @@ export interface SyncBucketData {
   /**
    * The `after` specified in the request.
    */
-  after: OpId;
+  after: ProtocolOpId;
   /**
    * Use this for the next request.
    */
-  next_after: OpId;
+  next_after: ProtocolOpId;
 }
 
 export interface OplogEntry {
-  op_id: OpId;
+  op_id: ProtocolOpId;
   op: 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
   object_type?: string;
   object_id?: string;

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, OpId, OplogEntry } from './protocol-types.js';
+import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
@@ -11,6 +11,13 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+/**
+ * op_id as used internally, for individual operations and checkpoints.
+ *
+ * This is just a type alias, but serves to document that we're working with an op_id.
+ */
+export type InternalOpId = bigint;
+
 export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
 
 export function escapeIdentifier(identifier: string) {
@@ -31,7 +38,11 @@ export function hashDelete(sourceKey: string) {
   return buffer.readUInt32LE(0);
 }
 
-export function timestampToOpId(ts: bigint): OpId {
+/**
+ * Internally we always use bigint for op_ids. Externally (in JSON) we use strings.
+ * This converts between the two.
+ */
+export function internalToExternalOpId(ts: InternalOpId): ProtocolOpId {
   // Dynamic values are passed in in some cases, so we make extra sure that the
   // number is a bigint and not number or Long.
   if (typeof ts != 'bigint') {