@powersync/service-core 0.0.2 → 0.1.1

package/src/replication/WalStream.ts

@@ -7,10 +7,10 @@ import { SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersyn
 import * as storage from '@/storage/storage-index.js';
 import * as util from '@/util/util-index.js';
 
-import { chunks_replicated_total, rows_replicated_total, transactions_replicated_total } from '../metrics/metrics.js';
 import { getPgOutputRelation, getRelId, PgRelation } from './PgRelation.js';
 import { getReplicationIdentityColumns } from './util.js';
 import { WalConnection } from './WalConnection.js';
+import { Metrics } from '@/metrics/Metrics.js';
 
 export const ZERO_LSN = '00000000/00000000';
 
@@ -403,7 +403,7 @@ WHERE  oid = $1::regclass`,
         await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record });
       }
       at += rows.length;
-      rows_replicated_total.add(rows.length);
+      Metrics.getInstance().rows_replicated_total.add(rows.length);
 
       // pgwire streaming uses reasonable chunk sizes, so we flush at the end
       // of each chunk.
@@ -493,18 +493,18 @@ WHERE  oid = $1::regclass`,
       }
 
       if (msg.tag == 'insert') {
-        rows_replicated_total.add(1);
+        Metrics.getInstance().rows_replicated_total.add(1);
         const baseRecord = util.constructAfterRecord(msg);
         return await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: baseRecord });
       } else if (msg.tag == 'update') {
-        rows_replicated_total.add(1);
+        Metrics.getInstance().rows_replicated_total.add(1);
         // "before" may be null if the replica id columns are unchanged
         // It's fine to treat that the same as an insert.
         const before = util.constructBeforeRecord(msg);
         const after = util.constructAfterRecord(msg);
         return await batch.save({ tag: 'update', sourceTable: table, before: before, after: after });
       } else if (msg.tag == 'delete') {
-        rows_replicated_total.add(1);
+        Metrics.getInstance().rows_replicated_total.add(1);
         const before = util.constructBeforeRecord(msg)!;
 
         return await batch.save({ tag: 'delete', sourceTable: table, before: before, after: undefined });
@@ -577,7 +577,7 @@ WHERE  oid = $1::regclass`,
           } else if (msg.tag == 'begin') {
             inTx = true;
           } else if (msg.tag == 'commit') {
-            transactions_replicated_total.add(1);
+            Metrics.getInstance().transactions_replicated_total.add(1);
             inTx = false;
             await batch.commit(msg.lsn!);
             await this.ack(msg.lsn!, replicationStream);
@@ -602,7 +602,7 @@ WHERE  oid = $1::regclass`,
           }
         }
 
-        chunks_replicated_total.add(1);
+        Metrics.getInstance().chunks_replicated_total.add(1);
       }
     });
   }

package/src/routes/socket-route.ts

@@ -3,10 +3,10 @@ import { SyncParameters, normalizeTokenParameters } from '@powersync/service-syn
 import * as micro from '@journeyapps-platform/micro';
 
 import * as util from '@/util/util-index.js';
-import { concurrent_connections, data_synced_bytes } from '../metrics/metrics.js';
 import { streamResponse } from '../sync/sync.js';
 import { SyncRoutes } from './sync-stream.js';
 import { SocketRouteGenerator } from './router-socket.js';
+import { Metrics } from '@/metrics/Metrics.js';
 
 export const sync_stream_reactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
@@ -64,7 +64,7 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
         observer.triggerCancel();
       });
 
-      concurrent_connections.add(1);
+      Metrics.getInstance().concurrent_connections.add(1);
       try {
         for await (const data of streamResponse({
           storage,
@@ -93,7 +93,7 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
             const serialized = serialize(data) as Buffer;
             responder.onNext({ data: serialized }, false);
             requestedN--;
-            data_synced_bytes.add(serialized.length);
+            Metrics.getInstance().data_synced_bytes.add(serialized.length);
           }
 
           if (requestedN <= 0) {
@@ -125,7 +125,7 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
         responder.onComplete();
         removeStopHandler();
         disposer();
-        concurrent_connections.add(-1);
+        Metrics.getInstance().concurrent_connections.add(-1);
       }
     }
   });

package/src/routes/sync-stream.ts

@@ -4,10 +4,10 @@ import { SyncParameters, normalizeTokenParameters } from '@powersync/service-syn
 
 import * as sync from '@/sync/sync-index.js';
 import * as util from '@/util/util-index.js';
-import { concurrent_connections } from '../metrics/metrics.js';
 
 import { authUser } from './auth.js';
 import { RouteGenerator } from './router.js';
+import { Metrics } from '@/metrics/Metrics.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -49,7 +49,7 @@ export const syncStreamed: RouteGenerator = (router) =>
 
       const controller = new AbortController();
       try {
-        concurrent_connections.add(1);
+        Metrics.getInstance().concurrent_connections.add(1);
         const stream = Readable.from(
           sync.transformToBytesTracked(
             sync.ndjson(
@@ -84,7 +84,7 @@ export const syncStreamed: RouteGenerator = (router) =>
         await res.send(stream);
       } finally {
         controller.abort();
-        concurrent_connections.add(-1);
+        Metrics.getInstance().concurrent_connections.add(-1);
         // Prevent double-send
         res.hijack();
       }

package/src/storage/BucketStorage.ts

@@ -91,6 +91,11 @@ export interface BucketStorageFactory {
    * Get storage size of active sync rules.
    */
   getStorageMetrics(): Promise<StorageMetrics>;
+
+  /**
+   * Get the unique identifier for this instance of Powersync
+   */
+  getPowerSyncInstanceId(): Promise<string>;
 }
 
 export interface WriteCheckpoint {

package/src/storage/MongoBucketStorage.ts

@@ -23,6 +23,8 @@ import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
 import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js';
 import { SyncRuleDocument, SyncRuleState } from './mongo/models.js';
 import { generateSlotName } from './mongo/util.js';
+import { locks } from '@journeyapps-platform/micro';
+import { v4 as uuid } from 'uuid';
 
 export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {}
 
@@ -341,6 +343,30 @@ export class MongoBucketStorage implements BucketStorageFactory {
     };
   }
 
+  async getPowerSyncInstanceId(): Promise<string> {
+    let instance = await this.db.instance.findOne({
+      _id: { $exists: true }
+    });
+
+    if (!instance) {
+      const manager = locks.createMongoLockManager(this.db.locks, {
+          name: `instance-id-insertion-lock`
+      });
+
+      await manager.lock(async () => {
+        await this.db.instance.insertOne({
+          _id: uuid(),
+        });
+      });
+
+      instance = await this.db.instance.findOne({
+        _id: { $exists: true }
+      });
+    }
+
+    return instance!._id;
+  }
+
   private makeActiveCheckpoint(doc: SyncRuleDocument | null) {
     return {
       checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),

package/src/storage/mongo/db.ts

@@ -1,4 +1,5 @@
 import * as mongo from 'mongodb';
+import * as micro from '@journeyapps-platform/micro';
 
 import * as db from '@/db/db-index.js';
 
@@ -7,6 +8,7 @@ import {
   BucketParameterDocument,
   CurrentDataDocument,
   IdSequenceDocument,
+  InstanceDocument,
   SourceTableDocument,
   SyncRuleDocument,
   WriteCheckpointDocument
@@ -33,6 +35,8 @@ export class PowerSyncMongo {
   readonly sync_rules: mongo.Collection<SyncRuleDocument>;
   readonly source_tables: mongo.Collection<SourceTableDocument>;
   readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
+  readonly instance: mongo.Collection<InstanceDocument>;
+  readonly locks: mongo.Collection<micro.locks.Lock>;
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
@@ -55,6 +59,8 @@ export class PowerSyncMongo {
     this.sync_rules = db.collection('sync_rules');
     this.source_tables = db.collection('source_tables');
     this.write_checkpoints = db.collection('write_checkpoints');
+    this.instance = db.collection('instance');
+    this.locks = this.db.collection('locks');
   }
 
   async clear() {
@@ -65,5 +71,7 @@ export class PowerSyncMongo {
     await this.sync_rules.deleteMany({});
     await this.source_tables.deleteMany({});
     await this.write_checkpoints.deleteMany({});
+    await this.instance.deleteOne({});
+    await this.locks.deleteMany({});
   }
 }

package/src/storage/mongo/models.ts

@@ -155,3 +155,8 @@ export interface WriteCheckpointDocument {
   lsns: Record<string, string>;
   client_id: bigint;
 }
+
+export interface InstanceDocument {
+  // The instance UUID
+  _id: string;
+}

package/src/sync/sync.ts

@@ -8,9 +8,9 @@ import * as auth from '@/auth/auth-index.js';
 import * as storage from '@/storage/storage-index.js';
 import * as util from '@/util/util-index.js';
 
-import { operations_synced_total } from '../metrics/metrics.js';
 import { mergeAsyncIterables } from './merge.js';
 import { TokenStreamOptions, tokenStream } from './util.js';
+import { Metrics } from '@/metrics/Metrics.js';
 
 /**
  * Maximum number of connections actively fetching data.
@@ -265,7 +265,7 @@ async function* bucketDataBatch(
         // iterator memory in case if large data sent.
         yield { data: null, done: false };
       }
-      operations_synced_total.add(r.data.length);
+      Metrics.getInstance().operations_synced_total.add(r.data.length);
 
       dataBuckets.set(r.bucket, r.next_after);
     }

package/src/sync/util.ts

@@ -1,7 +1,7 @@
 import * as timers from 'timers/promises';
 
 import * as util from '@/util/util-index.js';
-import { data_synced_bytes } from '../metrics/metrics.js';
+import { Metrics } from '@/metrics/Metrics.js';
 
 export type TokenStreamOptions = {
   /**
@@ -92,7 +92,7 @@ export async function* ndjson(iterator: AsyncIterable<string | null | Record<str
 export async function* transformToBytesTracked(iterator: AsyncIterable<string>): AsyncGenerator<Buffer> {
   for await (let data of iterator) {
     const encoded = Buffer.from(data, 'utf8');
-    data_synced_bytes.add(encoded.length);
+    Metrics.getInstance().data_synced_bytes.add(encoded.length);
     yield encoded;
   }
 }

package/src/util/config/compound-config-collector.ts

@@ -119,6 +119,10 @@ export class CompoundConfigCollector {
       token_max_expiration: '1d', // 1 day
       metadata: baseConfig.metadata ?? {},
       migrations: baseConfig.migrations,
+      telemetry: {
+        disable_telemetry_sharing: baseConfig.telemetry?.disable_telemetry_sharing ?? false,
+        internal_service_endpoint: baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
+      },
       slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
     };
     return config;

package/src/util/config/types.ts

@@ -55,6 +55,12 @@ export type ResolvedPowerSyncConfig = {
   migrations?: {
     disable_auto_migration?: boolean;
   };
+
+  telemetry: {
+    disable_telemetry_sharing: boolean;
+    internal_service_endpoint: string;
+  };
+
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
 };

package/test/src/util.ts

@@ -6,6 +6,15 @@ import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
 import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
 import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
 import { env } from './env.js';
+import { Metrics } from '@/metrics/Metrics.js';
+
+// The metrics need to be initialised before they can be used
+await Metrics.initialise({
+  disable_telemetry_sharing: true,
+  powersync_instance_id: 'test',
+  internal_metrics_endpoint: 'unused.for.tests.com'
+});
+Metrics.getInstance().resetCounters();
 
 export const TEST_URI = env.PG_TEST_URL;
 

package/test/src/wal_stream.test.ts

@@ -1,10 +1,10 @@
 import * as crypto from 'crypto';
 import { describe, expect, test } from 'vitest';
-import { BucketStorageFactory } from '../../src/storage/BucketStorage.js';
+import { BucketStorageFactory } from '@/storage/BucketStorage.js';
 import { MONGO_STORAGE_FACTORY } from './util.js';
 import { putOp, removeOp, walStreamTest } from './wal_stream_utils.js';
 import { pgwireRows } from '@powersync/service-jpgwire';
-import { getMetricValueForTests } from '../../src/metrics/metrics.js';
+import { Metrics } from '@/metrics/Metrics.js';
 
 type StorageFactory = () => Promise<BucketStorageFactory>;
 
@@ -41,8 +41,10 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const startRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -57,8 +59,9 @@ bucket_definitions:
       expect(data).toMatchObject([
         putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })
       ]);
-      const endRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
       expect(endRowCount - startRowCount).toEqual(1);
       expect(endTxCount - startTxCount).toEqual(1);
     })
@@ -80,8 +83,10 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const startRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -92,8 +97,9 @@ bucket_definitions:
       const data = await context.getBucketData('global[]');
 
       expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
-      const endRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
       expect(endRowCount - startRowCount).toEqual(1);
       expect(endTxCount - startTxCount).toEqual(1);
     })
@@ -287,8 +293,10 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const startRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -299,8 +307,9 @@ bucket_definitions:
       const data = await context.getBucketData('global[]');
 
       expect(data).toMatchObject([]);
-      const endRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       // There was a transaction, but we should not replicate any actual data
       expect(endRowCount - startRowCount).toEqual(0);

package/tsconfig.json

@@ -15,6 +15,18 @@
   "references": [
     {
       "path": "../types"
+    },
+    {
+      "path": "../rsocket-router"
+    },
+    {
+      "path": "../jsonbig"
+    },
+    {
+      "path": "../jpgwire"
+    },
+    {
+      "path": "../sync-rules"
     }
   ]
 }