@powersync/service-core 1.21.0 → 1.22.0

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,10 +1,11 @@
 import { Logger, ObserverClient } from '@powersync/lib-services-framework';
 import {
   BucketDataSource,
-  HydratedSyncRules,
+  HydratedSyncConfig,
   ParameterLookupRows,
   ScopedParameterLookup
 } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
@@ -12,6 +13,7 @@ import { BucketStorageFactory } from './BucketStorageFactory.js';
 import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
+import { StorageVersionConfig } from './StorageVersionConfig.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
 /**
@@ -22,15 +24,11 @@ export interface SyncRulesBucketStorage
     SyncStorageWriteCheckpointAPI {
   readonly group_id: number;
   readonly slot_name: string;
+  readonly storageConfig: StorageVersionConfig;
 
   readonly factory: BucketStorageFactory;
   readonly logger: Logger;
 
-  /**
-   * Resolve a table, keeping track of it internally.
-   */
-  resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
-
   /**
    * Create a new writer.
    *
@@ -46,7 +44,7 @@ export interface SyncRulesBucketStorage
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
-  getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules;
+  getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncConfig;
 
   /**
    * Terminate the replication stream.
@@ -156,18 +154,26 @@ export interface SyncRuleStatus {
   active: boolean;
   snapshot_done: boolean;
   snapshot_lsn: string | null;
+  /**
+   * Last persisted operation that must be included in the next checkpoint once checkpointing is unblocked.
+   */
+  keepalive_op: util.InternalOpId | null;
 }
-export interface ResolveTableOptions {
-  group_id: number;
+export interface ResolveTablesOptions {
   connection_id: number;
-  connection_tag: string;
-  entity_descriptor: SourceEntityDescriptor;
-
-  sync_rules: HydratedSyncRules;
+  source: SourceEntityDescriptor;
+  /**
+   * For tests only - custom id generator for stable ids.
+   */
+  idGenerator?: () => string | bson.ObjectId;
+  /**
+   * For tests only - override the sync rules used.
+   */
+  syncRules?: HydratedSyncConfig;
 }
 
-export interface ResolveTableResult {
-  table: SourceTable;
+export interface ResolveTablesResult {
+  tables: SourceTable[];
   dropTables: SourceTable[];
 }
 
@@ -198,11 +204,18 @@ export interface CreateWriterOptions extends ParseSyncRulesOptions {
    */
   markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
 
+  hooks?: StorageHooks;
+
   tracer?: PerformanceTracer<'storage' | 'evaluate'>;
 
   logger?: Logger;
 }
 
+export interface StorageHooks {
+  beforeBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+  afterBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+}
+
 /**
  * @deprecated Use `CreateWriterOptions`.
  */

package/src/sync/BucketChecksumState.ts

@@ -2,7 +2,7 @@ import {
   BucketParameterQuerier,
   BucketPriority,
   BucketSource,
-  HydratedSyncRules,
+  HydratedSyncConfig,
   mergeBuckets,
   QuerierError,
   RequestedStream,
@@ -28,7 +28,7 @@ import { getIntersection, hasIntersection } from './util.js';
 export interface BucketChecksumStateOptions {
   syncContext: SyncContext;
   bucketStorage: BucketChecksumStateStorage;
-  syncRules: HydratedSyncRules;
+  syncRules: HydratedSyncConfig;
   tokenPayload: JwtPayload;
   syncRequest: util.StreamingSyncRequest;
   logger?: Logger;
@@ -285,7 +285,7 @@ export class BucketChecksumState {
       const streamNameToIndex = new Map<string, number>();
       this.streamNameToIndex = streamNameToIndex;
 
-      for (const source of this.parameterState.syncRules.definition.bucketSources) {
+      for (const source of this.parameterState.syncRules.bucketSourceDefinitions) {
         if (this.parameterState.isSubscribedToStream(source)) {
           streamNameToIndex.set(source.name, subscriptions.length);
 
@@ -416,7 +416,7 @@ export interface CheckpointUpdate {
 export class BucketParameterState {
   private readonly context: SyncContext;
   public readonly bucketStorage: BucketChecksumStateStorage;
-  public readonly syncRules: HydratedSyncRules;
+  public readonly syncRules: HydratedSyncConfig;
   public readonly syncParams: RequestParameters;
   private readonly querier: BucketParameterQuerier;
   /**
@@ -439,7 +439,7 @@ export class BucketParameterState {
   constructor(
     context: SyncContext,
     bucketStorage: BucketChecksumStateStorage,
-    syncRules: HydratedSyncRules,
+    syncRules: HydratedSyncConfig,
     tokenPayload: JwtPayload,
     request: util.StreamingSyncRequest,
     logger: Logger

package/src/sync/sync.ts

@@ -1,5 +1,5 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import { BucketPriority, HydratedSyncRules, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
+import { BucketPriority, HydratedSyncConfig, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr
 export interface SyncStreamParameters {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
-  syncRules: HydratedSyncRules;
+  syncRules: HydratedSyncConfig;
   params: util.StreamingSyncRequest;
   token: auth.JwtPayload;
   logger?: Logger;
@@ -94,7 +94,7 @@ export async function* streamResponse(
 async function* streamResponseInner(
   syncContext: SyncContext,
   bucketStorage: storage.SyncRulesBucketStorage,
-  syncRules: HydratedSyncRules,
+  syncRules: HydratedSyncConfig,
   params: util.StreamingSyncRequest,
   tokenPayload: auth.JwtPayload,
   tracker: RequestTracker,

package/src/tracing/PerformanceTracer.ts

@@ -2,19 +2,38 @@ import { traceWriter } from './TraceWriter.js';
 
 export interface Span extends Disposable {
   name: string;
+  /**
+   * Start time in microseconds since an arbitrary epoch.
+   */
   startAt: number;
+  /**
+   * End time in microseconds since an arbitrary epoch.
+   */
   endAt: number;
+  /**
+   * Time spent not in nested spans.
+   */
   selfDuration: number;
+
   nestedSince: number | undefined;
   subtrackFromSelf: number;
+
+  /**
+   * Durations spent in nested spans, in microseconds.
+   */
   nestedDurations: Record<string, number>;
 
+  /**
+   * Total duration of this span in milliseconds, rounded up. Only valid after the span has ended.
+   */
+  durationMillis: number;
+
   /**
    * End the span - same as [Symbol.dispose]().
    *
    * Safe to call multiple times. Any nested spans will automatically end as well.
    *
-   * Returns an aggregate record of category -> "selfDuration".
+   * Returns an aggregate record of category -> "selfDuration", in microseconds.
    */
   end(): Record<string, number>;
 }
@@ -115,6 +134,10 @@ export class PerformanceTracer<K extends string> {
         }
         return this.nestedDurations;
       },
+
+      get durationMillis() {
+        return Math.ceil((this.endAt - this.startAt) / 1000);
+      },
       [Symbol.dispose]() {
         this.end();
       }

package/src/util/util-index.ts

@@ -17,6 +17,7 @@ export * from './config/collectors/config-collector.js';
 export * from './config/collectors/impl/base64-config-collector.js';
 export * from './config/collectors/impl/fallback-config-collector.js';
 export * from './config/collectors/impl/filesystem-config-collector.js';
+export * from './config/collectors/impl/yaml-env.js';
 
 export * from './config/sync-rules/impl/base64-sync-rules-collector.js';
 export * from './config/sync-rules/impl/filesystem-sync-rules-collector.js';

package/src/util/utils.ts

@@ -39,6 +39,14 @@ export function escapeIdentifier(identifier: string) {
   return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
 }
 
+/**
+ *  Sanitized name of the entity in the format of "{schema}.{entity name}"
+ *  Suitable for safe use in Postgres queries and in log output.
+ */
+export function qualifiedName(ref: sync_rules.SourceTableRef) {
+  return `${escapeIdentifier(ref.schema)}.${escapeIdentifier(ref.name)}`;
+}
+
 export function hashData(type: string, id: string, data: string): number {
   const hash = crypto.createHash('sha256');
   hash.update(`put.${type}.${id}.${data}`);

package/test/src/auth.test.ts

@@ -484,6 +484,17 @@ describe('JWT Auth', () => {
           }
         })
     ).toThrowError('IPs in this range are not supported');
+
+    // `URL.hostname` exposes IPv6 literals wrapped in brackets; ensure they are
+    // handled like other direct-IP cases.
+    expect(
+      () =>
+        new RemoteJWKSCollector('https://[::1]/.well-known/jwks.json', {
+          lookupOptions: {
+            reject_ip_ranges: ['local']
+          }
+        })
+    ).toThrowError('IPs in this range are not supported');
   });
 
   test('http not blocking local IPs', async () => {

package/test/src/diagnostics.test.ts

@@ -1,6 +1,6 @@
 import { DiagnosticsOptions, getSyncRulesStatus } from '@/api/diagnostics.js';
 import { RouteAPI, SlotWalBudgetInfo } from '@/api/RouteAPI.js';
-import { BucketStorageFactory } from '@/index.js';
+import { BucketStorageFactory, PersistedSyncRules, storage } from '@/index.js';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 
@@ -13,7 +13,8 @@ bucket_definitions:
       - SELECT id FROM test_table
 `;
 
-function makeSyncRulesContent(overrides?: { slot_name?: string }) {
+function makeSyncRulesContent(overrides?: { slot_name?: string }): storage.PersistedSyncRulesContent {
+  // We don't implement the entire interface correctly here - just enough to test the diagnostics logic.
   return {
     id: 1,
     slot_name: overrides?.slot_name ?? 'test_slot',
@@ -32,14 +33,17 @@ function makeSyncRulesContent(overrides?: { slot_name?: string }) {
         defaultSchema: 'public'
       });
       return {
-        sync_rules: syncRules
-      };
+        syncConfigWithErrors: syncRules
+      } as PersistedSyncRules;
     },
     lock() {
       throw new Error('Not implemented in mock');
     },
-    current_lock: undefined
-  } as any;
+    current_lock: null,
+    logger: null as any,
+    asUpdateOptions: null as any,
+    getStorageConfig: null as any
+  } as storage.PersistedSyncRulesContent;
 }
 
 function makeBucketStorage() {

package/test/src/routes/error-handler.integration.test.ts

@@ -0,0 +1,275 @@
+import { errors } from '@powersync/lib-services-framework';
+import Fastify, { FastifyInstance } from 'fastify';
+import { Readable } from 'node:stream';
+import * as zlib from 'node:zlib';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { registerFastifyErrorHandler } from '../../../src/routes/route-register.js';
+
+describe('Fastify error handler', () => {
+  let app: FastifyInstance;
+
+  beforeEach(() => {
+    app = Fastify();
+    registerFastifyErrorHandler(app);
+  });
+
+  afterEach(async () => {
+    await app.close();
+  });
+
+  describe('errors thrown in route before sending a response', () => {
+    it('returns a JSON error body with no content-encoding when none was set', async () => {
+      app.get('/boom', async () => {
+        throw new errors.ServiceError({
+          status: 503,
+          code: 'PSYNC_S2003' as any,
+          description: 'Service unavailable'
+        });
+      });
+      await app.ready();
+
+      const response = await app.inject({ method: 'GET', url: '/boom' });
+
+      expect(response.statusCode).toBe(503);
+      expect(response.headers['content-type']).toMatch(/application\/json/);
+      expect(response.headers['content-encoding']).toBeUndefined();
+      expect(JSON.parse(response.payload)).toMatchObject({
+        error: { code: 'PSYNC_S2003', status: 503 }
+      });
+    });
+
+    it('gzip-encodes the error body when content-encoding=gzip was set before the throw', async () => {
+      app.get('/boom', async (_request, reply) => {
+        reply.header('content-encoding', 'gzip');
+        throw new Error('kaboom');
+      });
+      await app.ready();
+
+      const response = await app.inject({ method: 'GET', url: '/boom' });
+
+      expect(response.statusCode).toBe(500);
+      expect(response.headers['content-encoding']).toBe('gzip');
+      const decoded = zlib.gunzipSync(response.rawPayload).toString('utf8');
+      expect(JSON.parse(decoded).error.code).toBe('PSYNC_S2001');
+    });
+
+    it('zstd-encodes the error body when content-encoding=zstd was set before the throw', async () => {
+      app.get('/boom', async (_request, reply) => {
+        reply.header('content-encoding', 'zstd');
+        throw new Error('kaboom');
+      });
+      await app.ready();
+
+      const response = await app.inject({ method: 'GET', url: '/boom' });
+
+      expect(response.statusCode).toBe(500);
+      expect(response.headers['content-encoding']).toBe('zstd');
+      const decoded = zlib.zstdDecompressSync(response.rawPayload).toString('utf8');
+      expect(JSON.parse(decoded).error.code).toBe('PSYNC_S2001');
+    });
+  });
+
+  describe('errors after responding with a stream, before any data is sent', () => {
+    it('still returns a JSON error response with no encoding', async () => {
+      app.get('/stream', async (_request, reply) => {
+        const stream = new Readable({
+          read() {
+            process.nextTick(() => this.destroy(new Error('pre-data failure')));
+          }
+        });
+        reply.header('content-type', 'application/x-ndjson');
+        return reply.send(stream);
+      });
+      await app.ready();
+
+      const response = await app.inject({ method: 'GET', url: '/stream' });
+
+      expect(response.statusCode).toBe(500);
+      expect(response.headers['content-encoding']).toBeUndefined();
+      expect(response.headers['content-type']).toMatch(/application\/json/);
+      expect(JSON.parse(response.payload).error.code).toBe('PSYNC_S2001');
+    });
+
+    it('still returns a gzip-encoded JSON error response when encoding was set first', async () => {
+      app.get('/stream', async (_request, reply) => {
+        reply.header('content-encoding', 'gzip');
+        reply.header('content-type', 'application/x-ndjson');
+        const stream = new Readable({
+          read() {
+            process.nextTick(() => this.destroy(new Error('pre-data failure')));
+          }
+        });
+        return reply.send(stream);
+      });
+      await app.ready();
+
+      const response = await app.inject({ method: 'GET', url: '/stream' });
+
+      expect(response.statusCode).toBe(500);
+      expect(response.headers['content-encoding']).toBe('gzip');
+      const decoded = zlib.gunzipSync(response.rawPayload).toString('utf8');
+      expect(JSON.parse(decoded).error.code).toBe('PSYNC_S2001');
+    });
+  });
+
+  describe('errors after a stream has sent data', () => {
+    it('lets fastify tear the response down without raising an uncaught exception', async () => {
+      app.get('/stream', async (_request, reply) => {
+        let chunks = 0;
+        const stream = new Readable({
+          read() {
+            if (chunks++ === 0) {
+              this.push('first chunk\n');
+            } else {
+              this.destroy(new Error('mid-stream failure'));
+            }
+          }
+        });
+        reply.header('content-type', 'application/x-ndjson');
+        return reply.send(stream);
+      });
+      await app.ready();
+
+      const uncaught: Error[] = [];
+      const onUncaught = (err: Error) => uncaught.push(err);
+      const onUnhandled = (err: any) => uncaught.push(err);
+      process.on('uncaughtException', onUncaught);
+      process.on('unhandledRejection', onUnhandled);
+
+      try {
+        await expect(app.inject({ method: 'GET', url: '/stream' })).rejects.toThrow(/destroyed/);
+        await new Promise((resolve) => setImmediate(resolve));
+      } finally {
+        process.off('uncaughtException', onUncaught);
+        process.off('unhandledRejection', onUnhandled);
+      }
+
+      expect(uncaught).toEqual([]);
+    });
+  });
+
+  describe('handler inheritance into child scopes', () => {
+    it('handles errors thrown in routes registered inside a child scope', async () => {
+      await app.register(async (child) => {
+        child.get('/boom', async () => {
+          throw new Error('child scope failure');
+        });
+      });
+      await app.ready();
+
+      const response = await app.inject({ method: 'GET', url: '/boom' });
+
+      expect(response.statusCode).toBe(500);
+      expect(response.headers['content-type']).toMatch(/application\/json/);
+      const body = JSON.parse(response.payload);
+      expect(body.error?.code).toBe('PSYNC_S2001');
+    });
+  });
+
+  describe('built-in Fastify errors', () => {
+    it('preserves the 400 status code for an invalid JSON body', async () => {
+      app.post('/echo', async (request) => ({ received: request.body }));
+      await app.ready();
+
+      const response = await app.inject({
+        method: 'POST',
+        url: '/echo',
+        headers: { 'content-type': 'application/json' },
+        payload: '{ not json'
+      });
+
+      expect(response.statusCode).toBe(400);
+      const body = JSON.parse(response.payload);
+      expect(body.error.status).toBe(400);
+      expect(body.error.code).toBe('FST_ERR_CTP_INVALID_JSON_BODY');
+    });
+
+    it('preserves the 400 status code for an empty JSON body', async () => {
+      app.post('/echo', async (request) => ({ received: request.body }));
+      await app.ready();
+
+      const response = await app.inject({
+        method: 'POST',
+        url: '/echo',
+        headers: { 'content-type': 'application/json' },
+        payload: ''
+      });
+
+      expect(response.statusCode).toBe(400);
+      const body = JSON.parse(response.payload);
+      expect(body.error.status).toBe(400);
+      expect(body.error.code).toBe('FST_ERR_CTP_EMPTY_JSON_BODY');
+    });
+
+    it('preserves the 400 status code for a schema validation failure', async () => {
+      app.post(
+        '/schema',
+        {
+          schema: {
+            body: {
+              type: 'object',
+              required: ['name'],
+              properties: { name: { type: 'string' } }
+            }
+          }
+        },
+        async () => ({ ok: true })
+      );
+      await app.ready();
+
+      const response = await app.inject({
+        method: 'POST',
+        url: '/schema',
+        headers: { 'content-type': 'application/json' },
+        payload: JSON.stringify({})
+      });
+
+      expect(response.statusCode).toBe(400);
+      const body = JSON.parse(response.payload);
+      expect(body.error.status).toBe(400);
+      expect(body.error.code).toBe('FST_ERR_VALIDATION');
+    });
+
+    it('preserves the 413 status code when the body exceeds the limit', async () => {
+      const tinyApp = Fastify({ bodyLimit: 16 });
+      registerFastifyErrorHandler(tinyApp);
+      tinyApp.post('/echo', async (request) => ({ received: request.body }));
+      await tinyApp.ready();
+
+      try {
+        const response = await tinyApp.inject({
+          method: 'POST',
+          url: '/echo',
+          headers: { 'content-type': 'application/json' },
+          payload: JSON.stringify({ data: 'x'.repeat(64) })
+        });
+
+        expect(response.statusCode).toBe(413);
+        const body = JSON.parse(response.payload);
+        expect(body.error.status).toBe(413);
+        expect(body.error.code).toBe('FST_ERR_CTP_BODY_TOO_LARGE');
+      } finally {
+        await tinyApp.close();
+      }
+    });
+
+    it('preserves the 415 status code for an unsupported media type', async () => {
+      app.post('/json-only', async (request) => ({ received: request.body }));
+      // Drop the default text/plain parser so non-JSON content types are rejected.
+      app.removeContentTypeParser('text/plain');
+      await app.ready();
+
+      const response = await app.inject({
+        method: 'POST',
+        url: '/json-only',
+        headers: { 'content-type': 'text/plain' },
+        payload: 'hello'
+      });
+
+      expect(response.statusCode).toBe(415);
+      const body = JSON.parse(response.payload);
+      expect(body.error.status).toBe(415);
+      expect(body.error.code).toBe('FST_ERR_CTP_INVALID_MEDIA_TYPE');
+    });
+  });
+});

package/test/src/routes/stream.test.ts

@@ -1,6 +1,12 @@
 import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
-import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
-import { DEFAULT_HYDRATION_STATE, SqlSyncRules } from '@powersync/service-sync-rules';
+import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
+import {
+  DEFAULT_HYDRATION_STATE,
+  HydrateSyncConfigParams,
+  nodeSqlite,
+  SqlSyncRules
+} from '@powersync/service-sync-rules';
+import * as sqlite from 'node:sqlite';
 import { Readable, Writable } from 'stream';
 import { pipeline } from 'stream/promises';
 import { describe, expect, it } from 'vitest';
@@ -10,6 +16,11 @@ import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../
 import { mockServiceContext } from './mocks.js';
 
 describe('Stream Route', () => {
+  const defaultHydrationOptions: HydrateSyncConfigParams = {
+    hydrationState: DEFAULT_HYDRATION_STATE,
+    sqlite: nodeSqlite(sqlite)
+  };
+
   describe('compressed stream', () => {
     it('handles missing sync rules', async () => {
       const context: Context = {
@@ -45,7 +56,7 @@ describe('Stream Route', () => {
 
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
+          return new SqlSyncRules('bucket_definitions: {}').hydrate(defaultHydrationOptions);
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');
@@ -83,7 +94,7 @@ describe('Stream Route', () => {
     it('logs the application metadata', async () => {
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
+          return new SqlSyncRules('bucket_definitions: {}').hydrate(defaultHydrationOptions);
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');