@powersync/service-core 0.0.2

package/src/sync/sync.ts

@@ -0,0 +1,312 @@
+import * as micro from '@journeyapps-platform/micro';
+import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
+import { SyncParameters } from '@powersync/service-sync-rules';
+import { Semaphore } from 'async-mutex';
+import { AbortError } from 'ix/aborterror.js';
+
+import * as auth from '@/auth/auth-index.js';
+import * as storage from '@/storage/storage-index.js';
+import * as util from '@/util/util-index.js';
+
+import { operations_synced_total } from '../metrics/metrics.js';
+import { mergeAsyncIterables } from './merge.js';
+import { TokenStreamOptions, tokenStream } from './util.js';
+
+/**
+ * Maximum number of connections actively fetching data.
+ */
+const MAX_ACTIVE_CONNECTIONS = 10;
+const syncSemaphore = new Semaphore(MAX_ACTIVE_CONNECTIONS);
+
+export interface SyncStreamParameters {
+  storage: storage.BucketStorageFactory;
+  params: util.StreamingSyncRequest;
+  syncParams: SyncParameters;
+  token: auth.JwtPayload;
+  /**
+   * If this signal is aborted, the stream response ends as soon as possible, without error.
+   */
+  signal?: AbortSignal;
+  tokenStreamOptions?: Partial<TokenStreamOptions>;
+}
+
+export async function* streamResponse(
+  options: SyncStreamParameters
+): AsyncIterable<util.StreamingSyncLine | string | null> {
+  const { storage, params, syncParams, token, tokenStreamOptions, signal } = options;
+  // We also need to be able to abort, so we create our own controller.
+  const controller = new AbortController();
+  if (signal) {
+    signal.addEventListener(
+      'abort',
+      () => {
+        controller.abort();
+      },
+      { once: true }
+    );
+    if (signal.aborted) {
+      controller.abort();
+    }
+  }
+  const ki = tokenStream(token, controller.signal, tokenStreamOptions);
+  const stream = streamResponseInner(storage, params, syncParams, controller.signal);
+  // Merge the two streams, and abort as soon as one of the streams end.
+  const merged = mergeAsyncIterables([stream, ki], controller.signal);
+
+  try {
+    yield* merged;
+  } catch (e) {
+    if (e instanceof AbortError) {
+      return;
+    } else {
+      throw e;
+    }
+  } finally {
+    // This ensures all the underlying streams are aborted as soon as possible if the
+    // parent loop stops.
+    controller.abort();
+  }
+}
+
+async function* streamResponseInner(
+  storage: storage.BucketStorageFactory,
+  params: util.StreamingSyncRequest,
+  syncParams: SyncParameters,
+  signal: AbortSignal
+): AsyncGenerator<util.StreamingSyncLine | string | null> {
+  // Bucket state of bucket id -> op_id.
+  // This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
+  let dataBuckets = new Map<string, string>();
+
+  let last_checksums: util.BucketChecksum[] | null = null;
+  let last_write_checkpoint: bigint | null = null;
+
+  const { raw_data, binary_data } = params;
+
+  if (params.buckets) {
+    for (let { name, after: start } of params.buckets) {
+      dataBuckets.set(name, start);
+    }
+  }
+
+  const stream = storage.watchWriteCheckpoint(syncParams.token_parameters.user_id as string, signal);
+  for await (const next of stream) {
+    const { base, writeCheckpoint } = next;
+    const checkpoint = base.checkpoint;
+
+    const storage = await base.getBucketStorage();
+    if (storage == null) {
+      // Sync rules deleted in the meantime - try again with the next checkpoint.
+      continue;
+    }
+    const sync_rules = storage.sync_rules;
+
+    const allBuckets = await sync_rules.queryBucketIds({
+      getParameterSets(lookups) {
+        return storage.getParameterSets(checkpoint, lookups);
+      },
+      parameters: syncParams
+    });
+
+    if (allBuckets.length > 1000) {
+      // TODO: Limit number of buckets even before we get to this point
+      throw new Error(`Too many buckets: ${allBuckets.length}`);
+    }
+
+    let checksums: util.BucketChecksum[] | undefined = undefined;
+
+    let dataBucketsNew = new Map<string, string>();
+    for (let bucket of allBuckets) {
+      dataBucketsNew.set(bucket, dataBuckets.get(bucket) ?? '0');
+    }
+    dataBuckets = dataBucketsNew;
+
+    checksums = await storage.getChecksums(checkpoint, [...dataBuckets.keys()]);
+
+    if (last_checksums) {
+      const diff = util.checksumsDiff(last_checksums, checksums);
+
+      if (
+        last_write_checkpoint == writeCheckpoint &&
+        diff.removed_buckets.length == 0 &&
+        diff.updated_buckets.length == 0
+      ) {
+        // No changes - don't send anything to the client
+        continue;
+      }
+
+      let message = `Updated checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
+      message += `buckets: ${allBuckets.length} | `;
+      message += `updated: ${limitedBuckets(diff.updated_buckets, 20)} | `;
+      message += `removed: ${limitedBuckets(diff.removed_buckets, 20)} | `;
+      micro.logger.info(message);
+
+      const checksum_line: util.StreamingSyncCheckpointDiff = {
+        checkpoint_diff: {
+          last_op_id: checkpoint,
+          write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
+          ...diff
+        }
+      };
+
+      yield checksum_line;
+    } else {
+      let message = `New checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
+      message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
+      micro.logger.info(message);
+      const checksum_line: util.StreamingSyncCheckpoint = {
+        checkpoint: {
+          last_op_id: checkpoint,
+          write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
+          buckets: checksums
+        }
+      };
+      yield checksum_line;
+    }
+
+    last_checksums = checksums;
+    last_write_checkpoint = writeCheckpoint;
+
+    yield* bucketDataInBatches(storage, checkpoint, dataBuckets, raw_data, binary_data, signal);
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+}
+
+async function* bucketDataInBatches(
+  storage: storage.SyncRulesBucketStorage,
+  checkpoint: string,
+  dataBuckets: Map<string, string>,
+  raw_data: boolean | undefined,
+  binary_data: boolean | undefined,
+  signal: AbortSignal
+) {
+  let isDone = false;
+  while (!signal.aborted && !isDone) {
+    // The code below is functionally the same as this for-await loop below.
+    // However, the for-await loop appears to have a memory leak, so we avoid it.
+    // for await (const { done, data } of bucketDataBatch(storage, checkpoint, dataBuckets, raw_data, signal)) {
+    //   yield data;
+    //   if (done) {
+    //     isDone = true;
+    //   }
+    //   break;
+    // }
+    const iter = bucketDataBatch(storage, checkpoint, dataBuckets, raw_data, binary_data, signal);
+    try {
+      while (true) {
+        const { value, done: iterDone } = await iter.next();
+        if (iterDone) {
+          break;
+        } else {
+          const { done, data } = value;
+          yield data;
+          if (done) {
+            isDone = true;
+          }
+        }
+      }
+    } finally {
+      await iter.return();
+    }
+  }
+}
+
+/**
+ * Extracted as a separate internal function just to avoid memory leaks.
+ */
+async function* bucketDataBatch(
+  storage: storage.SyncRulesBucketStorage,
+  checkpoint: string,
+  dataBuckets: Map<string, string>,
+  raw_data: boolean | undefined,
+  binary_data: boolean | undefined,
+  signal: AbortSignal
+) {
+  const [_, release] = await syncSemaphore.acquire();
+  try {
+    const data = storage.getBucketDataBatch(checkpoint, dataBuckets);
+
+    let has_more = false;
+
+    for await (let r of data) {
+      if (signal.aborted) {
+        return;
+      }
+      if (r.has_more) {
+        has_more = true;
+      }
+      if (r.data.length == 0) {
+        continue;
+      }
+      micro.logger.debug(`Sending data for ${r.bucket}`);
+
+      let send_data: any;
+      if (binary_data) {
+        // Send the object as is, will most likely be encoded as a BSON document
+        send_data = { data: r };
+      } else if (raw_data) {
+        // Data is a raw string - we can use the more efficient JSON.stringify.
+        const response: util.StreamingSyncData = {
+          data: r
+        };
+        send_data = JSON.stringify(response);
+      } else {
+        // We need to preserve the embedded data exactly, so this uses a JsonContainer
+        // and JSONBig to stringify.
+        const response: util.StreamingSyncData = {
+          data: transformLegacyResponse(r)
+        };
+        send_data = JSONBig.stringify(response);
+      }
+      yield { data: send_data, done: false };
+      if (send_data.length > 50_000) {
+        // IMPORTANT: This does not affect the output stream, but is used to flush
+        // iterator memory in case if large data sent.
+        yield { data: null, done: false };
+      }
+      operations_synced_total.add(r.data.length);
+
+      dataBuckets.set(r.bucket, r.next_after);
+    }
+
+    if (!has_more) {
+      const line: util.StreamingSyncCheckpointComplete = {
+        checkpoint_complete: {
+          last_op_id: checkpoint
+        }
+      };
+      yield { data: line, done: true };
+    }
+  } finally {
+    release();
+  }
+}
+
+function transformLegacyResponse(bucketData: util.SyncBucketData): any {
+  return {
+    ...bucketData,
+    data: bucketData.data.map((entry) => {
+      return {
+        ...entry,
+        data: entry.data == null ? null : new JsonContainer(entry.data as string),
+        checksum: BigInt(entry.checksum)
+      };
+    })
+  };
+}
+
+function limitedBuckets(buckets: string[] | util.BucketChecksum[], limit: number) {
+  buckets = buckets.map((b) => {
+    if (typeof b != 'string') {
+      return b.bucket;
+    } else {
+      return b;
+    }
+  });
+  if (buckets.length <= limit) {
+    return JSON.stringify(buckets);
+  }
+  const limited = buckets.slice(0, limit);
+  return `${JSON.stringify(limited)}...`;
+}

package/src/sync/util.ts

@@ -0,0 +1,98 @@
+import * as timers from 'timers/promises';
+
+import * as util from '@/util/util-index.js';
+import { data_synced_bytes } from '../metrics/metrics.js';
+
+export type TokenStreamOptions = {
+  /**
+   * Adds periodic keepalive events
+   */
+  keep_alive: boolean;
+  /**
+   * Warn before the token is going to expire
+   */
+  expire_warning_period: number;
+};
+
+const KEEPALIVE_INTERVAL = 20_000;
+
+const DEFAULT_TOKEN_STREAM_OPTIONS: TokenStreamOptions = {
+  keep_alive: true,
+  expire_warning_period: 20_000
+};
+
+/**
+ * An iterator that periodically yields token and optionally keepalive events, and returns once the
+ * provided token expiry is reached.
+ *
+ * @param token exp is expiry as a unix timestamp (seconds)
+ * @param signal abort the iterator with this
+ * @param options configure keepalive and expire warnings
+ */
+export async function* tokenStream(
+  token: { exp: number },
+  signal: AbortSignal,
+  options?: Partial<TokenStreamOptions>
+): AsyncGenerator<util.StreamingSyncKeepalive> {
+  const resolved_options: TokenStreamOptions = {
+    ...DEFAULT_TOKEN_STREAM_OPTIONS,
+    ...(options ?? {})
+  };
+
+  const { keep_alive, expire_warning_period } = resolved_options;
+
+  // Real tokens always have an integer for exp.
+  // In tests, we may use fractional seconds to reduce the delay.
+  // Both cases are handled here.
+  const expires_at = token.exp * 1000;
+  const expire_warning_at = expires_at - expire_warning_period;
+
+  let first_expire_test = true;
+
+  while (!signal.aborted) {
+    const token_expires_in = Math.max(0, Math.ceil(token.exp - Date.now() / 1000));
+    if (first_expire_test && token_expires_in > 0) {
+      first_expire_test = false;
+    } else {
+      yield { token_expires_in: token_expires_in };
+      if (token_expires_in == 0) {
+        return;
+      }
+    }
+
+    const keep_alive_delay = KEEPALIVE_INTERVAL * (0.9 + Math.random() * 0.2);
+
+    // Add margin due to setTimeout inaccuracies
+    const expiry_delay = Math.max(0, expires_at - Date.now() + 3);
+    const expiry_warning_delay = Math.max(0, expire_warning_at - Date.now() + 3);
+    // Either the warning has past or it's before
+    const relevant_expiry_delay = expiry_warning_delay != 0 ? expiry_warning_delay : expiry_delay;
+
+    const delay = keep_alive ? Math.min(relevant_expiry_delay, keep_alive_delay) : relevant_expiry_delay;
+    await timers.setTimeout(delay, null, { signal }).catch(() => {
+      // Ignore AbortError
+    });
+  }
+}
+
+export async function* ndjson(iterator: AsyncIterable<string | null | Record<string, any>>): AsyncGenerator<string> {
+  for await (let data of iterator) {
+    if (data == null) {
+      // Empty value to flush iterator memory
+      continue;
+    } else if (typeof data == 'string') {
+      // Pre-serialized value
+      yield data + '\n';
+    } else {
+      yield JSON.stringify(data) + '\n';
+    }
+  }
+}
+
+export async function* transformToBytesTracked(iterator: AsyncIterable<string>): AsyncGenerator<Buffer> {
+  for await (let data of iterator) {
+    const encoded = Buffer.from(data, 'utf8');
+    data_synced_bytes.add(encoded.length);
+    yield encoded;
+  }
+}

package/src/system/CorePowerSyncSystem.ts

@@ -0,0 +1,43 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import * as micro from '@journeyapps-platform/micro';
+
+import * as auth from '@/auth/auth-index.js';
+import * as storage from '@/storage/storage-index.js';
+import * as utils from '@/util/util-index.js';
+
+export abstract class CorePowerSyncSystem extends micro.system.MicroSystem {
+  abstract storage: storage.BucketStorageFactory;
+  abstract client_keystore: auth.KeyStore;
+  abstract dev_client_keystore: auth.KeyStore;
+  abstract pgwire_pool?: pgwire.PgClient;
+
+  protected stopHandlers: Set<() => void> = new Set();
+
+  closed: boolean;
+
+  constructor(public config: utils.ResolvedPowerSyncConfig) {
+    super();
+    this.closed = false;
+  }
+
+  abstract addTerminationHandler(): void;
+
+  addStopHandler(handler: () => void): () => void {
+    if (this.closed) {
+      handler();
+      return () => {};
+    }
+    this.stopHandlers.add(handler);
+    return () => {
+      this.stopHandlers.delete(handler);
+    };
+  }
+
+  requirePgPool() {
+    if (this.pgwire_pool == null) {
+      throw new Error('No source connection configured');
+    } else {
+      return this.pgwire_pool!;
+    }
+  }
+}

package/src/util/Mutex.ts

@@ -0,0 +1,159 @@
+export type PromiseFunction<T> = (context: MutexContext) => Promise<T>;
+export type SharedPromiseFunction<T> = (context: SharedMutexContext) => Promise<T>;
+
+const DEBUG_MUTEX = false;
+
+interface Task {
+  exclusive: boolean;
+  execute: (context: SharedMutexContext) => void;
+}
+
+export interface SharedMutexContext {
+  sharedLock<T>(promiseFn: SharedPromiseFunction<T>): Promise<T>;
+}
+
+export interface MutexContext extends SharedMutexContext {
+  exclusiveLock<T>(promiseFn: PromiseFunction<T>): Promise<T>;
+}
+
+class ExclusiveContext implements MutexContext {
+  constructor(private mutex: Mutex) {}
+
+  exclusiveLock<T>(promiseFn: PromiseFunction<T>): Promise<T> {
+    return promiseFn(this);
+  }
+
+  sharedLock<T>(promiseFn: PromiseFunction<T>): Promise<T> {
+    return promiseFn(this);
+  }
+}
+
+class SharedContext implements SharedMutexContext {
+  constructor(private mutex: Mutex) {}
+
+  async exclusiveLock<T>(promiseFn: PromiseFunction<T>): Promise<T> {
+    throw new Error('Cannot upgrade a shared lock to an exclusive lock.');
+  }
+
+  sharedLock<T>(promiseFn: SharedPromiseFunction<T>): Promise<T> {
+    return promiseFn(this);
+  }
+}
+
+/**
+ * Mutex maintains a queue of Promise-returning functions that
+ * are executed sequentially (whereas normally they would execute their async code concurrently).
+ */
+export class Mutex implements MutexContext {
+  private queue: Task[];
+  private sharedCount: number;
+  private exclusiveLocked: boolean;
+
+  constructor() {
+    this.queue = [];
+    this.sharedCount = 0;
+    this.exclusiveLocked = false;
+  }
+
+  /**
+   * Place a function on the queue.
+   * The function may either return a Promise or a value.
+   * Return a Promise that is resolved with the result of the function.
+   */
+  async exclusiveLock<T>(promiseFn: PromiseFunction<T>): Promise<T> {
+    return this.lock(promiseFn as any, true);
+  }
+
+  /**
+   * Place a function on the queue.
+   * This function may execute in parallel with other "multi" functions, but not with other functions on the exclusive
+   * queue.
+   */
+  sharedLock<T>(promiseFn: SharedPromiseFunction<T>): Promise<T> {
+    return this.lock(promiseFn, false);
+  }
+
+  private async lock<T>(promiseFn: SharedPromiseFunction<T>, exclusive?: boolean): Promise<T> {
+    const context = await this._lockNext(exclusive);
+    let timeout;
+    try {
+      if (DEBUG_MUTEX) {
+        const stack = new Error().stack;
+        timeout = setTimeout(() => {
+          console.warn('Mutex not released in 10 seconds\n', stack);
+        }, 10000);
+      }
+      return await promiseFn(context);
+    } finally {
+      if (DEBUG_MUTEX) {
+        clearTimeout(timeout);
+      }
+      if (!exclusive) {
+        this.sharedCount -= 1;
+      } else {
+        this.exclusiveLocked = false;
+      }
+      this._tryNext();
+    }
+  }
+
+  /**
+   * Convert a normal Promise-returning function into one that is automatically enqueued.
+   * The signature of the function stays the same - only the execution is potentially delayed.
+   * The only exception is that if the function would have returned a scalar value, it now
+   * returns a Promise.
+   */
+  qu<T>(fn: PromiseFunction<T>): () => Promise<T> {
+    var self = this;
+    return function () {
+      var args = arguments;
+      return self.exclusiveLock(function () {
+        return fn.apply(null, args as any);
+      });
+    };
+  }
+
+  /**
+   * Wait until we are ready to execute the next task on the queue.
+   *
+   * This places a "Task" marker on the queue, and waits until we get to it.
+   *
+   * @param exclusive
+   */
+  private _lockNext(exclusive?: boolean): Promise<SharedMutexContext> {
+    var self = this;
+    return new Promise<SharedMutexContext>(function (resolve, reject) {
+      const task: Task = {
+        execute: resolve,
+        exclusive: exclusive ?? false
+      };
+      self.queue.push(task);
+      self._tryNext();
+    });
+  }
+
+  private _tryNext() {
+    if (this.queue.length == 0) {
+      return false;
+    }
+
+    if (this.exclusiveLocked) {
+      return false;
+    }
+
+    var task = this.queue[0];
+    if (!task.exclusive) {
+      this.sharedCount += 1;
+      this.queue.shift();
+      task.execute(new SharedContext(this));
+    } else if (this.sharedCount == 0) {
+      this.exclusiveLocked = true;
+      this.queue.shift();
+      task.execute(new ExclusiveContext(this));
+    } else {
+      return false;
+    }
+
+    return true;
+  }
+}

package/src/util/PgManager.ts

@@ -0,0 +1,64 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import { NormalizedPostgresConnection } from '@powersync/service-types';
+
+export class PgManager {
+  /**
+   * Do not use this for any transactions.
+   */
+  public readonly pool: pgwire.PgClient;
+
+  private connectionPromises: Promise<pgwire.PgConnection>[] = [];
+
+  constructor(public options: NormalizedPostgresConnection, public poolOptions: pgwire.PgPoolOptions) {
+    // The pool is lazy - no connections are opened until a query is performed.
+    this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
+  }
+
+  /**
+   * Create a new replication connection.
+   */
+  async replicationConnection(): Promise<pgwire.PgConnection> {
+    const p = pgwire.connectPgWire(this.options, { type: 'replication' });
+    this.connectionPromises.push(p);
+    return await p;
+  }
+
+  /**
+   * Create a new standard connection, used for initial snapshot.
+   *
+   * This connection must not be shared between multiple async contexts.
+   */
+  async snapshotConnection(): Promise<pgwire.PgConnection> {
+    const p = pgwire.connectPgWire(this.options, { type: 'standard' });
+    this.connectionPromises.push(p);
+    return await p;
+  }
+
+  async end() {
+    for (let result of await Promise.allSettled([
+      this.pool.end(),
+      ...this.connectionPromises.map((promise) => {
+        return promise.then((connection) => connection.end());
+      })
+    ])) {
+      // Throw the first error, if any
+      if (result.status == 'rejected') {
+        throw result.reason;
+      }
+    }
+  }
+
+  async destroy() {
+    this.pool.destroy();
+    for (let result of await Promise.allSettled([
+      ...this.connectionPromises.map((promise) => {
+        return promise.then((connection) => connection.destroy());
+      })
+    ])) {
+      // Throw the first error, if any
+      if (result.status == 'rejected') {
+        throw result.reason;
+      }
+    }
+  }
+}

package/src/util/alerting.ts

@@ -0,0 +1,17 @@
+import * as micro from '@journeyapps-platform/micro';
+
+let globalTags: Record<string, string> = {};
+
+export function setTags(tags: Record<string, string>) {
+  globalTags = tags;
+}
+
+export function getGlobalTags() {
+  return globalTags;
+}
+
+export function captureException(error: any, options?: micro.alerts.CaptureOptions) {
+  micro.alerts.captureException(error, {
+    ...options
+  });
+}