@powersync/service-core 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219091224

package/src/storage/mongo/models.ts

@@ -1,179 +0,0 @@
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-
-/**
- * Replica id uniquely identifying a row on the source database.
- *
- * Can be any value serializable to BSON.
- *
- * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
- */
-export type ReplicaId = bson.UUID | bson.Document | any;
-
-export interface SourceKey {
-  /** group_id */
-  g: number;
-  /** source table id */
-  t: bson.ObjectId;
-  /** source key */
-  k: ReplicaId;
-}
-
-export interface BucketDataKey {
-  /** group_id */
-  g: number;
-  /** bucket name */
-  b: string;
-  /** op_id */
-  o: bigint;
-}
-
-export interface CurrentDataDocument {
-  _id: SourceKey;
-  data: bson.Binary;
-  buckets: CurrentBucket[];
-  lookups: bson.Binary[];
-}
-
-export interface CurrentBucket {
-  bucket: string;
-  table: string;
-  id: string;
-}
-
-export interface BucketParameterDocument {
-  _id: bigint;
-  key: SourceKey;
-  lookup: bson.Binary;
-  bucket_parameters: Record<string, SqliteJsonValue>[];
-}
-
-export interface BucketDataDocument {
-  _id: BucketDataKey;
-  op: OpType;
-  source_table?: bson.ObjectId;
-  source_key?: ReplicaId;
-  table?: string;
-  row_id?: string;
-  checksum: number;
-  data: string | null;
-  target_op?: bigint | null;
-}
-
-export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
-
-export interface SourceTableDocument {
-  _id: bson.ObjectId;
-  group_id: number;
-  connection_id: number;
-  relation_id: number | string | undefined;
-  schema_name: string;
-  table_name: string;
-  replica_id_columns: string[] | null;
-  replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
-  snapshot_done: boolean | undefined;
-}
-
-export interface IdSequenceDocument {
-  _id: string;
-  op_id: bigint;
-}
-
-export enum SyncRuleState {
-  /**
-   * New sync rules - needs to be processed (initial replication).
-   *
-   * While multiple sets of sync rules _can_ be in PROCESSING,
-   * it's generally pointless, so we only keep one in that state.
-   */
-  PROCESSING = 'PROCESSING',
-
-  /**
-   * Sync rule processing is done, and can be used for sync.
-   *
-   * Only one set of sync rules should be in ACTIVE state.
-   */
-  ACTIVE = 'ACTIVE',
-  /**
-   * This state is used when the sync rules has been replaced,
-   * and replication is or should be stopped.
-   */
-  STOP = 'STOP',
-  /**
-   * After sync rules have been stopped, the data needs to be
-   * deleted. Once deleted, the state is TERMINATED.
-   */
-  TERMINATED = 'TERMINATED'
-}
-
-export interface SyncRuleDocument {
-  _id: number;
-
-  state: SyncRuleState;
-
-  /**
-   * True if initial snapshot has been replicated.
-   *
-   * Can only be false if state == PROCESSING.
-   */
-  snapshot_done: boolean;
-
-  /**
-   * The last consistent checkpoint.
-   *
-   * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
-   */
-  last_checkpoint: bigint | null;
-
-  /**
-   * The LSN associated with the last consistent checkpoint.
-   */
-  last_checkpoint_lsn: string | null;
-
-  /**
-   * If set, no new checkpoints may be created < this value.
-   */
-  no_checkpoint_before: string | null;
-
-  slot_name: string | null;
-
-  /**
-   * Last time we persisted a checkpoint.
-   *
-   * This may be old if no data is incoming.
-   */
-  last_checkpoint_ts: Date | null;
-
-  /**
-   * Last time we persisted a checkpoint or keepalive.
-   *
-   * This should stay fairly current while replicating.
-   */
-  last_keepalive_ts: Date | null;
-
-  /**
-   * If an error is stopping replication, it will be stored here.
-   */
-  last_fatal_error: string | null;
-
-  content: string;
-}
-
-export interface CustomWriteCheckpointDocument {
-  _id: bson.ObjectId;
-  user_id: string;
-  checkpoint: bigint;
-  sync_rules_id: number;
-}
-
-export interface WriteCheckpointDocument {
-  _id: bson.ObjectId;
-  user_id: string;
-  lsns: Record<string, string>;
-  client_id: bigint;
-}
-
-export interface InstanceDocument {
-  // The instance UUID
-  _id: string;
-}

package/src/storage/mongo/util.ts

@@ -1,158 +0,0 @@
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-import * as crypto from 'crypto';
-import * as mongo from 'mongodb';
-import * as uuid from 'uuid';
-import { OplogEntry } from '../../util/protocol-types.js';
-import { ID_NAMESPACE, timestampToOpId } from '../../util/utils.js';
-import { BucketDataDocument, ReplicaId } from './models.js';
-
-/**
- * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
- * @param lookup
- */
-
-export function serializeLookup(lookup: SqliteJsonValue[]) {
-  const normalized = lookup.map((value) => {
-    if (typeof value == 'number' && Number.isInteger(value)) {
-      return BigInt(value);
-    } else {
-      return value;
-    }
-  });
-  return new bson.Binary(bson.serialize({ l: normalized }));
-}
-
-export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
-  let filter = {
-    $gte: {
-      ...prefix
-    } as any,
-    $lt: {
-      ...prefix
-    } as any
-  };
-
-  for (let key of rest) {
-    filter.$gte[key] = new bson.MinKey();
-    filter.$lt[key] = new bson.MaxKey();
-  }
-
-  return filter;
-}
-
-export function generateSlotName(prefix: string, sync_rules_id: number) {
-  const slot_suffix = crypto.randomBytes(2).toString('hex');
-  return `${prefix}${sync_rules_id}_${slot_suffix}`;
-}
-
-/**
- * Read a single batch of data from a cursor, then close it.
- *
- * We do our best to avoid MongoDB fetching any more data than this single batch.
- *
- * This is similar to using `singleBatch: true` in find options.
- * However, that makes `has_more` detection very difficult, since the cursor is always closed
- * after the first batch. Instead, we do a workaround to only fetch a single batch below.
- *
- * For this to be effective, set batchSize = limit in the find command.
- */
-export async function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{ data: T[]; hasMore: boolean }> {
-  try {
-    let data: T[];
-    let hasMore = true;
-    // Let MongoDB load the first batch of data
-    const hasAny = await cursor.hasNext();
-    // Now it's in memory, and we can read it
-    data = cursor.readBufferedDocuments();
-    if (!hasAny || cursor.id?.isZero()) {
-      // A zero id means the cursor is exhaused.
-      // No results (hasAny == false) means even this batch doesn't have data.
-      // This should similar results as `await cursor.hasNext()`, but without
-      // actually fetching the next batch.
-      //
-      // Note that it is safe (but slightly inefficient) to return `hasMore: true`
-      // without there being more data, as long as the next batch
-      // will return `hasMore: false`.
-      hasMore = false;
-    }
-    return { data, hasMore };
-  } finally {
-    // Match the from the cursor iterator logic here:
-    // https://github.com/mongodb/node-mongodb-native/blob/e02534e7d1c627bf50b85ca39f5995dbf165ad44/src/cursor/abstract_cursor.ts#L327-L331
-    if (!cursor.closed) {
-      await cursor.close();
-    }
-  }
-}
-
-export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
-  // use bigint instead of Long
-  useBigInt64: true
-};
-
-export function mapOpEntry(row: BucketDataDocument): OplogEntry {
-  if (row.op == 'PUT' || row.op == 'REMOVE') {
-    return {
-      op_id: timestampToOpId(row._id.o),
-      op: row.op,
-      object_type: row.table,
-      object_id: row.row_id,
-      checksum: Number(row.checksum),
-      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
-      data: row.data
-    };
-  } else {
-    // MOVE, CLEAR
-
-    return {
-      op_id: timestampToOpId(row._id.o),
-      op: row.op,
-      checksum: Number(row.checksum)
-    };
-  }
-}
-
-/**
- * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
- */
-export function replicaIdEquals(a: ReplicaId, b: ReplicaId) {
-  if (a === b) {
-    return true;
-  } else if (typeof a == 'string' && typeof b == 'string') {
-    return a == b;
-  } else if (isUUID(a) && isUUID(b)) {
-    return a.equals(b);
-  } else if (a == null && b == null) {
-    return true;
-  } else if (a != null || b != null) {
-    return false;
-  } else {
-    // There are many possible primitive values, this covers them all
-    return (bson.serialize({ id: a }) as Buffer).equals(bson.serialize({ id: b }));
-  }
-}
-
-export function replicaIdToSubkey(table: bson.ObjectId, id: ReplicaId): string {
-  if (isUUID(id)) {
-    // Special case for UUID for backwards-compatiblity
-    return `${table.toHexString()}/${id.toHexString()}`;
-  } else {
-    // Hashed UUID from the table and id
-    const repr = bson.serialize({ table, id });
-    return uuid.v5(repr, ID_NAMESPACE);
-  }
-}
-
-/**
- * True if this is a bson.UUID.
- *
- * Works even with multiple copies of the bson package.
- */
-export function isUUID(value: any): value is bson.UUID {
-  if (value == null || typeof value != 'object') {
-    return false;
-  }
-  const uuid = value as bson.UUID;
-  return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
-}

package/test/src/__snapshots__/sync.test.ts.snap

@@ -1,332 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -93886621,
-          "count": 2,
-        },
-      ],
-      "last_op_id": "2",
-      "write_checkpoint": undefined,
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
-[
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": -93886621n,
-          "op": "CLEAR",
-          "op_id": "2",
-        },
-      ],
-      "has_more": false,
-      "next_after": "2",
-    },
-  },
-  {
-    "checkpoint_diff": {
-      "last_op_id": "4",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 499012468,
-          "count": 4,
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "2",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 1859363232n,
-          "data": "{"id":"t1","description":"Test 1b"}",
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "3",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-        {
-          "checksum": 3028503153n,
-          "data": "{"id":"t2","description":"Test 2b"}",
-          "object_id": "t2",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "4",
-          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
-        },
-      ],
-      "has_more": false,
-      "next_after": "4",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "4",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > expired token 1`] = `
-[
-  {
-    "token_expires_in": 0,
-  },
-]
-`;
-
-exports[`sync - mongodb > expiring token 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 0,
-          "count": 0,
-        },
-      ],
-      "last_op_id": "0",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "0",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > expiring token 2`] = `
-[
-  {
-    "token_expires_in": 0,
-  },
-]
-`;
-
-exports[`sync - mongodb > sync global data 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -93886621,
-          "count": 2,
-        },
-      ],
-      "last_op_id": "2",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 920318466n,
-          "data": "{"id":"t1","description":"Test 1"}",
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "1",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-        {
-          "checksum": 3280762209n,
-          "data": "{"id":"t2","description":"Test 2"}",
-          "object_id": "t2",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "2",
-          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
-        },
-      ],
-      "has_more": false,
-      "next_after": "2",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "2",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync legacy non-raw data 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -852817836,
-          "count": 1,
-        },
-      ],
-      "last_op_id": "1",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 3442149460n,
-          "data": {
-            "description": "Test
-"string"",
-            "id": "t1",
-            "large_num": 12345678901234567890n,
-          },
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "1",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-      ],
-      "has_more": false,
-      "next_after": "1",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "1",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync updates to global data 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 0,
-          "count": 0,
-        },
-      ],
-      "last_op_id": "0",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "0",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync updates to global data 2`] = `
-[
-  {
-    "checkpoint_diff": {
-      "last_op_id": "1",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 920318466,
-          "count": 1,
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 920318466n,
-          "data": "{"id":"t1","description":"Test 1"}",
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "1",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-      ],
-      "has_more": false,
-      "next_after": "1",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "1",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync updates to global data 3`] = `
-[
-  {
-    "checkpoint_diff": {
-      "last_op_id": "2",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -93886621,
-          "count": 2,
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "1",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 3280762209n,
-          "data": "{"id":"t2","description":"Test 2"}",
-          "object_id": "t2",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "2",
-          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
-        },
-      ],
-      "has_more": false,
-      "next_after": "2",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "2",
-    },
-  },
-]
-`;